Setting Filters

Array attributes and dimensions accept compressors and other filters.

Creating a Filter List

Compression

The following example shows how to create a filter list with a GZIP compressor and compression level 10.

#include <tiledb/tiledb.h>

// Create context
tiledb_ctx_t* ctx;
tiledb_ctx_alloc(nullptr, &ctx);

// Create compressor as a filter
tiledb_filter_t* filter;
tiledb_filter_alloc(ctx, TILEDB_FILTER_GZIP, &filter);
int level = 10;
tiledb_filter_set_option(ctx, filter, TILEDB_COMPRESSION_LEVEL, &level);

// Create filter list
tiledb_filter_list_t* filter_list;
tiledb_filter_list_alloc(ctx, &filter_list);

// Add compressor to filter list
tiledb_filter_list_add_filter(ctx, filter_list, filter);

// Clean up
tiledb_filter_free(&filter);
tiledb_filter_list_free(&filter_list);
tiledb_ctx_free(&ctx);

#include <tiledb/tiledb>
using namespace tiledb;

// Create context
Context ctx;

// Create compressor as a filter
Filter filter(ctx, TILEDB_FILTER_GZIP);
filter.set_option(ctx, TILEDB_COMPRESSION_LEVEL, 10);

// Create filter list
FilterList filter_list(ctx);

// Add compressor to filter list
filter_list.add_filter(filter)

import tiledb

# tiledb.FilterList accepts an iterable of zero or more filters:
filter_list = tiledb.FilterList([tiledb.GZipFilter(level=10)])

# create a "GZIP" compression filter
flt <- tiledb_filter("GZIP")
# set the option 'COMPRESSION_LEVEL' to 10
tiledb_filter_set_option(flt, "COMPRESSION_LEVEL", 10)

# create a filter list with this filter
fltlst <- tiledb_filter_list(flt)

// Create context
Context ctx = new Context();

// Create compressor as a filter
Filter filter = new GzipFilter(ctx, 10);

// Create filter list
FilterList filterList = new FilterList(ctx);

// Add compressor to filter list
filterList.addFilter(filter);

// Create context
context, _ := NewContext(nil)

// Create compressor as a filter
filter, _ := NewFilter(context, TILEDB_FILTER_GZIP)
filter.SetOption(TILEDB_COMPRESSION_LEVEL, int32(10))

// Create filter list
filterList, _ := NewFilterList(context)

// Add compressor to filter list
filterList.AddFilter(filter)

using TileDB.CSharp;

using Context ctx = new Context();
using Filter filter = new Filter(ctx, FilterType.Gzip);
using FilterList filterList = new FilterList(ctx);

filterList.AddFilter(filter);

Supported compressors:

Compressor

Description

Option (type)

TILEDB_FILTER_GZIP

GZIP

TILEDB_COMPRESSION_LEVEL (int)

TILEDB_FILTER_ZSTD

Zstandard

TILEDB_COMPRESSION_LEVEL (int)

TILEDB_FILTER_LZ4

LZ4

TILEDB_COMPRESSION_LEVEL (int)

TILEDB_FILTER_RLE

RLE

TILEDB_COMPRESSION_LEVEL (int)

TILEDB_FILTER_BZIP2

BZIP2

TILEDB_COMPRESSION_LEVEL (int)

TILEDB_FILTER_DOUBLE_DELTA

Double Delta

None

Compressor

Description

Option (type)

tiledb.GzipFilter()

GZIP

level (np.int32)

tiledb.ZstdFilter()

Zstandard

level (np.int32)

tiledb.LZ4Filter()

LZ4

level (np.int32)

tiledb.RleFilter()

Run-length Encoding

level (np.int32)

tiledb.Bzip2Filter()

BZIP2

level (np.int32)

tiledb.DoubleDeltaFilter()

Double Delta

None

Compressor

Description

Option (type)

"GZIP"

GZIP

"COMPRESSION_LEVEL" (int32)

"ZSTD"

Zstandard

"COMPRESSION_LEVEL" (int32)

"LZ4"

LZ4

"COMPRESSION_LEVEL" (int32)

"RLE"

Run-length Encoding

"COMPRESSION_LEVEL" (int32)

"BZIP2"

BZIP2

"COMPRESSION_LEVEL" (int32)

"DOUBLE_DELTA"

Double Delta

None

Compressor

Description

Option (type)

GzipFilter

GZIP

Level (int)

ZstdFilter

Zstandard

Level (int)

LZ4Filter

LZ4

Level (int)

RleFilter

Run-length encoding

Level (int)

Bzip2Filter

BZIP2

Level (int)

DoubleDeltaFilter

Double delta

None

The compressors are members of the FilterType enum and the options of the FilterOption enum.

Compressor

Description

Option (type)

Gzip

GZIP

CompressionLevel (int)

Zstandard

Zstandard

CompressionLevel (int)

Lz4

LZ4

CompressionLevel (int)

RunLengthEncoding

RLE

CompressionLevel (int)

Bzip2

BZIP2

CompressionLevel (int)

DoubleDelta

Double Delta

None

Other Filters

TileDB supports some more filters:

#include <tiledb/tiledb.h>

// Create context
tiledb_ctx_t* ctx;
tiledb_ctx_alloc(nullptr, &ctx);

// Create filters
tiledb_filter_t *f1, *f2;
tiledb_filter_alloc(ctx, TILEDB_FILTER_BITWIDTH, &f1);
tiledb_filter_alloc(ctx, TILEDB_FILTER_ZSTD, &f2);

// Create filter list
tiledb_filter_list_t* filter_list;
tiledb_filter_list_alloc(ctx, &filter_list);

// Add filters to filter list
tiledb_filter_list_add_filter(ctx, filter_list, f1);
tiledb_filter_list_add_filter(ctx, filter_list, f2);

// Clean up
tiledb_filter_free(&f1);
tiledb_filter_free(&f2);
tiledb_filter_list_free(&filter_list);
tiledb_ctx_free(&ctx);

#include <tiledb/tiledb>
using namespace tiledb;

// Create context
Context ctx;

// Create filters
Filter f1(ctx, TILEDB_FILTER_BITWIDTH);
Filter f2(ctx, TILEDB_FILTER_ZSTD);

// Create filter list
FilterList filter_list(ctx);

// Add filters to filter list
filter_list.add_filter(f1)
           .add_filter(f2)

f1 = tiledb.BitWidthReductionFilter()
f2 = tiledb.ZstdFilter()

filter_list = tiledb.FilterList([f1, f2])

# create a filter list object with both
fltlst <- tiledb_filter_list(c(flt1, flt2))

// Create context
Context ctx = new Context();

// Create compressor as a filter
Filter f1 = new BitWidthReductionFilter(ctx);
Filter f2 = new ZstdFilter(ctx);

// Create filter list
FilterList filterList = new FilterList(ctx);

// Add compressor to filter list
filterList.addFilter(f1);
filterList.addFilter(f2);

// Create context
context, _ := NewContext(nil)

// Create filters
filter1, _ := NewFilter(context, TILEDB_FILTER_BIT_WIDTH_REDUCTION)
filter2, _ := NewFilter(context, TILEDB_FILTER_ZSTD)

// Create filter list
filterList, _ := NewFilterList(context)

// Add filters to filter list
filterList.AddFilter(filter1)
filterList.AddFilter(filter2)

using TileDB.CSharp;

using Context ctx = new Context();

using Filter filter1 = new Filter(ctx, FilterType.BitWidthReduction);
using Filter filter2 = new Filter(ctx, FilterType.Zstandard);

using FilterList filterList = new FilterList(ctx);

filterList.AddFilter(filter1);
filterList.AddFilter(filter2);

Supported filters (beyond compressors):

Filter

Description

Option

TILEDB_BIT_WIDTH_REDUCTION

Bit width reduction

TILEDB_BIT_WIDTH_MAX_WINDOW (uint32)

TILEDB_BITSHUFFLE

Bit shuffle

None

TILEDB_BYTESHUFFLE

Byte shuffle

None

TILEDB_POSITIVE_DELTA

Positive Delta

TILEDB_POSITIVE_DELTA_MAX_WINDOW (uint32)

TILEDB_FILTER_CHECKSUM_MD5

MD5 checksum filter

None

TILEDB_FILTER_CHECKSUM_SHA256

SHA256 checksum filter

None

Filter

Description

Option

tiledb.BitWidthReductionFilter()

Bit width reduction

window (np.uint32)

tiledb.BitShuffleFilter()

Bit shuffle

None

tiledb.ByteShuffleFilter()

Byte shuffle

None

tiledb.PositiveDeltaFilter()

Positive delta

window (np.uint32)

tiledb.ChecksumMD5Filter()

MD5 Checksum filter

None

tiledb.ChecksumSHA256Filter()

SHA256 Checksum filter

None

Filter

Description

Option

BIT_WIDTH_REDUCTION

Bit width reduction

BIT_WIDTH_MAX_WINDOW (int)

BITSHUFFLE

Bit shuffle

None

BYTESHUFFLE

Byte shuffle

None

POSITIVE_DELTA

Positive Delta

POSITIVE_DELTA_MAX_WINDOW (int)

CHECKSUM_MD5

MD5 Checksum filter

None

CHECKSUM_SHA256

SHA256 Checksum filter

None

Filter

Description

Option

BitWidthReductionFilter

Bit width reduction

window (int)

BitShuffleFilter

Bit shuffle

None

ByteShuffleFilter

Byte shuffle

None

PositiveDeltaFilter

Positive Delta

window (int)

CheckSumMD5Filter

MD5 Checksum filter

None

CheckSumSHA256Filter

SHA256 Checksum filter

None

Filter

Description

Option

BitWidthReduction

Bit width reduction

BitWidthMaxWindow (uint32)

BitShuffle

Bit shuffle

None

ByteShuffle

Byte shuffle

None

PositiveDelta

Positive Delta

PositiveDeltaMaxWindow (uint32)

ChecksumMd5

MD5 Checksum filter

None

ChecksumSha256

SHA256 Checksum filter

None

Setting the Tile Chunk Size

The default tile chunk size used by TileDB is 64KB, which is the size of many common processor L1 caches. You can control the chunk size by changing the option on a filter list:

// ... create context and filter list
tiledb_filter_list_set_max_chunk_size(ctx, filter_list, 10000);

// ... create filter list
filter_list.set_max_chunk_size(10000)

import tiledb
import numpy as np

filter_list = tiledb.FilterList([tiledb.GzipFilter()], chunksize=10000)

# ... create filter list
set_max_chunk_size(filter_list, 10000)

// ... create filter list
filter_list.setMaxChunkSize(10000)

// ... create filter list
filter_list.SetMaxChunkSize(uint32(10000))

// ... create filter list
filterList.SetMaxChunkSize(10000);

Setting a Filter List for an Attribute

This is done as follows:

// ... create context, filter list and attribute

// Set filter list to attribute
tiledb_attribute_set_filter_list(ctx, attr, filter_list);

// ... create context, filter list and attribute

// Set filter list to attribute
attr.set_filter_list(filter_list);

# A FilterList may be passed to the Attr constructor
attr = tiledb.Attr(filters=filter_list)

# create (or access) an attribute
attr <- tiledb_attr("a", "INT32")

# create a filter list
flt1 <- tiledb_filter("BIT_WIDTH_REDUCTION")
flt2 <- tiledb_filter("ZSTD")
fltlst <- tiledb_filter_list(c(flt1, flt2))

# set the filter list
filter_list(attr) <- fltlst

// ... create context, filter list and attribute

// Set filter list to attribute
attr.setFilterList(filterList);

// ... create context, filter list and attribute

// Set filter list to attribute
attr.SetFilterList(filter_list);

// ... create filter list and attribute

// Set filter list to attribute
attr.SetFilterList(filterList);

Setting a Filter List for a Dimension

This is done as follows:

// ... create context, filter list and dimension

// Set filter list to dimension
tiledb_dimension_set_filter_list(ctx, dim, filter_list);

// ... create context, filter list and dimension

// Set filter list to dimension
dim.set_filter_list(filter_list);

# A FilterList may be passed to the Dim constructor
dim = tiledb.Dim(filters=filter_list)

d <- tiledb_dim("d", c(1L, 10L), 1L, "INT32")

# create a filter list
flt1 <- tiledb_filter("BIT_WIDTH_REDUCTION")
flt2 <- tiledb_filter("ZSTD") 
fltlst <- tiledb_filter_list(c(flt1, flt2))

# assign the filter list
filter_list(d) <- fltlst

// ... create context, filter list and dimension

// Set filter list to dimension
dim.setFilterList(filterList);

// ... create context, filter list and dimension

// Set filter list to dimension
dim.SetFilterList(filter_list);

// ... create filter list and dimension

// Set filter list to dimension
dim.SetFilterList(filterList);

Setting a Filter List for All Dimensions

If you wish all the dimensions to have the same filter list, you can set it once as follows:

// ... create context ctx 
// ... create array schema
// ... create filter list fl

// Set coordinates filter list
tiledb_array_schema_set_coords_filter_list(ctx, schema, fl);

// ... create context ctx 
// ... create array schema
// ... create filter list fl

// Set coordinates filter lists
schema.set_coords_filter_list(fl);

import tiledb
import numpy as np

# ... create domain dom
# ... create attributes attr1, attr2
# ... create filter list fl

# Create the schema setting the coordinates filter list 
schema = tiledb.ArraySchema(domain=dom, sparse=False, attrs=[attr1, attr2], 
                            coords_filters=fl)

# ... create (or retrieve) array schema sch
# ... create filter list fl

# assign filter list to schema
tiledb_array_schema_set_coords_filter_list(sch, fl)

# Alternatively create the schema and set the coordinates filter list
sch <- tiledb_array_schema(dom, c(attr1, attr2),
                           coords_filter_list = fl)

// ... create array schema
// ... create filter list fl

// Set coordinates filter list
schema.setCoordsFilterList(fl);

// ... create array schema
// ... create filter list fl

// Set coordinates filter list
schema.SetCoordsFilterList(fl);

// ... create array schema and filter list

schema.SetCoordsFilterList(filterList);

If you do not specify a filter for a dimension separately, the dimension will inherit the filters set to all dimensions collectively as shown above.

Setting a Filter List for Variable-sized Value Offsets

You can set filter lists for the offsets of variable-sized attributes or dimensions as follows:

#import <tiledb/tiledb.h>

// ... create context ctx 
// ... create array schema
// ... create filter list fl

// Set offsets filter list
tiledb_array_schema_set_offsets_filter_list(ctx, schema, fl);

#import <tiledb/tiledb>
using namespace tiledb;

// ... create context ctx 
// ... create array schema
// ... create filter list fl

// Set offsets filter list
schema.set_offsets_filter_list(fl);

import tiledb
import numpy as np

# ... create domain dom
# ... create attributes attr1, attr2
# ... create filter list fl

# Create the schema setting the offsets filter list
schema = tiledb.ArraySchema(domain=dom, sparse=False, attrs=[attr1, attr2], 
                            offsets_filters=fl)

# ... create (or retrieve) array schema sch
# ... create filter list fl

# assign filter list to schema
tiledb_array_schema_set_offsets_filter_list(sch, fl)

# Create the schema setting the offsets filter list
sch <- tiledb_array_schema(dom, c(attr1, attr2),
                           offsets_filter_list = fl)

// ... create array schema
// ... create filter list fl

// Set offsets filter list
schema.setOffsetsFilterList(fl);

// ... create array schema
// ... create filter list fl

// Set offsets filter list
schema.SetOffsetsFilterList(fl);

// ... create array schema and filter list

schema.SetOffsetsFilterList(filterList);

The offset filters are applied to all variable-sized attributes and dimensions.

PreviousEvolving the Array Schema NextCreating the Array

Last updated 6 months ago