Setting Filters

Array attributes and dimensions accept compressors and other filters.

Creating a Filter List

Compression

The following example shows how to create a filter list with a GZIP compressor and compression level 10.

C

#include <tiledb/tiledb.h>

// Create context
tiledb_ctx_t* ctx;
tiledb_ctx_alloc(nullptr, &ctx);

// Create compressor as a filter
tiledb_filter_t* filter;
tiledb_filter_alloc(ctx, TILEDB_FILTER_GZIP, &filter);
int level = 10;
tiledb_filter_set_option(ctx, filter, TILEDB_COMPRESSION_LEVEL, &level);

// Create filter list
tiledb_filter_list_t* filter_list;
tiledb_filter_list_alloc(ctx, &filter_list);

// Add compressor to filter list
tiledb_filter_list_add_filter(ctx, filter_list, filter);

// Clean up
tiledb_filter_free(&filter);
tiledb_filter_list_free(&filter_list);
tiledb_ctx_free(&ctx);

C++

#include <tiledb/tiledb>
using namespace tiledb;

// Create context
Context ctx;

// Create compressor as a filter
Filter filter(ctx, TILEDB_FILTER_GZIP);
filter.set_option(ctx, TILEDB_COMPRESSION_LEVEL, 10);

// Create filter list
FilterList filter_list(ctx);

// Add compressor to filter list
filter_list.add_filter(filter)

Python

import tiledb

# tiledb.FilterList accepts an iterable of zero or more filters:
filter_list = tiledb.FilterList([tiledb.GZipFilter(level=10)])

R

# create a "GZIP" compression filter
flt <- tiledb_filter("GZIP")
# set the option 'COMPRESSION_LEVEL' to 10
tiledb_filter_set_option(flt, "COMPRESSION_LEVEL", 10)

# create a filter list with this filter
fltlst <- tiledb_filter_list(flt)

Java

// Create context
Context ctx = new Context();

// Create compressor as a filter
Filter filter = new GzipFilter(ctx, 10);

// Create filter list
FilterList filterList = new FilterList(ctx);

// Add compressor to filter list
filterList.addFilter(filter);

Go

// Create context
context, _ := NewContext(nil)

// Create compressor as a filter
filter, _ := NewFilter(context, TILEDB_FILTER_GZIP)
filter.SetOption(TILEDB_COMPRESSION_LEVEL, int32(10))

// Create filter list
filterList, _ := NewFilterList(context)

// Add compressor to filter list
filterList.AddFilter(filter)

C#

using TileDB.CSharp;

using Context ctx = new Context();
using Filter filter = new Filter(ctx, FilterType.Gzip);
using FilterList filterList = new FilterList(ctx);

filterList.AddFilter(filter);

Supported compressors:

C/C++/Go

Compressor	Description	Option (type)
TILEDB_FILTER_GZIP	GZIP	TILEDB_COMPRESSION_LEVEL (int)
TILEDB_FILTER_ZSTD	Zstandard	TILEDB_COMPRESSION_LEVEL (int)
TILEDB_FILTER_LZ4	LZ4	TILEDB_COMPRESSION_LEVEL (int)
TILEDB_FILTER_RLE	RLE	TILEDB_COMPRESSION_LEVEL (int)
TILEDB_FILTER_BZIP2	BZIP2	TILEDB_COMPRESSION_LEVEL (int)
TILEDB_FILTER_DOUBLE_DELTA	Double Delta	None

Python

Compressor	Description	Option (type)
tiledb.GzipFilter()	GZIP	level (np.int32)
tiledb.ZstdFilter()	Zstandard	level (np.int32)
tiledb.LZ4Filter()	LZ4	level (np.int32)
tiledb.RleFilter()	Run-length Encoding	level (np.int32)
tiledb.Bzip2Filter()	BZIP2	level (np.int32)
tiledb.DoubleDeltaFilter()	Double Delta	None

R

Compressor	Description	Option (type)
"GZIP"	GZIP	"COMPRESSION_LEVEL" (int32)
"ZSTD"	Zstandard	"COMPRESSION_LEVEL" (int32)
"LZ4"	LZ4	"COMPRESSION_LEVEL" (int32)
"RLE"	Run-length Encoding	"COMPRESSION_LEVEL" (int32)
"BZIP2"	BZIP2	"COMPRESSION_LEVEL" (int32)
"DOUBLE_DELTA"	Double Delta	None

Java

Compressor	Description	Option (type)
GzipFilter	GZIP	Level (int)
ZstdFilter	Zstandard	Level (int)
LZ4Filter	LZ4	Level (int)
RleFilter	Run-length encoding	Level (int)
Bzip2Filter	BZIP2	Level (int)
DoubleDeltaFilter	Double delta	None

C#

The compressors are members of the FilterType enum and the options of the FilterOption enum.

Compressor	Description	Option (type)
Gzip	GZIP	CompressionLevel (int)
Zstandard	Zstandard	CompressionLevel (int)
Lz4	LZ4	CompressionLevel (int)
RunLengthEncoding	RLE	CompressionLevel (int)
Bzip2	BZIP2	CompressionLevel (int)
DoubleDelta	Double Delta	None

Other Filters

TileDB supports some more filters:

C

#include <tiledb/tiledb.h>

// Create context
tiledb_ctx_t* ctx;
tiledb_ctx_alloc(nullptr, &ctx);

// Create filters
tiledb_filter_t *f1, *f2;
tiledb_filter_alloc(ctx, TILEDB_FILTER_BITWIDTH, &f1);
tiledb_filter_alloc(ctx, TILEDB_FILTER_ZSTD, &f2);

// Create filter list
tiledb_filter_list_t* filter_list;
tiledb_filter_list_alloc(ctx, &filter_list);

// Add filters to filter list
tiledb_filter_list_add_filter(ctx, filter_list, f1);
tiledb_filter_list_add_filter(ctx, filter_list, f2);

// Clean up
tiledb_filter_free(&f1);
tiledb_filter_free(&f2);
tiledb_filter_list_free(&filter_list);
tiledb_ctx_free(&ctx);

C++

#include <tiledb/tiledb>
using namespace tiledb;

// Create context
Context ctx;

// Create filters
Filter f1(ctx, TILEDB_FILTER_BITWIDTH);
Filter f2(ctx, TILEDB_FILTER_ZSTD);

// Create filter list
FilterList filter_list(ctx);

// Add filters to filter list
filter_list.add_filter(f1)
           .add_filter(f2)

Python

f1 = tiledb.BitWidthReductionFilter()
f2 = tiledb.ZstdFilter()

filter_list = tiledb.FilterList([f1, f2])

R

# create a filter list object with both
fltlst <- tiledb_filter_list(c(flt1, flt2))

Java

// Create context
Context ctx = new Context();

// Create compressor as a filter
Filter f1 = new BitWidthReductionFilter(ctx);
Filter f2 = new ZstdFilter(ctx);

// Create filter list
FilterList filterList = new FilterList(ctx);

// Add compressor to filter list
filterList.addFilter(f1);
filterList.addFilter(f2);

Go

// Create context
context, _ := NewContext(nil)

// Create filters
filter1, _ := NewFilter(context, TILEDB_FILTER_BIT_WIDTH_REDUCTION)
filter2, _ := NewFilter(context, TILEDB_FILTER_ZSTD)

// Create filter list
filterList, _ := NewFilterList(context)

// Add filters to filter list
filterList.AddFilter(filter1)
filterList.AddFilter(filter2)

C#

using TileDB.CSharp;

using Context ctx = new Context();

using Filter filter1 = new Filter(ctx, FilterType.BitWidthReduction);
using Filter filter2 = new Filter(ctx, FilterType.Zstandard);

using FilterList filterList = new FilterList(ctx);

filterList.AddFilter(filter1);
filterList.AddFilter(filter2);

Supported filters (beyond compressors):

C/C++/Go

Filter	Description	Option
TILEDB_BIT_WIDTH_REDUCTION	Bit width reduction	TILEDB_BIT_WIDTH_MAX_WINDOW (uint32)
TILEDB_BITSHUFFLE	Bit shuffle	None
TILEDB_BYTESHUFFLE	Byte shuffle	None
TILEDB_POSITIVE_DELTA	Positive Delta	TILEDB_POSITIVE_DELTA_MAX_WINDOW (uint32)
TILEDB_FILTER_CHECKSUM_MD5	MD5 checksum filter	None
TILEDB_FILTER_CHECKSUM_SHA256	SHA256 checksum filter	None

Python

Filter	Description	Option
tiledb.BitWidthReductionFilter()	Bit width reduction	window (np.uint32)
tiledb.BitShuffleFilter()	Bit shuffle	None
tiledb.ByteShuffleFilter()	Byte shuffle	None
tiledb.PositiveDeltaFilter()	Positive delta	window (np.uint32)
tiledb.ChecksumMD5Filter()	MD5 Checksum filter	None
tiledb.ChecksumSHA256Filter()	SHA256 Checksum filter	None

R

Filter	Description	Option
BIT_WIDTH_REDUCTION	Bit width reduction	BIT_WIDTH_MAX_WINDOW (int)
BITSHUFFLE	Bit shuffle	None
BYTESHUFFLE	Byte shuffle	None
POSITIVE_DELTA	Positive Delta	POSITIVE_DELTA_MAX_WINDOW (int)
CHECKSUM_MD5	MD5 Checksum filter	None
CHECKSUM_SHA256	SHA256 Checksum filter	None

Java

Filter	Description	Option
BitWidthReductionFilter	Bit width reduction	window (int)
BitShuffleFilter	Bit shuffle	None
ByteShuffleFilter	Byte shuffle	None
PositiveDeltaFilter	Positive Delta	window (int)
CheckSumMD5Filter	MD5 Checksum filter	None
CheckSumSHA256Filter	SHA256 Checksum filter	None

C#

Filter	Description	Option
BitWidthReduction	Bit width reduction	BitWidthMaxWindow (uint32)
BitShuffle	Bit shuffle	None
ByteShuffle	Byte shuffle	None
PositiveDelta	Positive Delta	PositiveDeltaMaxWindow (uint32)
ChecksumMd5	MD5 Checksum filter	None
ChecksumSha256	SHA256 Checksum filter	None

Setting the Tile Chunk Size

The default tile chunk size used by TileDB is 64KB, which is the size of many common processor L1 caches. You can control the chunk size by changing the option on a filter list:

C

// ... create context and filter list
tiledb_filter_list_set_max_chunk_size(ctx, filter_list, 10000);

C++

// ... create filter list
filter_list.set_max_chunk_size(10000)

Python

import tiledb
import numpy as np

filter_list = tiledb.FilterList([tiledb.GzipFilter()], chunksize=10000)

R

# ... create filter list
set_max_chunk_size(filter_list, 10000)

Java

// ... create filter list
filter_list.setMaxChunkSize(10000)

Go

// ... create filter list
filter_list.SetMaxChunkSize(uint32(10000))

C#

// ... create filter list
filterList.SetMaxChunkSize(10000);

Setting a Filter List for an Attribute

This is done as follows:

C

// ... create context, filter list and attribute

// Set filter list to attribute
tiledb_attribute_set_filter_list(ctx, attr, filter_list);

C++

// ... create context, filter list and attribute

// Set filter list to attribute
attr.set_filter_list(filter_list);

Python

# A FilterList may be passed to the Attr constructor
attr = tiledb.Attr(filters=filter_list)

R

# create (or access) an attribute
attr <- tiledb_attr("a", "INT32")

# create a filter list
flt1 <- tiledb_filter("BIT_WIDTH_REDUCTION")
flt2 <- tiledb_filter("ZSTD")
fltlst <- tiledb_filter_list(c(flt1, flt2))

# set the filter list
filter_list(attr) <- fltlst

Java

// ... create context, filter list and attribute

// Set filter list to attribute
attr.setFilterList(filterList);

Go

// ... create context, filter list and attribute

// Set filter list to attribute
attr.SetFilterList(filter_list);

C#

// ... create filter list and attribute

// Set filter list to attribute
attr.SetFilterList(filterList);

Setting a Filter List for a Dimension

This is done as follows:

C

// ... create context, filter list and dimension

// Set filter list to dimension
tiledb_dimension_set_filter_list(ctx, dim, filter_list);

C++

// ... create context, filter list and dimension

// Set filter list to dimension
dim.set_filter_list(filter_list);

Python

# A FilterList may be passed to the Dim constructor
dim = tiledb.Dim(filters=filter_list)

R

d <- tiledb_dim("d", c(1L, 10L), 1L, "INT32")

# create a filter list
flt1 <- tiledb_filter("BIT_WIDTH_REDUCTION")
flt2 <- tiledb_filter("ZSTD") 
fltlst <- tiledb_filter_list(c(flt1, flt2))

# assign the filter list
filter_list(d) <- fltlst

Java

// ... create context, filter list and dimension

// Set filter list to dimension
dim.setFilterList(filterList);

Go

// ... create context, filter list and dimension

// Set filter list to dimension
dim.SetFilterList(filter_list);

C#

// ... create filter list and dimension

// Set filter list to dimension
dim.SetFilterList(filterList);

Setting a Filter List for All Dimensions

If you wish all the dimensions to have the same filter list, you can set it once as follows:

C

// ... create context ctx 
// ... create array schema
// ... create filter list fl

// Set coordinates filter list
tiledb_array_schema_set_coords_filter_list(ctx, schema, fl);

C++

// ... create context ctx 
// ... create array schema
// ... create filter list fl

// Set coordinates filter lists
schema.set_coords_filter_list(fl);

Python

import tiledb
import numpy as np

# ... create domain dom
# ... create attributes attr1, attr2
# ... create filter list fl

# Create the schema setting the coordinates filter list 
schema = tiledb.ArraySchema(domain=dom, sparse=False, attrs=[attr1, attr2], 
                            coords_filters=fl)

R

# ... create (or retrieve) array schema sch
# ... create filter list fl

# assign filter list to schema
tiledb_array_schema_set_coords_filter_list(sch, fl)

# Alternatively create the schema and set the coordinates filter list
sch <- tiledb_array_schema(dom, c(attr1, attr2),
                           coords_filter_list = fl)

Java

// ... create array schema
// ... create filter list fl

// Set coordinates filter list
schema.setCoordsFilterList(fl);

Go

// ... create array schema
// ... create filter list fl

// Set coordinates filter list
schema.SetCoordsFilterList(fl);

C#

// ... create array schema and filter list

schema.SetCoordsFilterList(filterList);

If you do not specify a filter for a dimension separately, the dimension will inherit the filters set to all dimensions collectively as shown above.

Setting a Filter List for Variable-sized Value Offsets

You can set filter lists for the offsets of variable-sized attributes or dimensions as follows:

C

#import <tiledb/tiledb.h>

// ... create context ctx 
// ... create array schema
// ... create filter list fl

// Set offsets filter list
tiledb_array_schema_set_offsets_filter_list(ctx, schema, fl);

C++

#import <tiledb/tiledb>
using namespace tiledb;

// ... create context ctx 
// ... create array schema
// ... create filter list fl

// Set offsets filter list
schema.set_offsets_filter_list(fl);

Python

import tiledb
import numpy as np

# ... create domain dom
# ... create attributes attr1, attr2
# ... create filter list fl

# Create the schema setting the offsets filter list
schema = tiledb.ArraySchema(domain=dom, sparse=False, attrs=[attr1, attr2], 
                            offsets_filters=fl)

R

# ... create (or retrieve) array schema sch
# ... create filter list fl

# assign filter list to schema
tiledb_array_schema_set_offsets_filter_list(sch, fl)

# Create the schema setting the offsets filter list
sch <- tiledb_array_schema(dom, c(attr1, attr2),
                           offsets_filter_list = fl)

Java

// ... create array schema
// ... create filter list fl

// Set offsets filter list
schema.setOffsetsFilterList(fl);

Go

// ... create array schema
// ... create filter list fl

// Set offsets filter list
schema.SetOffsetsFilterList(fl);

C#

// ... create array schema and filter list

schema.SetOffsetsFilterList(filterList);

The offset filters are applied to all variable-sized attributes and dimensions.