Links

Setting Filters

Array attributes and dimensions accept compressors and other filters.

Creating a Filter List

Compression

The following example shows how to create a filter list with a GZIP compressor and compression level 10.
C
C++
Python
R
Java
Go
#include <tiledb/tiledb.h>
// Create context
tiledb_ctx_t* ctx;
tiledb_ctx_alloc(nullptr, &ctx);
// Create compressor as a filter
tiledb_filter_t* filter;
tiledb_filter_alloc(ctx, TILEDB_FILTER_GZIP, &filter);
int level = 10;
tiledb_filter_set_option(ctx, filter, TILEDB_COMPRESSION_LEVEL, &level);
// Create filter list
tiledb_filter_list_t* filter_list;
tiledb_filter_list_alloc(ctx, &filter_list);
// Add compressor to filter list
tiledb_filter_list_add_filter(ctx, filter_list, filter);
// Clean up
tiledb_filter_free(&filter);
tiledb_filter_list_free(&filter_list);
tiledb_ctx_free(&ctx);
#include <tiledb/tiledb>
using namespace tiledb;
// Create context
Context ctx;
// Create compressor as a filter
Filter filter(ctx, TILEDB_FILTER_GZIP);
filter.set_option(ctx, TILEDB_COMPRESSION_LEVEL, 10);
// Create filter list
FilterList filter_list(ctx);
// Add compressor to filter list
filter_list.add_filter(filter)
import tiledb
# tiledb.FilterList accepts an iterable of zero or more filters:
filter_list = tiledb.FilterList([tiledb.GZipFilter(level=10)])
# create a "GZIP" compression filter
flt <- tiledb_filter("GZIP")
# set the option 'COMPRESSION_LEVEL' to 10
tiledb_filter_set_option(flt, "COMPRESSION_LEVEL", 10)
# create a filter list with this filter
fltlst <- tiledb_filter_list(flt)
// Create context
Context ctx = new Context();
// Create compressor as a filter
Filter filter = new GzipFilter(ctx, 10);
// Create filter list
FilterList filterList = new FilterList(ctx);
// Add compressor to filter list
filterList.addFilter(filter);
// Create context
context, _ := NewContext(nil)
// Create compressor as a filter
filter, _ := NewFilter(context, TILEDB_FILTER_GZIP)
filter.SetOption(TILEDB_COMPRESSION_LEVEL, int32(10))
// Create filter list
filterList, _ := NewFilterList(context)
// Add compressor to filter list
filterList.AddFilter(filter)
Supported compressors:
C/C++/Go
Python
R
Java
Compressor
Description
Option
TILEDB_FILTER_GZIP
GZIP
TILEDB_COMPRESSION_LEVEL (int)
TILEDB_FILTER_ZSTD
ZStandard
TILEDB_COMPRESSION_LEVEL (int)
TILEDB_FILTER_LZ4
LZ4
TILEDB_COMPRESSION_LEVEL (int)
TILEDB_FILTER_RLE
RLE
TILEDB_COMPRESSION_LEVEL (int)
TILEDB_FILTER_BZIP2
BZIP2
TILEDB_COMPRESSION_LEVEL (int)
TILEDB_FILTER_DOUBLE_DELTA
Double Delta
None
Compressor
Description
Option
tiledb.GzipFilter()
GZIP
level (np.int32)
tiledb.ZstdFilter()
ZStandard
level (np.int32)
tiledb.LZ4Filter()
LZ4
level (np.int32)
tiledb.RleFilter()
Run-length Encoding
level (np.int32)
tiledb.Bzip2Filter()
BZIP2
level (np.int32)
tiledb.DoubleDeltaFilter()
Double Delta
None
Compressor
Description
Option
"GZIP"
GZIP
"COMPRESSION_LEVEL" (int32)
"ZSTD"
ZStandard
"COMPRESSION_LEVEL" (int32)
"LZ4"
LZ4
"COMPRESSION_LEVEL" (int32)
"RLE"
Run-length Encoding
"COMPRESSION_LEVEL" (int32)
"BZIP2"
BZIP2
"COMPRESSION_LEVEL" (int32)
"DOUBLE_DELTA"
Double Delta
None
Compressor
Description
Option
GzipFilter
GZIP
Level (int)
ZstdFilter
ZStandard
Level (int)
LZ4Filter
LZ4
Level (int)
RleFilter
Run-length encoding
Level (int)
Bzip2Filter
BZIP2
Level (int)
DoubleDeltaFilter
Double delta
None

Other Filters

TileDB supports some more filters:
C
C++
Python
R
Java
Go
#include <tiledb/tiledb.h>
// Create context
tiledb_ctx_t* ctx;
tiledb_ctx_alloc(nullptr, &ctx);
// Create filters
tiledb_filter_t *f1, *f2;
tiledb_filter_alloc(ctx, TILEDB_FILTER_BITWIDTH, &f1);
tiledb_filter_alloc(ctx, TILEDB_FILTER_ZSTD, &f2);
// Create filter list
tiledb_filter_list_t* filter_list;
tiledb_filter_list_alloc(ctx, &filter_list);
// Add filters to filter list
tiledb_filter_list_add_filter(ctx, filter_list, f1);
tiledb_filter_list_add_filter(ctx, filter_list, f2);
// Clean up
tiledb_filter_free(&f1);
tiledb_filter_free(&f2);
tiledb_filter_list_free(&filter_list);
tiledb_ctx_free(&ctx);
#include <tiledb/tiledb>
using namespace tiledb;
// Create context
Context ctx;
// Create filters
Filter f1(ctx, TILEDB_FILTER_BITWIDTH);
Filter f2(ctx, TILEDB_FILTER_ZSTD);
// Create filter list
FilterList filter_list(ctx);
// Add filters to filter list
filter_list.add_filter(f1)
.add_filter(f2)
f1 = tiledb.BitWidthReductionFilter()
f2 = tiledb.ZstdFilter()
filter_list = tiledb.FilterList([f1, f2])
# create a filter list object with both
fltlst <- tiledb_filter_list(c(flt1, flt2))
// Create context
Context ctx = new Context();
// Create compressor as a filter
Filter f1 = new BitWidthReductionFilter(ctx);
Filter f2 = new ZstdFilter(ctx);
// Create filter list
FilterList filterList = new FilterList(ctx);
// Add compressor to filter list
filterList.addFilter(f1);
filterList.addFilter(f2);
// Create context
context, _ := NewContext(nil)
// Create filters
filter1, _ := NewFilter(context, TILEDB_FILTER_BIT_WIDTH_REDUCTION)
filter2, _ := NewFilter(context, TILEDB_FILTER_ZSTD)
// Create filter list
filterList, _ := NewFilterList(context)
// Add filters to filter list
filterList.AddFilter(filter1)
filterList.AddFilter(filter2)
Supported filters (beyond compressors):
C/C++/Go
Python
R
Java
Filter
Description
Option
TILEDB_BIT_WIDTH_REDUCTION
Bit width reduction
TILEDB_BIT_WIDTH_MAX_WINDOW (uint32)
TILEDB_BITSHUFFLE
Bit shuffle
None
TILEDB_BYTESHUFFLE
Byte shuffle
None
TILEDB_POSITIVE_DELTA
Positive Delta
TILEDB_POSITIVE_DELTA_MAX_WINDOW (uint32)
Filter
Description
Option
tiledb.BitWidthReductionFilter()
Bit width reduction
window (np.uint32)
tiledb.BitShuffleFilter()
Bit shuffle
None
tiledb.ByteShuffleFilter()
Byte shuffle
None
tiledb.PositiveDeltaFilter()
Positive delta
window (np.uint32)
Filter
Description
Option
BIT_WIDTH_REDUCTION
Bit width reduction
BIT_WIDTH_MAX_WINDOW (int)
BITSHUFFLE
Bit shuffle
None
BYTESHUFFLE
Byte shuffle
None
POSITIVE_DELTA
Positive Delta
POSITIVE_DELTA_MAX_WINDOW (int)
Filter
Description
Option
BitWidthReductionFilter
Bit width reduction
window (int)
BitShuffleFilter
Bit shuffle
None
ByteShuffleFilter
Byte shuffle
None
PositiveDeltaFilter
Positive Delta
window (int)

Setting the Tile Chunk Size

The default tile chunk size used by TileDB is 64KB, which is the size of many common processor L1 caches. You can control the chunk size by changing the option on a filter list:
C
C++
Python
R
Java
Go
// ... create context and filter list
tiledb_filter_list_set_max_chunk_size(ctx, filter_list, 10000);
// ... create filter list
filter_list.set_max_chunk_size(10000)
import tiledb
import numpy as np
filter_list = tiledb.FilterList([tiledb.GzipFilter()], chunksize=10000)
# ... create filter list
set_max_chunk_size(filter_list, 10000)
// ... create filter list
filter_list.setMaxChunkSize(10000)
// ... create filter list
filter_list.SetMaxChunkSize(uint32(10000))

Setting a Filter List for an Attribute

This is done as follows:
C
C++
Python
R
Java
Go
// ... create context, filter list and attribute
// Set filter list to attribute
tiledb_attribute_set_filter_list(ctx, attr, filter_list);
// ... create context, filter list and attribute
// Set filter list to attribute
attr.set_filter_list(filter_list);
# A FilterList may be passed to the Attr constructor
attr = tiledb.Attr(filters=filter_list)
# create (or access) an attribute
attr <- tiledb_attr("a", "INT32")
# create a filter list
flt1 <- tiledb_filter("BIT_WIDTH_REDUCTION")
flt2 <- tiledb_filter("ZSTD")
fltlst <- tiledb_filter_list(c(flt1, flt2))
# set the filter list
filter_list(attr) <- fltlst
// ... create context, filter list and attribute
// Set filter list to attribute
attr.setFilterList(filterList);
// ... create context, filter list and attribute
// Set filter list to attribute
attr.SetFilterList(filter_list);

Setting a Filter List for a Dimension

This is done as follows:
C
C++
Python
R
Java
Go
// ... create context, filter list and dimension
// Set filter list to dimension
tiledb_dimension_set_filter_list(ctx, dim, filter_list);
// ... create context, filter list and dimension
// Set filter list to dimension
dim.set_filter_list(filter_list);
# A FilterList may be passed to the Dim constructor
dim = tiledb.Dim(filters=filter_list)
d <- tiledb_dim("d", c(1L, 10L), 1L, "INT32")
# create a filter list
flt1 <- tiledb_filter("BIT_WIDTH_REDUCTION")
flt2 <- tiledb_filter("ZSTD")
fltlst <- tiledb_filter_list(c(flt1, flt2))
# assign the filter list
filter_list(d) <- fltlst
// ... create context, filter list and dimension
// Set filter list to dimension
dim.setFilterList(filterList);
// ... create context, filter list and dimension
// Set filter list to dimension
dim.SetFilterList(filter_list);

Setting a Filter List for All Dimensions

If you wish all the dimensions to have the same filter list, you can add
C
C++
Python
R
Java
Go
// ... create context ctx
// ... create array schema
// ... create filter list fl
// Set coordinates filter list
tiledb_array_schema_set_coords_filter_list(ctx, schema, fl);
// ... create context ctx
// ... create array schema
// ... create filter list fl
// Set coordinates filter lists
schema.set_coords_filter_list(fl);
import tiledb
import numpy as np
# ... create domain dom
# ... create attributes attr1, attr2
# ... create filter list fl
# Create the schema setting the coordinates filter list
schema = tiledb.ArraySchema(domain=dom, sparse=False, attrs=[attr1, attr2],
coords_filters=fl)
# ... create (or retrieve) array schema sch
# ... create filter list fl
# assign filter list to schema
tiledb_array_schema_set_coords_filter_list(sch, fl)
# Alternatively create the schema and set the coordinates filter list
sch <- tiledb_array_schema(dom, c(attr1, attr2),
coords_filter_list = fl)
// ... create array schema
// ... create filter list fl
// Set coordinates filter list
schema.setCoordsFilterList(fl);
// ... create array schema
// ... create filter list fl
// Set coordinates filter list
schema.SetCoordsFilterList(fl);
If you do not specify a filter for a dimension separately, the dimension will inherit the filters set to all dimensions collectively as shown above.

Setting a Filter List for Variable-sized Value Offsets

You can set filter lists for the offsets of variable-sized attributes or dimensions as follows:
C
C++
Python
R
Java
Go
#import <tiledb/tiledb.h>
// ... create context ctx
// ... create array schema
// ... create filter list fl
// Set offsets filter list
tiledb_array_schema_set_offsets_filter_list(ctx, schema, fl);
#import <tiledb/tiledb>
using namespace tiledb;
// ... create context ctx
// ... create array schema
// ... create filter list fl
// Set offsets filter list
schema.set_offsets_filter_list(fl);
import tiledb
import numpy as np
# ... create domain dom
# ... create attributes attr1, attr2
# ... create filter list fl
# Create the schema setting the offsets filter list
schema = tiledb.ArraySchema(domain=dom, sparse=False, attrs=[attr1, attr2],
offsets_filters=fl)
# ... create (or retrieve) array schema sch
# ... create filter list fl
# assign filter list to schema
tiledb_array_schema_set_offsets_filter_list(sch, fl)
# Create the schema setting the offsets filter list
sch <- tiledb_array_schema(dom, c(attr1, attr2),
offsets_filter_list = fl)
// ... create array schema
// ... create filter list fl
// Set offsets filter list
schema.setOffsetsFilterList(fl);
// ... create array schema
// ... create filter list fl
// Set offsets filter list
schema.SetOffsetsFilterList(fl);
The offset filters are applied to all variable-sized attributes and dimensions.