Setting Filters

Array attributes and dimensions accept compressors and other filters.

Creating a Filter List

Compression

The following example shows how to create a filter list with a GZIP compressor and compression level 10.

C

#include <tiledb/tiledb.h>

// Create context
tiledb_ctx_t* ctx;
tiledb_ctx_alloc(nullptr, &ctx);

// Create compressor as a filter
tiledb_filter_t* filter;
tiledb_filter_alloc(ctx, TILEDB_FILTER_GZIP, &filter);
int level = 10;
tiledb_filter_set_option(ctx, filter, TILEDB_COMPRESSION_LEVEL, &level);

// Create filter list
tiledb_filter_list_t* filter_list;
tiledb_filter_list_alloc(ctx, &filter_list);

// Add compressor to filter list
tiledb_filter_list_add_filter(ctx, filter_list, filter);

// Clean up
tiledb_filter_free(&filter);
tiledb_filter_list_free(&filter_list);
tiledb_ctx_free(&ctx);

C++

#include <tiledb/tiledb>
using namespace tiledb;

// Create context
Context ctx;

// Create compressor as a filter
Filter filter(ctx, TILEDB_FILTER_GZIP);
filter.set_option(ctx, TILEDB_COMPRESSION_LEVEL, 10);

// Create filter list
FilterList filter_list(ctx);

// Add compressor to filter list
filter_list.add_filter(filter)

Python

import tiledb

# tiledb.FilterList accepts an iterable of zero or more filters:
filter_list = tiledb.FilterList([tiledb.GZipFilter(level=10)])

R

# create a "GZIP" compression filter
flt <- tiledb_filter("GZIP")
# set the option 'COMPRESSION_LEVEL' to 10
tiledb_filter_set_option(flt, "COMPRESSION_LEVEL", 10)

# create a filter list with this filter
fltlst <- tiledb_filter_list(flt)

Java

// Create context
Context ctx = new Context();

// Create compressor as a filter
Filter filter = new GzipFilter(ctx, 10);

// Create filter list
FilterList filterList = new FilterList(ctx);

// Add compressor to filter list
filterList.addFilter(filter);

Go

// Create context
context, _ := NewContext(nil)

// Create compressor as a filter
filter, _ := NewFilter(context, TILEDB_FILTER_GZIP)
filter.SetOption(TILEDB_COMPRESSION_LEVEL, int32(10))

// Create filter list
filterList, _ := NewFilterList(context)

// Add compressor to filter list
filterList.AddFilter(filter)

C#

using TileDB.CSharp;

using Context ctx = new Context();
using Filter filter = new Filter(ctx, FilterType.Gzip);
using FilterList filterList = new FilterList(ctx);

filterList.AddFilter(filter);

Supported compressors:

C/C++/Go

Python

R

Java

C#

The compressors are members of the FilterType enum and the options of the FilterOption enum.

Other Filters

TileDB supports some more filters:

C

#include <tiledb/tiledb.h>

// Create context
tiledb_ctx_t* ctx;
tiledb_ctx_alloc(nullptr, &ctx);

// Create filters
tiledb_filter_t *f1, *f2;
tiledb_filter_alloc(ctx, TILEDB_FILTER_BITWIDTH, &f1);
tiledb_filter_alloc(ctx, TILEDB_FILTER_ZSTD, &f2);

// Create filter list
tiledb_filter_list_t* filter_list;
tiledb_filter_list_alloc(ctx, &filter_list);

// Add filters to filter list
tiledb_filter_list_add_filter(ctx, filter_list, f1);
tiledb_filter_list_add_filter(ctx, filter_list, f2);

// Clean up
tiledb_filter_free(&f1);
tiledb_filter_free(&f2);
tiledb_filter_list_free(&filter_list);
tiledb_ctx_free(&ctx);

C++

#include <tiledb/tiledb>
using namespace tiledb;

// Create context
Context ctx;

// Create filters
Filter f1(ctx, TILEDB_FILTER_BITWIDTH);
Filter f2(ctx, TILEDB_FILTER_ZSTD);

// Create filter list
FilterList filter_list(ctx);

// Add filters to filter list
filter_list.add_filter(f1)
           .add_filter(f2)

Python

f1 = tiledb.BitWidthReductionFilter()
f2 = tiledb.ZstdFilter()

filter_list = tiledb.FilterList([f1, f2])

R

# create a filter list object with both
fltlst <- tiledb_filter_list(c(flt1, flt2))

Java

// Create context
Context ctx = new Context();

// Create compressor as a filter
Filter f1 = new BitWidthReductionFilter(ctx);
Filter f2 = new ZstdFilter(ctx);

// Create filter list
FilterList filterList = new FilterList(ctx);

// Add compressor to filter list
filterList.addFilter(f1);
filterList.addFilter(f2);

Go

// Create context
context, _ := NewContext(nil)

// Create filters
filter1, _ := NewFilter(context, TILEDB_FILTER_BIT_WIDTH_REDUCTION)
filter2, _ := NewFilter(context, TILEDB_FILTER_ZSTD)

// Create filter list
filterList, _ := NewFilterList(context)

// Add filters to filter list
filterList.AddFilter(filter1)
filterList.AddFilter(filter2)

C#

using TileDB.CSharp;

using Context ctx = new Context();

using Filter filter1 = new Filter(ctx, FilterType.BitWidthReduction);
using Filter filter2 = new Filter(ctx, FilterType.Zstandard);

using FilterList filterList = new FilterList(ctx);

filterList.AddFilter(filter1);
filterList.AddFilter(filter2);

Supported filters (beyond compressors):

C/C++/Go

Python

R

Java

C#

Setting the Tile Chunk Size

The default tile chunk size used by TileDB is 64KB, which is the size of many common processor L1 caches. You can control the chunk size by changing the option on a filter list:

C

// ... create context and filter list
tiledb_filter_list_set_max_chunk_size(ctx, filter_list, 10000);

C++

// ... create filter list
filter_list.set_max_chunk_size(10000)

Python

import tiledb
import numpy as np

filter_list = tiledb.FilterList([tiledb.GzipFilter()], chunksize=10000)

R

# ... create filter list
set_max_chunk_size(filter_list, 10000)

Java

// ... create filter list
filter_list.setMaxChunkSize(10000)

Go

// ... create filter list
filter_list.SetMaxChunkSize(uint32(10000))

C#

// ... create filter list
filterList.SetMaxChunkSize(10000);

Setting a Filter List for an Attribute

This is done as follows:

C

// ... create context, filter list and attribute

// Set filter list to attribute
tiledb_attribute_set_filter_list(ctx, attr, filter_list);

C++

// ... create context, filter list and attribute

// Set filter list to attribute
attr.set_filter_list(filter_list);

Python

# A FilterList may be passed to the Attr constructor
attr = tiledb.Attr(filters=filter_list)

R

# create (or access) an attribute
attr <- tiledb_attr("a", "INT32")

# create a filter list
flt1 <- tiledb_filter("BIT_WIDTH_REDUCTION")
flt2 <- tiledb_filter("ZSTD")
fltlst <- tiledb_filter_list(c(flt1, flt2))

# set the filter list
filter_list(attr) <- fltlst

Java

// ... create context, filter list and attribute

// Set filter list to attribute
attr.setFilterList(filterList);

Go

// ... create context, filter list and attribute

// Set filter list to attribute
attr.SetFilterList(filter_list);

C#

// ... create filter list and attribute

// Set filter list to attribute
attr.SetFilterList(filterList);

Setting a Filter List for a Dimension

This is done as follows:

C

// ... create context, filter list and dimension

// Set filter list to dimension
tiledb_dimension_set_filter_list(ctx, dim, filter_list);

C++

// ... create context, filter list and dimension

// Set filter list to dimension
dim.set_filter_list(filter_list);

Python

# A FilterList may be passed to the Dim constructor
dim = tiledb.Dim(filters=filter_list)

R

d <- tiledb_dim("d", c(1L, 10L), 1L, "INT32")

# create a filter list
flt1 <- tiledb_filter("BIT_WIDTH_REDUCTION")
flt2 <- tiledb_filter("ZSTD") 
fltlst <- tiledb_filter_list(c(flt1, flt2))

# assign the filter list
filter_list(d) <- fltlst

Java

// ... create context, filter list and dimension

// Set filter list to dimension
dim.setFilterList(filterList);

Go

// ... create context, filter list and dimension

// Set filter list to dimension
dim.SetFilterList(filter_list);

C#

// ... create filter list and dimension

// Set filter list to dimension
dim.SetFilterList(filterList);

Setting a Filter List for All Dimensions

If you wish all the dimensions to have the same filter list, you can set it once as follows:

C

// ... create context ctx 
// ... create array schema
// ... create filter list fl

// Set coordinates filter list
tiledb_array_schema_set_coords_filter_list(ctx, schema, fl);

C++

// ... create context ctx 
// ... create array schema
// ... create filter list fl

// Set coordinates filter lists
schema.set_coords_filter_list(fl);

Python

import tiledb
import numpy as np

# ... create domain dom
# ... create attributes attr1, attr2
# ... create filter list fl

# Create the schema setting the coordinates filter list 
schema = tiledb.ArraySchema(domain=dom, sparse=False, attrs=[attr1, attr2], 
                            coords_filters=fl)

R

# ... create (or retrieve) array schema sch
# ... create filter list fl

# assign filter list to schema
tiledb_array_schema_set_coords_filter_list(sch, fl)

# Alternatively create the schema and set the coordinates filter list
sch <- tiledb_array_schema(dom, c(attr1, attr2),
                           coords_filter_list = fl)

Java

// ... create array schema
// ... create filter list fl

// Set coordinates filter list
schema.setCoordsFilterList(fl);

Go

// ... create array schema
// ... create filter list fl

// Set coordinates filter list
schema.SetCoordsFilterList(fl);

C#

// ... create array schema and filter list

schema.SetCoordsFilterList(filterList);

If you do not specify a filter for a dimension separately, the dimension will inherit the filters set to all dimensions collectively as shown above.

Setting a Filter List for Variable-sized Value Offsets

You can set filter lists for the offsets of variable-sized attributes or dimensions as follows:

C

#import <tiledb/tiledb.h>

// ... create context ctx 
// ... create array schema
// ... create filter list fl

// Set offsets filter list
tiledb_array_schema_set_offsets_filter_list(ctx, schema, fl);

C++

#import <tiledb/tiledb>
using namespace tiledb;

// ... create context ctx 
// ... create array schema
// ... create filter list fl

// Set offsets filter list
schema.set_offsets_filter_list(fl);

Python

import tiledb
import numpy as np

# ... create domain dom
# ... create attributes attr1, attr2
# ... create filter list fl

# Create the schema setting the offsets filter list
schema = tiledb.ArraySchema(domain=dom, sparse=False, attrs=[attr1, attr2], 
                            offsets_filters=fl)

R

# ... create (or retrieve) array schema sch
# ... create filter list fl

# assign filter list to schema
tiledb_array_schema_set_offsets_filter_list(sch, fl)

# Create the schema setting the offsets filter list
sch <- tiledb_array_schema(dom, c(attr1, attr2),
                           offsets_filter_list = fl)

Java

// ... create array schema
// ... create filter list fl

// Set offsets filter list
schema.setOffsetsFilterList(fl);

Go

// ... create array schema
// ... create filter list fl

// Set offsets filter list
schema.SetOffsetsFilterList(fl);

C#

// ... create array schema and filter list

schema.SetOffsetsFilterList(filterList);

The offset filters are applied to all variable-sized attributes and dimensions.