Nullable Attributes

Fixed-length, nullable attributes

Writing nullable, fixed-length attribute cells to an array involves populating a data buffer and a validity buffer. This is illustrated in the code block below with a sparse write, but it is applicable to dense writes as well.

C

#include <tiledb/tiledb.h>

// Create TileDB context
tiledb_ctx_t* ctx;
tiledb_ctx_alloc(NULL, &ctx);

// Open array for writing
tiledb_array_t* array;
tiledb_array_alloc(ctx, array_name, &array);
tiledb_array_open(ctx, array, TILEDB_WRITE);

// Prepare some data for the array
long long d1[] = {1, 2, 3, 4};
unsigned long long d1_size = sizeof(d1);
long long d2[] = {2, 1, 3, 4};
unsigned long long d2_size = sizeof(d2);
float a[] = {1.1f, 1.2f, 2.1f, 2.2f, 3.1f, 3.2f, 4.1f, 4.2f};
unsigned long long a_size = sizeof(a);
uint8_t a_validity[] = {1, 1, 0, 1};
unsigned long long a_validity_size = sizeof(a_validity);

// Create the query
tiledb_query_t* query;
tiledb_query_alloc(ctx, array, TILEDB_WRITE, &query);
tiledb_query_set_layout(ctx, query, TILEDB_UNORDERED);
tiledb_query_set_data_buffer(ctx, query, "d1", d1, &d1_size);
tiledb_query_set_data_buffer(ctx, query, "d2", d2, &d2_size);
tiledb_query_set_data_buffer(ctx, query, "a", a, &a_size);
tiledb_query_set_validity_buffer(
    ctx, query, "a", a_validity, &a_validity_size);

// Submit query
tiledb_query_submit(ctx, query);

// Close array
tiledb_array_close(ctx, array);

// Clean up
tiledb_array_free(&array);
tiledb_query_free(&query);
tiledb_ctx_free(&ctx);

C++

#include <tiledb/tiledb>
using namespace tiledb;

// Create context
Context ctx;

// Create Array
Domain domain(ctx);
auto d1 = Dimension::create<int64_t>(ctx, "d1", {{1, 10}}, 5);
auto d2 = Dimension::create<int64_t>(ctx, "d2", {{1, 10}}, 5);
domain.add_dimension(d1).add_dimension(d2);

auto a = Attribute::create<double>(ctx, "a");
a.set_cell_val_num(2);   // two values per cell
a.set_nullable(true);    // set nullable

ArraySchema schema(ctx, TILEDB_SPARSE);
schema.set_domain(domain);
schema.set_order({{TILEDB_COL_MAJOR, TILEDB_COL_MAJOR}});
schema.add_attributes(a);

Array::create(array_name, schema);

// Prepare some data for the array
std::vector<int64_t> d1 = {1, 2, 3, 4};
std::vector<int64_t> d2 = {2, 1, 3, 4};
std::vector<double> a = {1.1f, 1.2f, 2.1f, 2.2f, 3.1f, 3.2f, 4.1f, 4.2f};
std::vector<uint8_t> a_validity = {1, 1, 0, 1};

// Open array for writing
Array array(ctx, array_name, TILEDB_WRITE);

// Create the query
Query query(ctx, array);
query.set_layout(TILEDB_UNORDERED)
  .set_data_buffer("d1", d1)
  .set_data_buffer("d2", d2)
  .set_data_buffer("a", a)
  .set_validity_buffer("a", a_validity);

// Submit query
query.submit();
  
// Close the array  
array.close();

Python

# TODO

R

array_name <- "nullable"

dom <- tiledb_domain(dims = c(tiledb_dim("d1", c(1L, 10L), 5L, "INT32"),
                              tiledb_dim("d2", c(1L, 10L), 5L, "INT32")))
attr <- tiledb_attr("a", type = "FLOAT64", nullable = TRUE, ncells = 2)
sch <- tiledb_array_schema(dom, attr, sparse = TRUE)

rv <- tiledb_array_create(array_name, sch)

# We can also supply the values as a data.frame, and here we use one
# with a list column containing two elements each with one tuple being NULL
D <- data.frame(d1 = c(1, 2, 3, 4),
                d2 = c(2, 1, 3, 4),
                a = I(list(c(1.1, 1.2), c(2.1, 2.2), 
                           c(NA, NA), c(4.1, 4.2))))
arr <- tiledb_array(array_name)
arr[] <- D

## alternatively, we can use higher-level format: we pass the
## data.frame object, the desired URI and indicate index columns
fromDataFrame(D, array_name, col_index=1:2)

## from either write, we can check the content with a high-level accessor
chk <- tiledb_array(array_name, return_as="data.frame")
chk[]

Java

//Create an array with two dimesnions
Dimension<Integer> rows =
  new Dimension<>(ctx, "rows", Integer.class, new Pair<Integer, Integer>(1, 2), 2);
Dimension<Integer> cols =
  new Dimension<>(ctx, "cols", Integer.class, new Pair<Integer, Integer>(1, 2), 2);

// Create and set getDomain
Domain domain = new Domain(ctx);
domain.addDimension(rows);
domain.addDimension(cols);

// Add two attributes "a1" and "a2", so each (i,j) cell can store
// a character on "a1" and a vector of two floats on "a2".
Attribute a1 = new Attribute(ctx, "a1", Character.class);
Attribute a2 = new Attribute(ctx, "a2", Float.class);

//make fixed-length attributes nullable
a1.setNullable(true);
a2.setNullable(true);

ArraySchema schema = new ArraySchema(ctx, TILEDB_DENSE);
schema.setTileOrder(TILEDB_ROW_MAJOR);
schema.setCellOrder(TILEDB_ROW_MAJOR);
schema.setDomain(domain);
schema.addAttribute(a1);
schema.addAttribute(a2);

Array.create(arrayURI, schema);

// Prepare cell buffers
NativeArray a1 = new NativeArray(ctx, "abcd", String.class);
NativeArray a2 = new NativeArray(ctx, new float[] {0.1f, 0.2f, 1.1f, 1.2f}, Float.class);

// Create query
try (Array array = new Array(ctx, arrayURI, TILEDB_WRITE);
    Query query = new Query(array)) {
  query.setLayout(TILEDB_ROW_MAJOR);
  NativeArray a1Bytemap =
      new NativeArray(ctx, new short[] {1, 0, 0, 1}, Datatype.TILEDB_UINT8);
  NativeArray a2Bytemap =
      new NativeArray(ctx, new short[] {1, 1, 1, 0}, Datatype.TILEDB_UINT8);

  //set the buffers
  query.setBufferNullable("a1", a1, a1Bytemap);
  query.setBufferNullable("a2", a2, a2Bytemap);

  // Submit query
  query.submit();
}

Go

// TODO

C#

using TileDB.CSharp;

// Create TileDB context
using Context ctx = new Context();

// Open array for writing
using Array array = new Array(ctx, array_name);
array.Open(QueryType.Write);

// Prepare some data for the array
long[] d1 = new long[] { 1, 2, 3, 4 };
long[] d2 = new long[] { 2, 1, 3, 4 };
float[] a = new float[] { 1.1f, 1.2f, 2.1f, 2.2f, 3.1f, 3.2f, 4.1f, 4.2f };
byte[] aValidity = new byte[] { 1, 1, 0, 1 };

// Create the query
using Query query = new Query(ctx, array, QueryType.Write);
query.SetLayout(LayoutType.Unordered);
query.SetDataBuffer("d1", d1);
query.SetDataBuffer("d2", d2);
query.SetDataBuffer("a", a);
query.SetValidityBuffer("a", aValidity);

// Submit query
query.Submit();

// Close array
array.Close();

The code snippet above produces the following sparse fragment.

Variable-length, Nullable Attributes

Writing variable-length attribute values to an array involves passing three buffers to TileDB, one for the variable-length cell values, one for the starting offset of each value in the first buffer, and one for the cell validity values. We illustrate this in the code block below with a sparse write, but it is also applicable for dense writes.

C

#include <tiledb/tiledb.h>

// Create TileDB context
tiledb_ctx_t* ctx;
tiledb_ctx_alloc(NULL, &ctx);

// Open array for writing
tiledb_array_t* array;
tiledb_array_alloc(ctx, array_name, &array);
tiledb_array_open(ctx, array, TILEDB_WRITE);

// Prepare some data for the array
long long d1[] = {1, 2, 3, 4};
unsigned long long d1_size = sizeof(d1);
long long d2[] = {2, 1, 3, 4};
unsigned long long d2_size = sizeof(d2);
int data[] = {1, 1, 2, 3, 3, 3, 4};
unsigned long long a_size = sizeof(a);
unsigned long long a_off[] = {0, 8, 12, 24};
unsigned long long a_off_size = sizeof(a_off);
uint8_t a_validity[] = {1, 1, 0, 1};
unsigned long long a_validity_size = sizeof(a_validity);

// Create the query
tiledb_query_t* query;
tiledb_query_alloc(ctx, array, TILEDB_WRITE, &query);
tiledb_query_set_layout(ctx, query, TILEDB_UNORDERED);
tiledb_query_set_data_buffer(ctx, query, "d1", d1, &d1_size);
tiledb_query_set_data_buffer(ctx, query, "d2", d2, &d2_size);
tiledb_query_set_data_buffer(ctx, query, "a", a, &a_size);
tiledb_query_set_validity_buffer(ctx, query, "a", a_validity, &a_validity_size);

// Submit query
tiledb_query_submit(ctx, query);

// Close array
tiledb_array_close(ctx, array);

// Clean up
tiledb_array_free(&array);
tiledb_query_free(&query);
tiledb_ctx_free(&ctx);

C++

#include <tiledb/tiledb>
using namespace tiledb;

// Create context
Context ctx;

// Prepare some data for the array
std::vector<int64_t> d1 = {1, 2, 3, 4};
std::vector<int64_t> d2 = {2, 1, 3, 4};
std::vector<int32_t> a = {1, 1, 2, 3, 3, 3, 4};
std::vector<uint64_t> a_off = {0, 8, 12, 24};
std::vector<uint8_t> a_validity = {1, 1, 0, 1};

// Open array for writing
Array array(ctx, array_name, TILEDB_WRITE);

// Create the query
Query query(ctx, array);
query.set_layout(TILEDB_UNORDERED)
  .set_data_buffer("d1", d1)
  .set_data_buffer("d2", d2)
  .set_data_buffer("a", a)
  .set_offsets_buffer("a", a_off)
  .set_validity_buffer("a", a_validity);

// Submit query
query.submit();

// Close the array
array.close();

Python

# TODO

R

## Variable-length nullable (numerical) attributes are not yet supported

Java

//Create an array with two dimesnions
Dimension<Integer> rows =
  new Dimension<>(ctx, "rows", Integer.class, new Pair<Integer, Integer>(1, 2), 2);
Dimension<Integer> cols =
  new Dimension<>(ctx, "cols", Integer.class, new Pair<Integer, Integer>(1, 2), 2);

// Create and set getDomain
Domain domain = new Domain(ctx);
domain.addDimension(rows);
domain.addDimension(cols);

// Add two attributes "a1" and "a2", so each (i,j) cell can store
// a character on "a1" and a vector of two floats on "a2".
Attribute a1 = new Attribute(ctx, "a1", Float.class);
Attribute a2 = new Attribute(ctx, "a2", Datatype.TILEDB_STRING_ASCII);

//make fixed-length attributes nullable
a1.setNullable(true);
a2.setNullable(true);

//make a2 a variable length attribute
a2.setCellVar();

ArraySchema schema = new ArraySchema(ctx, TILEDB_DENSE);
schema.setTileOrder(TILEDB_ROW_MAJOR);
schema.setCellOrder(TILEDB_ROW_MAJOR);
schema.setDomain(domain);
schema.addAttribute(a1);
schema.addAttribute(a2);

Array.create(arrayURI, schema);

// Prepare cell buffers
NativeArray a1 = new NativeArray(ctx, new float[] {0.1f, 0.2f, 1.1f, 1.2f}, Float.class)
NativeArray a2_data = new NativeArray(ctx, "aabbccdd", Datatype.TILEDB_STRING_ASCII);
NativeArray a2_off = new NativeArray(ctx, new long[] {0, 2, 4, 6}, Datatype.TILEDB_UINT64); Float.class);

// Create query
try (Array array = new Array(ctx, arrayURI, TILEDB_WRITE);
    Query query = new Query(array)) {
  query.setLayout(TILEDB_ROW_MAJOR);
  NativeArray a1Bytemap =
      new NativeArray(ctx, new short[] {1, 0, 0, 1}, Datatype.TILEDB_UINT8);
  NativeArray a2Bytemap =
      new NativeArray(ctx, new short[] {1, 1, 1, 0}, Datatype.TILEDB_UINT8);

  //set the buffers
  query.setBufferNullable("a1", a1, a1Bytemap);
  query.setBufferNullable("a2", a2_off, a2_data, a2Bytemap);

  // Submit query
  query.submit();
}

Go

// TODO

C#

using TileDB.CSharp;

// Create TileDB context
using Context ctx = new Context();

// Open array for writing
using Array array = new Array(ctx, array_name);
array.Open(QueryType.Write);

// Prepare some data for the array
long[] d1 = new long[] { 1, 2, 3, 4 };
long[] d2 = new long[] { 2, 1, 3, 4 };
float[] a = new float[] { 1.1f, 1.2f, 2.1f, 2.2f, 3.1f, 3.2f, 4.1f, 4.2f };
ulong[] aOffsets = new ulong[] { 0, 8, 12, 24 };
byte[] aValidity = new byte[] { 1, 1, 0, 1 };

// Create the query
using Query query = new Query(ctx, array, QueryType.Write);
query.SetLayout(LayoutType.Unordered);
query.SetDataBuffer("d1", d1);
query.SetDataBuffer("d2", d2);
query.SetDataBuffer("a", a);
query.SetOffsetsBuffer("a", aOffsets);
query.SetValidityBuffer("a", aValidity);

// Submit query
query.Submit();

// Close array
array.Close();

The code snippet above produces the following sparse fragment.