1 of 13

Write Arrays

Writing in Dense Subarrays

This is applicable only to dense arrays (see for more details). You can write into a subarray of the array domain as follows:

The code snippets above create the dense fragment shown in the figure below, assuming a 2D array with domain [1, 4] x [1, 4] , 2x2 space tiling, row-major tile and cell order, and a single int32 attribute called a. Observe that you can select the layout in which you can write the cells into. If you switch to column-major or global order, make sure to organize your cells properly before writing (see figure below).

Currently, if your array consists of more than one attributes, TileDB requires you to provide values for all the attributes in each write operation.

Writing Sparse Cells

This is applicable to sparse arrays (see Sparse Writes for more details). You can write sparse cells in an array as follows:

#include <tiledb/tiledb.h>

// Create TileDB context
tiledb_ctx_t* ctx;
tiledb_ctx_alloc(NULL, &ctx);

// Open array for writing
tiledb_array_t* array;
tiledb_array_alloc(ctx, array_name, &array);
tiledb_array_open(ctx, array, TILEDB_WRITE);

// Prepare some data for the array
long long d1[] = {3, 4, 1, 2};
unsigned long long d1_size = sizeof(d1);
long long d2[] = {3, 4, 2, 1};
unsigned long long d2_size = sizeof(d2);
int a[] = {3, 4, 1, 2};
unsigned long long a_size = sizeof(a);

// Create the query
tiledb_query_t* query;
tiledb_query_alloc(ctx, array, TILEDB_WRITE, &query);
tiledb_query_set_layout(ctx, query, TILEDB_UNORDERED);
tiledb_query_set_data_buffer(ctx, query, "d1", d1, &d1_size);
tiledb_query_set_data_buffer(ctx, query, "d2", d2, &d2_size);
tiledb_query_set_data_buffer(ctx, query, "a", a, &a_size);

// Submit query
tiledb_query_submit(ctx, query);

// Close array
tiledb_array_close(ctx, array);

// Clean up
tiledb_array_free(&array);
tiledb_query_free(&query);
tiledb_ctx_free(&ctx);

#include <tiledb/tiledb>
using namespace tiledb;

// Create context
Context ctx;

// Prepare some data for the array
std::vector<int64_t> d1 = {3, 4, 1, 2};
std::vector<int64_t> d2 = {3, 4, 2, 1};
std::vector<int> a = {3, 4, 1, 2};

// Open array for writing
Array array(ctx, array_name, TILEDB_WRITE);

// Create the query
Query query(ctx, array);
query.set_layout(TILEDB_UNORDERED)
     .set_data_buffer("d1", d1)
     .set_data_buffer("d2", d2)
     .set_data_buffer("a", a);

// Submit query
query.submit();
  
// Close the array  
array.close();

import tiledb
import numpy as np

d1 = np.array([3,4,1,2])
d2 = np.array([3,4,2,1])

a = np.array([3,4,1,2])

with tiledb.open(array_name, 'w') as A:
    A[d1, d2] = a

# Prepare some data
a <- c(3L, 4L, 1L, 2L)

d1 <- c(3, 4, 1, 2)
d2 <- c(3, 4, 2, 1)

# Open the array and write the data to it
A <- tiledb_array(uri = array_name, is.sparse = TRUE)
A[d1, d2] <- a

// Create context, array and query
try(Context ctx = new Context(),
     Array array = new Array(ctx, array_name, TILEDB_WRITE),
     Query query = new Query(ctx, array);) {

     // Prepare some data for the array
     NativeArray d1 = new NativeArray(ctx, new long[] {3l, 4l, 1l, 2l}, Long.class);
     NativeArray d2 = new NativeArray(ctx, new long[] {3l, 4l, 2l, 1l}, Long.class);
     NativeArray a = new NativeArray(ctx, new int[] {3, 4, 1, 2}, Integer.class);
     
     // Create the query
     query.setLayout(TILEDB_ROW_MAJOR)
          .setBuffer("d1", d1)
          .setBuffer("d2", d2)
          .setBuffer("a", a);
     
     // Submit query
     query.submit();
}

import tiledb "github.com/TileDB-Inc/TileDB-Go"

// Create context
Context ctx;

// Prepare some data for the array
d1 := []int64{3, 4, 1, 2}
d2 := []int64{3, 4, 2, 1}
a := []int32{3, 4, 1, 2}

// Open array for writing
array, _ := tiledb.NewArray(ctx, array_name)
array.Open(tiledb.TILEDB_WRITE)

// Create the query
query, _ := tiledb.NewQuery(ctx, array)
query.SetLayout(tiledb.TILEDB_UNORDERED)
_, err = query.SetBuffer("d1", d1)
_, err = query.SetBuffer("d2", d2)
_, err = query.SetBuffer("a", a)

// Submit query
query.Submit()
 
// Close the array  
array.Close();

using TileDB.CSharp;

// Create TileDB context
using Context ctx = new Context();

// Open array for writing
using Array array = new Array(ctx, array_name);
array.Open(QueryType.Write);

// Prepare some data for the array
long[] d1 = new long[] { 3, 1, 4, 2 };
long[] d2 = new long[] { 3, 4, 2, 1 };
int[] a = new int[] { 3, 4, 1, 2 };

// Create the query
using Query query = new Query(ctx, array, QueryType.Write);
query.SetLayout(LayoutType.Unordered);
query.SetDataBuffer("d1", d1);
query.SetDataBuffer("d2", d2);
query.SetDataBuffer("a", a);

// Submit query
query.Submit();

// Close array
array.Close();

The code snippets above write the sparse cells shown in the figure below, assuming a 2D array with a int64 domain [1, 4] x [1, 4] , 2x2 space tiling, row-major tile and cell order, and a single int32 attribute called a. The unordered layout means that you do no have to worry about the order of the values when writing them. Since you must explicitly set the coordinates, TileDB knows how to properly sort on the global order before writing them cells to persistent storage. If you switch to global order, make sure to organize your cells properly before writing (see figure below).

Currently, if your array consists of more than one attributes, TileDB requires you to provide values for all the attributes in each write operation.

Writing Encrypted Arrays

To write to an encrypted array, you simply need to open it for writing using the encryption key you used to create it.

// ... create context ctx

// Open encrypted array for writing
const char key[] = "0123456789abcdeF0123456789abcdeF";
tiledb_config_t* config;
tiledb_config_alloc(&config, &error);
tiledb_config_set(config, "sm.encryption_type", "AES_256_GCM", &error);
tiledb_config_set(config, "sm.encryption_key", key, &error);

tiledb_array_t* array;
tiledb_array_alloc(ctx, "<array-uri>", &array);
tiledb_array_set_config(ctx, array, config);
tiledb_array_open(ctx, array, TILEDB_WRITE);

// ... create context ctx

// Open encrypted array for writing
const char key[] = "0123456789abcdeF0123456789abcdeF";
Array array(ctx,
    "<array-uri>",
    TILEDB_WRITE,
    // Load all fragments available
    TemporalPolicy(0, std::numeric_limits<uint64_t>::max()),
    EncryptionAlgorithm(AESGCM, key));

key = "0123456789abcdeF0123456789abcdeF"
tiledb.SparseArray.create(array_name, schema, key=key)
with tiledb.SparseArray(array_name, 'w', key=key) as A:
    # A[] = ...

with tiledb.open(array_name, key=key) as A:
    # read A[...]

ctx <- tiledb_ctx()
arrptr <- tiledb:::libtiledb_array_open_with_key(ctx@ptr, uridensewkey, 
                                                 "WRITE", encryption_key)

// ... create context ctx

// Open encrypted array for writing
String key = "0123456789abcdeF0123456789abcdeF";
Array array(ctx, "<array-uri>", TILEDB_WRITE, TILEDB_AES_256_GCM, key..getBytes(StandardCharsets.UTF_8));

// ... create context ctx

// The 256-bit encryption key, stored as a string for convenience.
var encryption_key = "0123456789abcdeF0123456789abcdeF"

// Open encrypted array for writing
array, _ := tiledb.NewArray(ctx, "<array-uri>")
array.CreateWithKey(schema, tiledb.TILEDB_AES_256_GCM, encryption_key)

Fixed-length Attributes

Writing fixed-length attribute cells to an array involves populating a single buffer and serializing all values in that buffer according to the layout you are writing into (see Writing Dense Subarrays and Writing Sparse Cells). The same holds even if you defined the attribute to store any fixed number of values (>1) per cell. This is illustrated in the code block below with a sparse write, but it is applicable to dense writes as well.

#include <tiledb/tiledb.h>

// Create TileDB context
tiledb_ctx_t* ctx;
tiledb_ctx_alloc(NULL, &ctx);

// Open array for writing
tiledb_array_t* array;
tiledb_array_alloc(ctx, array_name, &array);
tiledb_array_open(ctx, array, TILEDB_WRITE);

// Prepare some data for the array
long long d1[] = {1, 2, 3, 4};
unsigned long long d1_size = sizeof(d1);
long long d2[] = {2, 1, 3, 4};
unsigned long long d2_size = sizeof(d2);
float a[] = {1.1f, 1.2f, 2.1f, 2.2f, 3.1f, 3.2f, 4.1f, 4.2f};
unsigned long long a_size = sizeof(a);

// Create the query
tiledb_query_t* query;
tiledb_query_alloc(ctx, array, TILEDB_WRITE, &query);
tiledb_query_set_layout(ctx, query, TILEDB_UNORDERED);
tiledb_query_set_data_buffer(ctx, query, "d1", d1, &d1_size);
tiledb_query_set_data_buffer(ctx, query, "d2", d2, &d2_size);
tiledb_query_set_data_buffer(ctx, query, "a", a, &a_size);

// Submit query
tiledb_query_submit(ctx, query);

// Close array
tiledb_array_close(ctx, array);

// Clean up
tiledb_array_free(&array);
tiledb_query_free(&query);
tiledb_ctx_free(&ctx);

#include <tiledb/tiledb>
using namespace tiledb;

// Create context
Context ctx;

// Create Array
Domain domain(ctx);
auto d1 = Dimension::create<int64_t>(ctx, "d1", {{1, 10}}, 5);
auto d2 = Dimension::create<int64_t>(ctx, "d2", {{1, 10}}, 5);
domain.add_dimension(d1).add_dimension(d2);

auto a = Attribute::create<double>(ctx, "a");
a.set_cell_val_num(2);   // two values per cell

ArraySchema schema(ctx, TILEDB_SPARSE);
schema.set_domain(domain);
schema.set_order({{TILEDB_COL_MAJOR, TILEDB_COL_MAJOR}});
schema.add_attributes(a);

Array::create(array_name, schema);

// Prepare some data for the array
std::vector<int64_t> d1 = {1, 2, 3, 4};
std::vector<int64_t> d2 = {2, 1, 3, 4};
std::vector<double> a = {1.1f, 1.2f, 2.1f, 2.2f, 3.1f, 3.2f, 4.1f, 4.2f};

// Open array for writing
Array array(ctx, array_name, TILEDB_WRITE);

// Create the query
Query query(ctx, array);
query.set_layout(TILEDB_UNORDERED)
  .set_data_buffer("d1", d1)
  .set_data_buffer("d2", d2)
  .set_data_buffer("a", a);

// Submit query
query.submit();
  
// Close the array  
array.close();

# The Python API treats homogenous NumPy record arrays
# as fixed-length cells.

d1 = np.array([1,2,3,4])
d2 = np.array([2,1,3,2])

a = np.array([(1.1,1.2),(2.1,2.2),(3.1,3.2),(4.1,4.2)], dtype='f4,f4')

with tiledb.open(array_name, 'w') as A:
    A[d1, d2] = a

# Prepare and write the array schema
d1  <- tiledb_dim("d1", domain = c(1L, 10L))
d2  <- tiledb_dim("d2", domain = c(1L, 10L))
dom <- tiledb_domain(c(d1, d2))
attr <- tiledb_attr("a", type = "FLOAT64")

## set to two values per cell
tiledb_attribute_set_cell_val_num(attr, 2)

sch <- tiledb_array_schema(dom, attr, sparse=TRUE)
res <- tiledb_array_create(tmp, sch)

# Prepare some data for the array
a <- c(1.1, 1.2, 2.1, 2.2, 3.1, 3.2, 4.1, 4.2)
d1 <- c(1L, 2L, 3L, 4L)
d2 <- c(2L, 1L, 3L, 4L)

arr <- tiledb_array(tmp, "WRITE")
qry <- tiledb_query(arr, "WRITE")
qry <- tiledb_query_set_layout(qry, "UNORDERED")
qry <- tiledb_query_set_buffer(qry, "a", a)
qry <- tiledb_query_set_buffer(qry, "d1", d1)
qry <- tiledb_query_set_buffer(qry, "d2", d2)
qry <- tiledb_query_submit(qry)
res <- tiledb_array_close(arr)

// Create context, array and query
try(Context ctx = new Context(),
     Array array = new Array(ctx, array_name, TILEDB_WRITE),
     Query query = new Query(ctx, array);) {

     // Prepare some data for the array
     NativeArray d1 = new NativeArray(ctx, new long[] {1l, 2l, 3l, 4l}, Long.class);     
     NativeArray d2 = new NativeArray(ctx, new long[] {2l, 1l, 3l, 4l}, Long.class);
     NativeArray a = new NativeArray(ctx, new float[] {1.1f, 1.2f, 2.1f, 2.2f, 3.1f, 3.2f, 4.1f, 4.2f}, Float.class);
     
     // Create the query
     query.setLayout(TILEDB_ROW_MAJOR)
          .setBuffer("d1", d1)
          .setBuffer("d2", d2)
          .setBuffer("a", a);
     
     // Submit query
     query.submit();
}

import "github.com/TileDB-Inc/TileDB-Go"

// Create context
ctx, _ := tiledb.NewContext(nil)

// Prepare some data for the array
d1 := []int32{1, 2, 3, 4}
d2 := []int32{2, 1, 3, 4}
a := []float32{1.1, 1.2, 2.1, 2.2, 3.1, 3.2, 4.1, 4.2}

// Open array for writing
array, _ := tiledb.NewArray(ctx, array_name)
array.Open(tiledb.TILEDB_WRITE)

// Create the query
query, _ := tiledb.NewQuery(ctx, array)
query.SetLayout(tiledb.TILEDB_UNORDERED)
_, err = query.SetBuffer("d1", d1)
_, err = query.SetBuffer("d2", d2)
_, err = query.SetBuffer("a", a)

// Submit query
query.Submit();
 
// Close the array  
array.Close();

using TileDB.CSharp;

// Create TileDB context
using Context ctx = new Context();

// Open array for writing
using Array array = new Array(ctx, array_name);
array.Open(QueryType.Write);

// Prepare some data for the array
long[] d1 = new long[] { 1, 2, 3, 4 };
long[] d2 = new long[] { 2, 1, 3, 4 };
float[] a = new float[] { 1.1f, 1.2f, 2.1f, 2.2f, 3.1f, 3.2f, 4.1f, 4.2f };

// Create the query
using Query query = new Query(ctx, array, QueryType.Write);
query.SetLayout(LayoutType.Unordered);
query.SetDataBuffer("d1", d1);
query.SetDataBuffer("d2", d2);
query.SetDataBuffer("a", a);

// Submit query
query.Submit();

// Close array
array.Close();

The code snippet above produces the following sparse fragment.

Var-length Attributes

Writing variable-length attribute values to an array involves passing two buffers to TileDB, one for the variable-length cell values and one for the starting offset of each value in the first buffer. We illustrate this in the code block below with a sparse write, but it is also applicable to dense writes.

The code snippet above produces the following sparse fragment.

Nullable Attributes

Fixed-length, nullable attributes

Writing nullable, fixed-length attribute cells to an array involves populating a data buffer and a validity buffer. This is illustrated in the code block below with a sparse write, but it is applicable to dense writes as well.

The code snippet above produces the following sparse fragment.

Variable-length, Nullable Attributes

Writing variable-length attribute values to an array involves passing three buffers to TileDB, one for the variable-length cell values, one for the starting offset of each value in the first buffer, and one for the cell validity values. We illustrate this in the code block below with a sparse write, but it is also applicable for dense writes.

The code snippet above produces the following sparse fragment.

Writing at a Timestamp

There are cases where it is very useful to write at a specific timestamp (e.g., in the past or the future). TileDB allows you to do that by simply opening the array for writing at the desired timestamp, similar to reads:

#include <tiledb/tiledb.h>

// ... create TileDB context

// Open array for writing at a timestamp
tiledb_array_t* array;
tiledb_array_alloc(ctx, array_name, &array);
tiledb_array_open_at(ctx, array, TILEDB_WRITE, timestamp);
// or use tiledb_array_open_at_with_key for encrypted arrays

// Create the query
tiledb_query_t* query;
tiledb_query_alloc(ctx, array, TILEDB_WRITE, &query);

// ... perform the write as usual

// Close array
tiledb_array_close(ctx, array);

// Clean up
tiledb_array_free(&array);
tiledb_query_free(&query);

#include <tiledb/tiledb>
using namespace tiledb;

// Create context
Context ctx;

// Open array for writing at a timestamp
Array array(ctx, array_name, TILEDB_WRITE, timestamp);
// Or array(ctx, array_name, TILEDB_WRITE, TILEDB_AES_256_GCM, key, sizeof(key), timestamp)
// for encrypted arrays

// Create the query
Query query(ctx, array);

// ... perform the query

// Close the array
array.close();

import tiledb

# ... create array with required schema
# ... decide the timestamp you would like to write to

with tiledb.open(path, mode='w', ctx=ctx, timestamp=timestamp) as T:
    # the data written will be written at `timestamp`
    T[:] = A

# 'at' uses Sys.time() from R in seconds, and shifts back 10 minutes
at <- Sys.time() - 10*60

# 'arr' is an already created array, could also be encrypted and carry key
arr <- tiledb_array_open_at(arr, "WRITE", Sys.time() - 600)

# arr is now open for writing, any suitable content can be written the usual way

// Create context
Context ctx = new Context();

// Open array for writing at a timestamp
Array array = new Array(ctx, "<array-uri>", TILEDB_WRITE, timestamp);
// Or new Array(ctx, "<array-uri>", TILEDB_WRITE, TILEDB_AES_256_GCM, key, timestamp)
// for encrypted arrays

// Create the query
Query query(ctx, array);

// ... perform the query

// Close the array
array.close();

// Create context
ctx, _ := tiledb.NewContext(nil)

// Open array for writing
array, _ := tiledb.NewArray(ctx, "array_name")

// Open array for writing at a timestamp
array.OpenAt(TILEDB_WRITE, uint64(time.Now().UnixNano()/1000000))

Getting the Written Fragment Info

A write operation can create one or more fragments. You can get information about the written fragments as follows:

Array Upgrade

TileDB arrays have a format version that is set by the version of TileDB used in creating the array. The format version controls items such as the directory layout and the binary format. Writes to TileDB arrays are done in the same format version as the dataset in order to maintain client compatibility. When a new format version is available, you can upgrade your arrays to take advantage of the newer format's improvements.

Updating the array version is a logical change. This allows for future writes or consolidation to be written in the newer version. Calling the upgrade API will not rewrite the dataset. After calling the upgrade API, you can use consolidation to perform a rewrite of existing fragments.

// ... create context ctx
// ... create config config

int rc = tiledb_array_upgrade_version(ctx, array_name, config);

// ... create context ctx
// ... create config config

tiledb::Array::upgrade_version(ctx, "<array_uri>", config);

# ... create context ctx
# ... create config config

Array.upgrade_version()

# optionally pass a config
Array.upgrade_version(config=config)

// Create array and context

array.upgradeVersion(context, array.getConfig());

Consolidation and Vacuuming

Consolidation

Fragments

You can consolidate an array as follows. See for how to fine-tune the consolidation algorithm with configuration parameters, and for more details on creating configuration objects.

If you do not set a configuration object to the consolidation function, the consolidation algorithm will inherit the (default or set) configuration of the context. Otherwise, the set options in the passed configuration object will override those of the context's, but the rest of the options will still be inherited from the context's configuration.

Fragment Metadata

Array Metadata

Commits

The same consolidation API is used for consolidating commits. The only change is configuration parameter "sm.consolidation.mode" which should now be set to "commits".

Vacuuming

Fragments

After consolidation, you may optionally remove the consolidated fragments with the "vacuum" step. Note that vacuuming must not be run if you intend to use the time-traveling feature (opening array at a specific prior timestamp).

As with consolidation above, vacuuming inherits the context configuration if no configuration override is specified.

Fragment Metadata

Array Metadata

Commits

The same vacuuming API is used for vacumming commits. The only change is configuration parameter "sm.vacuum.mode" which should now be set to "commits".

Deleting

TileDB supports deletion using to specify which cells to remove. When a delete query is committed, any cells matching the given condition are ignored on subsequent reads.

Consolidation Plan

This will create a plan for an array that can be later used during consolidation. The desired fragment size gives the maximum size for a fragment. The maximum fragment size will be used to find fragments that are already too large and should be split as well as fragments that are too small and should be combined. The value should be used as is in the consolidation parameters ("sm.consolidation.max_fragment_size") when running the different nodes. The plan divides the fragments of the array into different nodes that need to be consolidated together. The plan is created as follows:

The code snippets above also shows how to print the different nodes with the fragment URIs they contain. The list of fragments can be passed into the tiledb_array_consolidate_fragments API. Note that at the moment there is no CPP API that allows to consolidate a list of fragments.

Fixed-length Attributes

#include <tiledb/tiledb.h>

// Create TileDB context
tiledb_ctx_t* ctx;
tiledb_ctx_alloc(NULL, &ctx);

// Open array for writing
tiledb_array_t* array;
tiledb_array_alloc(ctx, array_name, &array);
tiledb_array_open(ctx, array, TILEDB_WRITE);

// Prepare some data for the array
long long d1[] = {1, 2, 3, 4};
unsigned long long d1_size = sizeof(d1);
long long d2[] = {2, 1, 3, 4};
unsigned long long d2_size = sizeof(d2);
float a[] = {1.1f, 1.2f, 2.1f, 2.2f, 3.1f, 3.2f, 4.1f, 4.2f};
unsigned long long a_size = sizeof(a);

// Create the query
tiledb_query_t* query;
tiledb_query_alloc(ctx, array, TILEDB_WRITE, &query);
tiledb_query_set_layout(ctx, query, TILEDB_UNORDERED);
tiledb_query_set_data_buffer(ctx, query, "d1", d1, &d1_size);
tiledb_query_set_data_buffer(ctx, query, "d2", d2, &d2_size);
tiledb_query_set_data_buffer(ctx, query, "a", a, &a_size);

// Submit query
tiledb_query_submit(ctx, query);

// Close array
tiledb_array_close(ctx, array);

// Clean up
tiledb_array_free(&array);
tiledb_query_free(&query);
tiledb_ctx_free(&ctx);

#include <tiledb/tiledb>
using namespace tiledb;

// Create context
Context ctx;

// Create Array
Domain domain(ctx);
auto d1 = Dimension::create<int64_t>(ctx, "d1", {{1, 10}}, 5);
auto d2 = Dimension::create<int64_t>(ctx, "d2", {{1, 10}}, 5);
domain.add_dimension(d1).add_dimension(d2);

auto a = Attribute::create<double>(ctx, "a");
a.set_cell_val_num(2);   // two values per cell

ArraySchema schema(ctx, TILEDB_SPARSE);
schema.set_domain(domain);
schema.set_order({{TILEDB_COL_MAJOR, TILEDB_COL_MAJOR}});
schema.add_attributes(a);

Array::create(array_name, schema);

// Prepare some data for the array
std::vector<int64_t> d1 = {1, 2, 3, 4};
std::vector<int64_t> d2 = {2, 1, 3, 4};
std::vector<double> a = {1.1f, 1.2f, 2.1f, 2.2f, 3.1f, 3.2f, 4.1f, 4.2f};

// Open array for writing
Array array(ctx, array_name, TILEDB_WRITE);

// Create the query
Query query(ctx, array);
query.set_layout(TILEDB_UNORDERED)
  .set_data_buffer("d1", d1)
  .set_data_buffer("d2", d2)
  .set_data_buffer("a", a);

// Submit query
query.submit();
  
// Close the array  
array.close();

# The Python API treats homogenous NumPy record arrays
# as fixed-length cells.

d1 = np.array([1,2,3,4])
d2 = np.array([2,1,3,2])

a = np.array([(1.1,1.2),(2.1,2.2),(3.1,3.2),(4.1,4.2)], dtype='f4,f4')

with tiledb.open(array_name, 'w') as A:
    A[d1, d2] = a

# Prepare and write the array schema
d1  <- tiledb_dim("d1", domain = c(1L, 10L))
d2  <- tiledb_dim("d2", domain = c(1L, 10L))
dom <- tiledb_domain(c(d1, d2))
attr <- tiledb_attr("a", type = "FLOAT64")

## set to two values per cell
tiledb_attribute_set_cell_val_num(attr, 2)

sch <- tiledb_array_schema(dom, attr, sparse=TRUE)
res <- tiledb_array_create(tmp, sch)

# Prepare some data for the array
a <- c(1.1, 1.2, 2.1, 2.2, 3.1, 3.2, 4.1, 4.2)
d1 <- c(1L, 2L, 3L, 4L)
d2 <- c(2L, 1L, 3L, 4L)

arr <- tiledb_array(tmp, "WRITE")
qry <- tiledb_query(arr, "WRITE")
qry <- tiledb_query_set_layout(qry, "UNORDERED")
qry <- tiledb_query_set_buffer(qry, "a", a)
qry <- tiledb_query_set_buffer(qry, "d1", d1)
qry <- tiledb_query_set_buffer(qry, "d2", d2)
qry <- tiledb_query_submit(qry)
res <- tiledb_array_close(arr)

// Create context, array and query
try(Context ctx = new Context(),
     Array array = new Array(ctx, array_name, TILEDB_WRITE),
     Query query = new Query(ctx, array);) {

     // Prepare some data for the array
     NativeArray d1 = new NativeArray(ctx, new long[] {1l, 2l, 3l, 4l}, Long.class);     
     NativeArray d2 = new NativeArray(ctx, new long[] {2l, 1l, 3l, 4l}, Long.class);
     NativeArray a = new NativeArray(ctx, new float[] {1.1f, 1.2f, 2.1f, 2.2f, 3.1f, 3.2f, 4.1f, 4.2f}, Float.class);
     
     // Create the query
     query.setLayout(TILEDB_ROW_MAJOR)
          .setBuffer("d1", d1)
          .setBuffer("d2", d2)
          .setBuffer("a", a);
     
     // Submit query
     query.submit();
}

import "github.com/TileDB-Inc/TileDB-Go"

// Create context
ctx, _ := tiledb.NewContext(nil)

// Prepare some data for the array
d1 := []int32{1, 2, 3, 4}
d2 := []int32{2, 1, 3, 4}
a := []float32{1.1, 1.2, 2.1, 2.2, 3.1, 3.2, 4.1, 4.2}

// Open array for writing
array, _ := tiledb.NewArray(ctx, array_name)
array.Open(tiledb.TILEDB_WRITE)

// Create the query
query, _ := tiledb.NewQuery(ctx, array)
query.SetLayout(tiledb.TILEDB_UNORDERED)
_, err = query.SetBuffer("d1", d1)
_, err = query.SetBuffer("d2", d2)
_, err = query.SetBuffer("a", a)

// Submit query
query.Submit();
 
// Close the array  
array.Close();

using TileDB.CSharp;

// Create TileDB context
using Context ctx = new Context();

// Open array for writing
using Array array = new Array(ctx, array_name);
array.Open(QueryType.Write);

// Prepare some data for the array
long[] d1 = new long[] { 1, 2, 3, 4 };
long[] d2 = new long[] { 2, 1, 3, 4 };
float[] a = new float[] { 1.1f, 1.2f, 2.1f, 2.2f, 3.1f, 3.2f, 4.1f, 4.2f };

// Create the query
using Query query = new Query(ctx, array, QueryType.Write);
query.SetLayout(LayoutType.Unordered);
query.SetDataBuffer("d1", d1);
query.SetDataBuffer("d2", d2);
query.SetDataBuffer("a", a);

// Submit query
query.Submit();

// Close array
array.Close();

The code snippet above produces the following sparse fragment.

Writing Sparse Cells

This is applicable to sparse arrays (see Sparse Writes for more details). You can write sparse cells in an array as follows:

#include <tiledb/tiledb.h>

// Create TileDB context
tiledb_ctx_t* ctx;
tiledb_ctx_alloc(NULL, &ctx);

// Open array for writing
tiledb_array_t* array;
tiledb_array_alloc(ctx, array_name, &array);
tiledb_array_open(ctx, array, TILEDB_WRITE);

// Prepare some data for the array
long long d1[] = {3, 4, 1, 2};
unsigned long long d1_size = sizeof(d1);
long long d2[] = {3, 4, 2, 1};
unsigned long long d2_size = sizeof(d2);
int a[] = {3, 4, 1, 2};
unsigned long long a_size = sizeof(a);

// Create the query
tiledb_query_t* query;
tiledb_query_alloc(ctx, array, TILEDB_WRITE, &query);
tiledb_query_set_layout(ctx, query, TILEDB_UNORDERED);
tiledb_query_set_data_buffer(ctx, query, "d1", d1, &d1_size);
tiledb_query_set_data_buffer(ctx, query, "d2", d2, &d2_size);
tiledb_query_set_data_buffer(ctx, query, "a", a, &a_size);

// Submit query
tiledb_query_submit(ctx, query);

// Close array
tiledb_array_close(ctx, array);

// Clean up
tiledb_array_free(&array);
tiledb_query_free(&query);
tiledb_ctx_free(&ctx);

#include <tiledb/tiledb>
using namespace tiledb;

// Create context
Context ctx;

// Prepare some data for the array
std::vector<int64_t> d1 = {3, 4, 1, 2};
std::vector<int64_t> d2 = {3, 4, 2, 1};
std::vector<int> a = {3, 4, 1, 2};

// Open array for writing
Array array(ctx, array_name, TILEDB_WRITE);

// Create the query
Query query(ctx, array);
query.set_layout(TILEDB_UNORDERED)
     .set_data_buffer("d1", d1)
     .set_data_buffer("d2", d2)
     .set_data_buffer("a", a);

// Submit query
query.submit();
  
// Close the array  
array.close();

import tiledb
import numpy as np

d1 = np.array([3,4,1,2])
d2 = np.array([3,4,2,1])

a = np.array([3,4,1,2])

with tiledb.open(array_name, 'w') as A:
    A[d1, d2] = a

# Prepare some data
a <- c(3L, 4L, 1L, 2L)

d1 <- c(3, 4, 1, 2)
d2 <- c(3, 4, 2, 1)

# Open the array and write the data to it
A <- tiledb_array(uri = array_name, is.sparse = TRUE)
A[d1, d2] <- a

// Create context, array and query
try(Context ctx = new Context(),
     Array array = new Array(ctx, array_name, TILEDB_WRITE),
     Query query = new Query(ctx, array);) {

     // Prepare some data for the array
     NativeArray d1 = new NativeArray(ctx, new long[] {3l, 4l, 1l, 2l}, Long.class);
     NativeArray d2 = new NativeArray(ctx, new long[] {3l, 4l, 2l, 1l}, Long.class);
     NativeArray a = new NativeArray(ctx, new int[] {3, 4, 1, 2}, Integer.class);
     
     // Create the query
     query.setLayout(TILEDB_ROW_MAJOR)
          .setBuffer("d1", d1)
          .setBuffer("d2", d2)
          .setBuffer("a", a);
     
     // Submit query
     query.submit();
}

import tiledb "github.com/TileDB-Inc/TileDB-Go"

// Create context
Context ctx;

// Prepare some data for the array
d1 := []int64{3, 4, 1, 2}
d2 := []int64{3, 4, 2, 1}
a := []int32{3, 4, 1, 2}

// Open array for writing
array, _ := tiledb.NewArray(ctx, array_name)
array.Open(tiledb.TILEDB_WRITE)

// Create the query
query, _ := tiledb.NewQuery(ctx, array)
query.SetLayout(tiledb.TILEDB_UNORDERED)
_, err = query.SetBuffer("d1", d1)
_, err = query.SetBuffer("d2", d2)
_, err = query.SetBuffer("a", a)

// Submit query
query.Submit()
 
// Close the array  
array.Close();

using TileDB.CSharp;

// Create TileDB context
using Context ctx = new Context();

// Open array for writing
using Array array = new Array(ctx, array_name);
array.Open(QueryType.Write);

// Prepare some data for the array
long[] d1 = new long[] { 3, 1, 4, 2 };
long[] d2 = new long[] { 3, 4, 2, 1 };
int[] a = new int[] { 3, 4, 1, 2 };

// Create the query
using Query query = new Query(ctx, array, QueryType.Write);
query.SetLayout(LayoutType.Unordered);
query.SetDataBuffer("d1", d1);
query.SetDataBuffer("d2", d2);
query.SetDataBuffer("a", a);

// Submit query
query.Submit();

// Close array
array.Close();

Currently, if your array consists of more than one attributes, TileDB requires you to provide values for all the attributes in each write operation.

Deleting

TileDB supports deletion using to specify which cells to remove. When a delete query is committed, any cells matching the given condition are ignored on subsequent reads.

#include <iostream>
#include <tiledb/tiledb>

using namespace tiledb;

std::string array_uri = "deletes_array";

// Create TileDB context.
Context ctx;

// Create the array.
Domain domain(ctx);
ArraySchema schema(ctx, TILEDB_SPARSE);
domain.add_dimension(Dimension::create<int32_t>(ctx, "cols", {{1, 6}}, 2));
schema.set_domain(domain);
schema.add_attribute(Attribute::create<int32_t>(ctx, "a1"));
Array::create(array_uri, schema);

// Write initial data to the array.
Array array(ctx, array_uri, TILEDB_WRITE);
Query write_query(ctx, array, TILEDB_WRITE);
std::vector<int32_t> cols_data = {1, 2, 3, 4, 5, 6};
std::vector<int32_t> a1_data = {1, 2, 3, 4, 5, 6};
write_query.set_layout(TILEDB_UNORDERED)
    .set_data_buffer("cols", cols_data)
    .set_data_buffer("a1", a1_data);
write_query.submit();
array.close();

// Delete cells where a1 attribute is >= 4
array.open(TILEDB_DELETE);
Query delete_query(ctx, array, TILEDB_DELETE);
QueryCondition qc(ctx);
int32_t val = 4;
qc.init("a1", &val, sizeof(val), TILEDB_GE);
delete_query.set_condition(qc);
delete_query.submit();
array.close();

// Read data from the array.
array.open(TILEDB_READ);
Query read_query(ctx, array, TILEDB_READ);
std::vector<int32_t> cols_data_read(6);
std::vector<int32_t> a1_data_read(6);
Subarray subarray(ctx, array);
subarray.add_range(0, 1, 6);
read_query.set_layout(TILEDB_UNORDERED)
    .set_subarray(subarray)
    .set_data_buffer("cols", cols_data_read)
    .set_data_buffer("a1", a1_data_read);
read_query.submit();
array.close();

// Output: 1, 2, 3, 0, 0, 0
for (size_t i = 0; i < 5; i++) {
  std::cout << a1_data_read[i] << ", ";
}
std::cout << a1_data_read[5] << std::endl;

import tiledb, numpy as np
import tempfile

path = tempfile.mkdtemp("example_sparse_coord_delete")
print(path)

# Create array with one int dimension and one uint attribute
dom = tiledb.Domain(tiledb.Dim(name="x", domain=(1, 10), tile=1, dtype=np.uint32))
attrs = [tiledb.Attr("ints", dtype=np.uint32)]
schema = tiledb.ArraySchema(domain=dom, attrs=attrs, sparse=True)
tiledb.Array.create(path, schema)

# Generate 10 random integers
data = np.random.randint(1, 10, 10)

# Write data
with tiledb.open(path, "w") as A:
    A[np.arange(1, 11)] = data

# Delete all cells with x value less than 5
qc = "x < 5"

# Issue delete query
with tiledb.open(path, "d") as A:
    A.query(cond=qc).submit()

# Read data back and print: cells matching condition should be removed
with tiledb.open(path, "r") as A:
    print(A[:])

uri <- "deletes_array"

## Create a (sparse) array from a data frame, indexed on 'cols'
DF <- data.frame(cols = 1:6, a1 = 1:6)
fromDataFrame(DF, uri, col_index="cols")

## Create a query condition object corresponding to 'a1 >= 4'
qc <- parse_query_condition(a1 >= 4)
## and apply it as a DELETE query
arr <- tiledb_array(uri)
qry <- tiledb_query(arr, "DELETE")
qry <- tiledb_query_set_condition(qry, qc)
tiledb_query_submit(qry)
tiledb_query_finalize(qry)

## Read back: only first see three rows where 'a1 < 4'
D2 <- tiledb_array(uri, return_as="data.frame")[]
print(D2)

using System.Linq;
using TileDB.CSharp;

string arrayUri = "deletes_array";

// Create TileDB context.
using Context ctx = new Context();

// Create the array.
using (ArraySchema schema = new ArraySchema(ctx, ArrayType.Sparse))
{
    using Domain domain = new Domain(ctx);
    using Dimension cols = Dimension.Create(ctx, "cols", 1, 6, 2);
    domain.AddDimension(cols);
    schema.SetDomain(domain);
    using Attribute a1 = Attribute.Create<int>(ctx, "a1");
    schema.AddAttribute(a1);
    Array.Create(ctx, arrayUri, schema);
}

using Array array = new Array(ctx, arrayUri);

// Write initial data to the array.
array.Open(QueryType.Write);
using (Query writeQuery = new Query(ctx, array, QueryType.Write))
{
    int[] colsData = { 1, 2, 3, 4, 5, 6 };
    int[] a1Data = { 1, 2, 3, 4, 5, 6 };

    writeQuery.SetLayout(LayoutType.Unordered);
    writeQuery.SetDataBuffer("cols", colsData);
    writeQuery.SetDataBuffer("a1", a1Data);
    writeQuery.Submit();
}
array.Close();

// Delete cells where a1 attribute is >= 4
array.Open(QueryType.Delete);
using (Query deleteQuery = new Query(ctx, array, QueryType.Delete))
{
    using QueryCondition qc = QueryCondition.Create(ctx, "a1", 4,
        QueryConditionOperatorType.GreaterThanOrEqual);
    deleteQuery.SetCondition(qc);
    deleteQuery.Submit();
}
array.Close();

// Read data from the array.
array.Open(QueryType.Read);
using (Query readQuery = new Query(ctx, array, QueryType.Read))
{
    int[] colsDataRead = new int[6];
    int[] a1DataRead = new int[6];
    using Subarray subarray = new Subarray(array);
    subarray.AddRange(0, 1, 6);
    readQuery.SetLayout(LayoutType.Unordered);
    readQuery.SetSubarray(subarray);
    readQuery.SetDataBuffer("cols", colsDataRead);
    readQuery.SetDataBuffer("a1", a1DataRead);
    readQuery.Submit();

    int a1Count = (int)readQuery.GetResultDataElements("a1");
    // Output: 1, 2, 3
    System.Console.WriteLine(string.Join(", ", a1DataRead.Take(a1Count)));
}
array.Close();

// create array
Dimension<Integer> d1 =
    new Dimension<Integer>(ctx, "d1", Datatype.TILEDB_STRING_ASCII, null, null);

// Create and set getDomain
Domain domain = new Domain(ctx);
domain.addDimension(d1);

Attribute a1 = new Attribute(ctx, "a1", Integer.class);

ArraySchema schema = new ArraySchema(ctx, TILEDB_SPARSE);
schema.setTileOrder(TILEDB_ROW_MAJOR);
schema.setCellOrder(TILEDB_ROW_MAJOR);
schema.setDomain(domain);
schema.addAttribute(a1);

Array.create(arrayURISparse, schema);

// write array
NativeArray d_data = new NativeArray(ctx, "aabbccddee", Datatype.TILEDB_STRING_ASCII);
NativeArray d_off = new NativeArray(ctx, new long[] {0, 2, 4, 6, 8}, Datatype.TILEDB_UINT64);

// Prepare cell buffers
NativeArray a1 = new NativeArray(ctx, new int[] {1, 2, 3, 4, 5}, Integer.class);

// Create query
Array array = new Array(ctx, arrayURISparse, QueryType.TILEDB_WRITE);
Query query = new Query(array);
query.setLayout(TILEDB_GLOBAL_ORDER);

query.setDataBuffer("d1", d_data);
query.setOffsetsBuffer("d1", d_off);
query.setDataBuffer("a1", a1);

// Submit query
query.submit();

query.finalizeQuery();
query.close();
array.close();

// delete data with appropriate QC
Array array = new Array(ctx, arrayURISparse, QueryType.TILEDB_DELETE);
Query query = new Query(array, TILEDB_DELETE);
QueryCondition deleteQc = new QueryCondition(ctx, "a1", 3, Integer.class, TILEDB_GT);
query.setCondition(deleteQc);
query.submit();
// close resources
query.close();
deleteQc.close();
array.close();

// check if data was deleted
array = new Array(ctx, arrayURISparse, QueryType.TILEDB_READ);

query = new Query(array, QueryType.TILEDB_READ);
query.setDataBuffer("a1", new NativeArray(ctx, 40, Integer.class));

while (query.getQueryStatus() != QueryStatus.TILEDB_COMPLETED) {
  query.submit();
}
int[] a1_buff = (int[]) query.getBuffer("a1");
// a1_buff will be {1, 2, 3};
array.close();
query.close();
deleteQc.close();

Nullable Attributes

Fixed-length, nullable attributes

#include <tiledb/tiledb>
using namespace tiledb;

// Create context
Context ctx;

// Create Array
Domain domain(ctx);
auto d1 = Dimension::create<int64_t>(ctx, "d1", {{1, 10}}, 5);
auto d2 = Dimension::create<int64_t>(ctx, "d2", {{1, 10}}, 5);
domain.add_dimension(d1).add_dimension(d2);

auto a = Attribute::create<double>(ctx, "a");
a.set_cell_val_num(2);   // two values per cell
a.set_nullable(true);    // set nullable

ArraySchema schema(ctx, TILEDB_SPARSE);
schema.set_domain(domain);
schema.set_order({{TILEDB_COL_MAJOR, TILEDB_COL_MAJOR}});
schema.add_attributes(a);

Array::create(array_name, schema);

// Prepare some data for the array
std::vector<int64_t> d1 = {1, 2, 3, 4};
std::vector<int64_t> d2 = {2, 1, 3, 4};
std::vector<double> a = {1.1f, 1.2f, 2.1f, 2.2f, 3.1f, 3.2f, 4.1f, 4.2f};
std::vector<uint8_t> a_validity = {1, 1, 0, 1};

// Open array for writing
Array array(ctx, array_name, TILEDB_WRITE);

// Create the query
Query query(ctx, array);
query.set_layout(TILEDB_UNORDERED)
  .set_data_buffer("d1", d1)
  .set_data_buffer("d2", d2)
  .set_data_buffer("a", a)
  .set_validity_buffer("a", a_validity);

// Submit query
query.submit();
  
// Close the array  
array.close();

# TODO

array_name <- "nullable"

dom <- tiledb_domain(dims = c(tiledb_dim("d1", c(1L, 10L), 5L, "INT32"),
                              tiledb_dim("d2", c(1L, 10L), 5L, "INT32")))
attr <- tiledb_attr("a", type = "FLOAT64", nullable = TRUE, ncells = 2)
sch <- tiledb_array_schema(dom, attr, sparse = TRUE)

rv <- tiledb_array_create(array_name, sch)

# We can also supply the values as a data.frame, and here we use one
# with a list column containing two elements each with one tuple being NULL
D <- data.frame(d1 = c(1, 2, 3, 4),
                d2 = c(2, 1, 3, 4),
                a = I(list(c(1.1, 1.2), c(2.1, 2.2), 
                           c(NA, NA), c(4.1, 4.2))))
arr <- tiledb_array(array_name)
arr[] <- D

## alternatively, we can use higher-level format: we pass the
## data.frame object, the desired URI and indicate index columns
fromDataFrame(D, array_name, col_index=1:2)

## from either write, we can check the content with a high-level accessor
chk <- tiledb_array(array_name, return_as="data.frame")
chk[]

//Create an array with two dimesnions
Dimension<Integer> rows =
  new Dimension<>(ctx, "rows", Integer.class, new Pair<Integer, Integer>(1, 2), 2);
Dimension<Integer> cols =
  new Dimension<>(ctx, "cols", Integer.class, new Pair<Integer, Integer>(1, 2), 2);

// Create and set getDomain
Domain domain = new Domain(ctx);
domain.addDimension(rows);
domain.addDimension(cols);

// Add two attributes "a1" and "a2", so each (i,j) cell can store
// a character on "a1" and a vector of two floats on "a2".
Attribute a1 = new Attribute(ctx, "a1", Character.class);
Attribute a2 = new Attribute(ctx, "a2", Float.class);

//make fixed-length attributes nullable
a1.setNullable(true);
a2.setNullable(true);

ArraySchema schema = new ArraySchema(ctx, TILEDB_DENSE);
schema.setTileOrder(TILEDB_ROW_MAJOR);
schema.setCellOrder(TILEDB_ROW_MAJOR);
schema.setDomain(domain);
schema.addAttribute(a1);
schema.addAttribute(a2);

Array.create(arrayURI, schema);

// Prepare cell buffers
NativeArray a1 = new NativeArray(ctx, "abcd", String.class);
NativeArray a2 = new NativeArray(ctx, new float[] {0.1f, 0.2f, 1.1f, 1.2f}, Float.class);

// Create query
try (Array array = new Array(ctx, arrayURI, TILEDB_WRITE);
    Query query = new Query(array)) {
  query.setLayout(TILEDB_ROW_MAJOR);
  NativeArray a1Bytemap =
      new NativeArray(ctx, new short[] {1, 0, 0, 1}, Datatype.TILEDB_UINT8);
  NativeArray a2Bytemap =
      new NativeArray(ctx, new short[] {1, 1, 1, 0}, Datatype.TILEDB_UINT8);

  //set the buffers
  query.setBufferNullable("a1", a1, a1Bytemap);
  query.setBufferNullable("a2", a2, a2Bytemap);

  // Submit query
  query.submit();
}

// TODO

using TileDB.CSharp;

// Create TileDB context
using Context ctx = new Context();

// Open array for writing
using Array array = new Array(ctx, array_name);
array.Open(QueryType.Write);

// Prepare some data for the array
long[] d1 = new long[] { 1, 2, 3, 4 };
long[] d2 = new long[] { 2, 1, 3, 4 };
float[] a = new float[] { 1.1f, 1.2f, 2.1f, 2.2f, 3.1f, 3.2f, 4.1f, 4.2f };
byte[] aValidity = new byte[] { 1, 1, 0, 1 };

// Create the query
using Query query = new Query(ctx, array, QueryType.Write);
query.SetLayout(LayoutType.Unordered);
query.SetDataBuffer("d1", d1);
query.SetDataBuffer("d2", d2);
query.SetDataBuffer("a", a);
query.SetValidityBuffer("a", aValidity);

// Submit query
query.Submit();

// Close array
array.Close();

The code snippet above produces the following sparse fragment.

Variable-length, Nullable Attributes

#include <tiledb/tiledb.h>

// Create TileDB context
tiledb_ctx_t* ctx;
tiledb_ctx_alloc(NULL, &ctx);

// Open array for writing
tiledb_array_t* array;
tiledb_array_alloc(ctx, array_name, &array);
tiledb_array_open(ctx, array, TILEDB_WRITE);

// Prepare some data for the array
long long d1[] = {1, 2, 3, 4};
unsigned long long d1_size = sizeof(d1);
long long d2[] = {2, 1, 3, 4};
unsigned long long d2_size = sizeof(d2);
int data[] = {1, 1, 2, 3, 3, 3, 4};
unsigned long long a_size = sizeof(a);
unsigned long long a_off[] = {0, 8, 12, 24};
unsigned long long a_off_size = sizeof(a_off);
uint8_t a_validity[] = {1, 1, 0, 1};
unsigned long long a_validity_size = sizeof(a_validity);

// Create the query
tiledb_query_t* query;
tiledb_query_alloc(ctx, array, TILEDB_WRITE, &query);
tiledb_query_set_layout(ctx, query, TILEDB_UNORDERED);
tiledb_query_set_data_buffer(ctx, query, "d1", d1, &d1_size);
tiledb_query_set_data_buffer(ctx, query, "d2", d2, &d2_size);
tiledb_query_set_data_buffer(ctx, query, "a", a, &a_size);
tiledb_query_set_validity_buffer(ctx, query, "a", a_validity, &a_validity_size);

// Submit query
tiledb_query_submit(ctx, query);

// Close array
tiledb_array_close(ctx, array);

// Clean up
tiledb_array_free(&array);
tiledb_query_free(&query);
tiledb_ctx_free(&ctx);

#include <tiledb/tiledb>
using namespace tiledb;

// Create context
Context ctx;

// Prepare some data for the array
std::vector<int64_t> d1 = {1, 2, 3, 4};
std::vector<int64_t> d2 = {2, 1, 3, 4};
std::vector<int32_t> a = {1, 1, 2, 3, 3, 3, 4};
std::vector<uint64_t> a_off = {0, 8, 12, 24};
std::vector<uint8_t> a_validity = {1, 1, 0, 1};

// Open array for writing
Array array(ctx, array_name, TILEDB_WRITE);

// Create the query
Query query(ctx, array);
query.set_layout(TILEDB_UNORDERED)
  .set_data_buffer("d1", d1)
  .set_data_buffer("d2", d2)
  .set_data_buffer("a", a)
  .set_offsets_buffer("a", a_off)
  .set_validity_buffer("a", a_validity);

// Submit query
query.submit();

// Close the array
array.close();

# TODO

## Variable-length nullable (numerical) attributes are not yet supported

//Create an array with two dimesnions
Dimension<Integer> rows =
  new Dimension<>(ctx, "rows", Integer.class, new Pair<Integer, Integer>(1, 2), 2);
Dimension<Integer> cols =
  new Dimension<>(ctx, "cols", Integer.class, new Pair<Integer, Integer>(1, 2), 2);

// Create and set getDomain
Domain domain = new Domain(ctx);
domain.addDimension(rows);
domain.addDimension(cols);

// Add two attributes "a1" and "a2", so each (i,j) cell can store
// a character on "a1" and a vector of two floats on "a2".
Attribute a1 = new Attribute(ctx, "a1", Float.class);
Attribute a2 = new Attribute(ctx, "a2", Datatype.TILEDB_STRING_ASCII);

//make fixed-length attributes nullable
a1.setNullable(true);
a2.setNullable(true);

//make a2 a variable length attribute
a2.setCellVar();

ArraySchema schema = new ArraySchema(ctx, TILEDB_DENSE);
schema.setTileOrder(TILEDB_ROW_MAJOR);
schema.setCellOrder(TILEDB_ROW_MAJOR);
schema.setDomain(domain);
schema.addAttribute(a1);
schema.addAttribute(a2);

Array.create(arrayURI, schema);

// Prepare cell buffers
NativeArray a1 = new NativeArray(ctx, new float[] {0.1f, 0.2f, 1.1f, 1.2f}, Float.class)
NativeArray a2_data = new NativeArray(ctx, "aabbccdd", Datatype.TILEDB_STRING_ASCII);
NativeArray a2_off = new NativeArray(ctx, new long[] {0, 2, 4, 6}, Datatype.TILEDB_UINT64); Float.class);

// Create query
try (Array array = new Array(ctx, arrayURI, TILEDB_WRITE);
    Query query = new Query(array)) {
  query.setLayout(TILEDB_ROW_MAJOR);
  NativeArray a1Bytemap =
      new NativeArray(ctx, new short[] {1, 0, 0, 1}, Datatype.TILEDB_UINT8);
  NativeArray a2Bytemap =
      new NativeArray(ctx, new short[] {1, 1, 1, 0}, Datatype.TILEDB_UINT8);

  //set the buffers
  query.setBufferNullable("a1", a1, a1Bytemap);
  query.setBufferNullable("a2", a2_off, a2_data, a2Bytemap);

  // Submit query
  query.submit();
}

// TODO

using TileDB.CSharp;

// Create TileDB context
using Context ctx = new Context();

// Open array for writing
using Array array = new Array(ctx, array_name);
array.Open(QueryType.Write);

// Prepare some data for the array
long[] d1 = new long[] { 1, 2, 3, 4 };
long[] d2 = new long[] { 2, 1, 3, 4 };
float[] a = new float[] { 1.1f, 1.2f, 2.1f, 2.2f, 3.1f, 3.2f, 4.1f, 4.2f };
ulong[] aOffsets = new ulong[] { 0, 8, 12, 24 };
byte[] aValidity = new byte[] { 1, 1, 0, 1 };

// Create the query
using Query query = new Query(ctx, array, QueryType.Write);
query.SetLayout(LayoutType.Unordered);
query.SetDataBuffer("d1", d1);
query.SetDataBuffer("d2", d2);
query.SetDataBuffer("a", a);
query.SetOffsetsBuffer("a", aOffsets);
query.SetValidityBuffer("a", aValidity);

// Submit query
query.Submit();

// Close array
array.Close();

The code snippet above produces the following sparse fragment.

Writing in Dense Subarrays

This is applicable only to dense arrays (see for more details). You can write into a subarray of the array domain as follows:

#include <tiledb/tiledb>
using namespace tiledb;

// Create context
Context ctx;

// Prepare some data for the array
std::vector<int> data = {1, 2, 3, 4, 5, 6};

// Open array for writing
Array array(ctx, array_name, TILEDB_WRITE);

// Set the subarray to write into
Subarray subarray(ctx, array);
subarray.set_subarray({1, 2, 2, 4});

// Create the query
Query query(ctx, array);
query.set_layout(TILEDB_ROW_MAJOR)
     .set_data_buffer("a", data)
     .set_subarray(subarray);

// NOTE: If the array has more than one attributes, you should
// set the a buffer for every attribute with set_data_buffer.

// You could have set the query layout to TILEDB_COL_MAJOR, 
// and set the data vector to {1, 4, 2, 5, 3, 6}, 
// and the result would be the same

// You could have set the query layout to TILEDB_GLOBAL_ORDER, 
// and set the data vector to {INT_MIN, 1, INT_MIN, 4, 2, 3, 5, 6}, 
// and the result would be the same

// Submit query
query.submit();
  
// Close the array  
array.close();

import tiledb
import numpy as np

data = np.array([[1,2],[3,4],[5,6]])

with tiledb.open(array_name, 'w') as A:
    A[1:3,2:5] = data

# Prepare a 4x3 dense array
# Contrary to Python, R by default stores arrays in col-major order
data <-  array(c(1L, 4L, 2L, 5L, 3L, 6L), dim=c(2,3))

# Prepare the [1,2] x [2,4] subarray to write to
I <- 1:2
J <- 2:4

# Open the array and write the data to it
A <- tiledb_array(uri = array_name)
A[I, J] <- data

// Create context, array and query
try(Context ctx = new Context(),
     Array array = new Array(ctx, array_name, TILEDB_WRITE),
     Query query = new Query(ctx, array);) {

     // Prepare some data for the array
     NativeArray subarray = new NativeArray(ctx, new int[] {1, 2, 2, 4}, Integer.class);
     
     NativeArray data = new NativeArray(ctx, new int[] {1, 2, 3, 4, 5, 6}, Integer.class);
     
     // Create the query
     query.setLayout(TILEDB_ROW_MAJOR)
          .setSubarray(subarray)
          .setBuffer("a", data);


     // NOTE: If the array has more than one attributes, you should
     // set the a buffer for every attribute with set_buffer.
          
     // You could have set the query layout to TILEDB_COL_MAJOR, 
     // and set the data vector to {1, 4, 2, 5, 3, 6}, 
     // and the result would be the same
     
     // You could have set the query layout to TILEDB_GLOBAL_ORDER, 
     // and set the data vector to {INT_MIN, 1, INT_MIN, 4, 2, 3, 5, 6}, 
     // and the result would be the same

     // Submit query
     query.submit();
}

import "github.com/TileDB-Inc/TileDB-Go"

// Create context
ctx, _ := tiledb.NewContext(nil)

// Prepare some data for the array
data := []int32{1, 2, 3, 4, 5, 6}

// Set the subarray to write into
subarray := []int32{1, 2, 2, 4}

// Open array for writing
array, _ := tiledb.NewArray(ctx, array_name)
_ = array.Open(tiledb.TILEDB_WRITE)

// Create the query
query, _ := tiledb.NewQuery(ctx, array)
query.SetLayout(tiledb.TILEDB_ROW_MAJOR)
query.SetBuffer("a", data)
query.SetSubArray(subarray)

// NOTE: If the array has more than one attributes, you should
// set the a buffer for every attribute with set_buffer.
     
// You could have set the query layout to TILEDB_COL_MAJOR,
// and set the data vector to {1, 4, 2, 5, 3, 6},
// and the result would be the same

// You could have set the query layout to TILEDB_GLOBAL_ORDER,
// and set the data vector to {INT_MIN, 1, INT_MIN, 4, 2, 3, 5, 6},
// and the result would be the same

// Submit query
query.Submit()
 
// Close the array  
query.Finalize()
array.Close()

using TileDB.CSharp;

// Create TileDB context
using Context ctx = new Context();

// Open array for writing
using Array array = new Array(ctx, array_name);
array.Open(QueryType.Write);

// Prepare some data for the array
int[] data = new int[] { 1, 2, 3, 4, 5, 6 };

// Set the subarray to write into
using Subarray subarray = new Subarray(array);
subarray.SetSubarray(1, 2, 2, 4);

// Create the query
using Query query = new Query(ctx, array, QueryType.Write);
query.SetLayout(LayoutType.RowMajor);
query.SetSubarray(subarray);
query.SetDataBuffer("a", data);

// NOTE: If the array has more than one attributes, you should
// set the a buffer for every attribute with SetDataBuffer.

// You could have set the query layout to ColumnMajor, 
// and set the data buffer to {1, 4, 2, 5, 3, 6}, 
// and the result would be the same.

// You could have set the query layout to GlobalOrder, 
// and set the data buffer to {int.MinValue, 1, int.MinValue, 4, 2, 3, 5, 6}, 
// and the result would be the same.

// Submit query
query.Submit();

// Close array
array.Close();

Currently, if your array consists of more than one attributes, TileDB requires you to provide values for all the attributes in each write operation.

Consolidation and Vacuuming

Consolidation

Fragments

You can consolidate an array as follows. See for how to fine-tune the consolidation algorithm with configuration parameters, and for more details on creating configuration objects.

# Consolidate array at array_uri
tiledb.consolidate(array_uri)

# Alternatively, you can create and pass a configuration object
config = tiledb.Config(
    {"sm.consolidation.steps": 3, "sm.consolidation.mode": "fragments"}
)
ctx = tiledb.Ctx(config)
tiledb.consolidate(array_uri, ctx=ctx)

# You may also consolidate a subset of fragments across a timestamp range

# Set tuple of timestamp ranges as an argument to the timestamp parameter
tiledb.consolidate(array_uri, timestamp=(5, 9))

# Or set it in the configuration object
config = tiledb.Config(
    {
        "sm.consolidation.timestamp_start": 5,
        "sm.consolidation.timestamp_end": 9,
        "sm.consolidation.mode": "fragments",
    }
)
ctx = tiledb.Ctx(config)
tiledb.consolidate(array_uri, ctx=ctx)

# An array URI
uri <- "<array_uri>"

# Consolidate with default configuration
array_consolidate(uri)
 
# Alteratively, create and pass a configuration object
cfg <- tiledb_config()
cfg["sm.consolidation.steps"] <- "3"
cfg["sm.consolidation.mode"] <- "fragments"
array_consolidate(uri, cfg)

# We can also consolidate over a timestamp range
windowStart <- as.POSIXct("2021-01-01 00:00:00")
windowEnd <- as.POSIXct("2021-06-30 23:59:59.999")
array_consolidate(uri, start_time=windowStart, end_time=windowEnd)

try(Context ctx = new Context() {
    // Consolidate, using ctx's configuration
    consolidate(ctx, "<array-uri>");
    
    // Alteratively, you can create and pass a configuration object
    Config config = new Config();
    config.set("sm.consolidation.steps") = "3";
    config.set("sm.consolidation.mode") = "fragments";
    consolidate(ctx, "<array-uri>", config);
}

// Create configuration
config, _ := NewConfig()

// Test context with config
ctx, _ := NewContext(config)

// Consolidate, using ctx's configuration
config.Set("sm.consolidation.steps", "3")
config.Set("sm.consolidation.mode", "fragments")
array.Consolidate(ctx, config)

using TileDB.CSharp;

// Create TileDB context
using Context ctx = new Context();

// Consolidate, using ctx's configuration
Array.Consolidate(ctx, array_name);

// Alternatively, you can create and pass a configuration object
using Config config = new Config();
config.Set("sm.consolidation.steps", "3");
config.Set("sm.consolidation.mode", "fragments");
Array.Consolidate(ctx, array_name, config);

Fragment Metadata

The same consolidation API is used for consolidating . The only change is configuration parameter "sm.consolidation.mode" which should now be set to "fragment_meta".

Array Metadata

The same consolidation API is used for consolidating . The only change is configuration parameter "sm.consolidation.mode" which should now be set to "array_meta".

Commits

The same consolidation API is used for consolidating commits. The only change is configuration parameter "sm.consolidation.mode" which should now be set to "commits".

Vacuuming

Fragments

// Create TileDB context
tiledb_ctx_t* ctx;
tiledb_ctx_alloc(NULL, &ctx);

// Vacuum, using ctx's configuration
tiledb_array_vacuum(ctx, array_name, nullptr);

// Alteratively, you can create and pass a configuration object
tiledb_config_t* config;
tiledb_config_alloc(&config, NULL);
tiledb_config_set(config, "sm.vacuum.mode", "fragments", &error);
tiledb_array_vacuum(ctx, array_name, config);

// Clean-up
tiledb_ctx_free(&ctx);
tiledb_config_free(&config);

Context ctx;

// Consolidate, using ctx's configuration
Array::vacuum(ctx, "<array-uri>");

// Alteratively, you can create and pass a configuration object
Config config;
config["sm.vacuum.mode"] = "fragments";
Array::vacuum(ctx, "<array-uri>", config);

# Consolidate array at array_uri
tiledb.vacuum(array_uri)

# Alteratively, you can create and pass a configuration object
config = tiledb.Config({"sm.vacuum.mode": "fragments"})

tiledb.vacuum(array_uri, config=config)

# An array URI
uri <- "<array_uri>"

# Vacuum with default configuration
array_vacuum(uri)
 
# Alternatively, create and pass a configuration object
cfg <- tiledb_config()
cfg["sm.vacuum.mode"] <- "fragments"
array_vacuum(uri, cfg)

# We can also vacuum over a timestamp range
windowStart <- as.POSIXct("2021-01-01 00:00:00")
windowEnd <- as.POSIXct("2021-06-30 23:59:59.999")
array_vacuum(uri, start_time=windowStart, end_time=windowEnd)

try(Context ctx = new Context()){
      // Vacuum, using ctx's configuration
      Config conf = new Config();
      conf.set("sm.vacuum.mode", "fragments");
      
      // Alteratively, you can create and pass a configuration object 
      Array.vacuum(ctx,"<array-uri>", conf);
}

// Create configuration
config, _ := NewConfig()

// Test context with config
ctx, _ := NewContext(config)

// Consolidate, using ctx's configuration
config.Set("sm.vacuum.mode", "fragments")
array.Vacuum(ctx, config)I

using TileDB.CSharp;

// Create TileDB context
using Context ctx = new Context();

// Vacuum, using ctx's configuration
Array.Vacuum(ctx, array_name);

// Alternatively, you can create and pass a configuration object
using Config config = new Config();
config.Set("sm.vacuum.mode", "fragments");
Array.Vacuum(ctx, array_name, config);

As with consolidation above, vacuuming inherits the context configuration if no configuration override is specified.

Fragment Metadata

The same vacuuming API is used for vacuuming . The only change is configuration parameter "sm.vacuum.mode" which should now be set to "fragment_meta".

Array Metadata

The same vacuuming API is used for vacuuming . The only change is configuration parameter "sm.vacuum.mode" which should now be set to "array_meta".

Commits

The same vacuuming API is used for vacumming commits. The only change is configuration parameter "sm.vacuum.mode" which should now be set to "commits".