Incomplete Queries
What happens if the buffer you set to a read query is not big enough to hold the results? TileDB is smart enough not to crash in that case. Instead, it will try to fill as many results as possible to your buffers, and through the query status inform you on whether the query completed or if it is incomplete. In the latter case, you can consume the results and resubmit the query (with the same buffers or newly set buffers), and TileDB will pick up where it left off. TileDB also allows you to get a result estimate, but even that does not guarantee whether the buffers will indeed large enough to hold the actual result.
The example below shows how to typically submit a read query to account for the possible case of incomplete queries.
// ... create context ctx
// ... create query
// ... suppose you query attribute "a" with buffer size "a_size"
// Create a loop
tiledb_query_status_t status;
do {
// Submit query and get status
tiledb_query_submit(ctx, query);
tiledb_query_get_status(ctx, query, &status);
// IMPORTANT: check if there are any results, as your buffer
// could have been too small to fit even a single result
if (status == TILEDB_INCOMPLETE && a_size == 0) { // No results
// You need to reallocate your buffers, otherwise
// you will get an infinite loop
} else if (a_size > 0) { // There are results
// Do something with the results
// You could set new buffers to the query here
}
} while (status == TILEDB_INCOMPLETE);
// Other statuses:
// TILEDB_{FAILED, COMPLETED, INPROGRESS, UNINITIALIZED}// ... create context ctx
// ... create query
// ... suppose you query attribute "a"
// Create a loop
Query::Status status;
do {
// Submit query and get status
query.submit();
status = query.query_status();
// IMPORTANT: check if there are any results, as your buffer
// could have been too small to fit even a single result
bool has_results = query.result_buffer_elements()["a"].second != 0
if (status == Query::Status::INCOMPLETE && !has_results)) {
// You need to reallocate your buffers, otherwise
// you will get an infinite loop
} else if (has_results) {
// Do something with the results
// You could set new buffers to the query here
}
} while (status == Query::Status::INCOMPLETE);
// Other statuses:
// Query::Status::{FAILED, COMPLETED, INPROGRESS, UNINITIALIZED}# Note that incomplete queries are only supported for
# sparse arrays in TileDB-Py at this time.
# Dense reads will internally reallocate buffers and resubmit the
# query until successful completion.
with tiledb.open(uri) as A:
# iterate results as an OrderedDict
iterable = A.query(return_incomplete=True).multi_index[:]
# -- or --
# iterate results as a dataframe
iterable = A.query(return_incomplete=True).df[:]
for result in iterable:
# this loop will iterate until the query has
# returned all results for the given range
print(result)
# Querying estimated result size for query:
# Create query object as above:
# iterable = A.query(return_incomplete=True).multi_index[:]
# then call `estimated_result_sizes`, which will return an
# OrderedDict of {'result name': estimate}
# iterable.estimated_result_sizes()
# We create buffers to fit the entire result in memory. If
# there is not enough memory allocated in the buffers to hold
# the complete result, TileDB will signal it, consume as much
# as it can after which one can consume the remainder
ctx <- tiledb_ctx()
arrptr <- tiledb:::libtiledb_array_open(ctx@ptr, uridense,
"READ")
qryptr <- tiledb:::libtiledb_query(ctx@ptr, arrptr, "READ")
subarr <- c(1L,4L, 1L,4L)
qryptr <- tiledb:::libtiledb_query_set_subarray(qryptr, subarr)
vec <- integer(4) # reserve (insufficient) space
qryptr <- tiledb:::libtiledb_query_set_buffer(qryptr, "a", vec)
finished <- FALSE
while (!finished) {
qryptr <- tiledb:::libtiledb_query_submit(qryptr)
print(vec)
finished <-
tiledb:::libtiledb_query_status(qryptr) == "COMPLETE"
}
res <- tiledb:::libtiledb_array_close(arrptr)// ... create context ctx
// ... create query
// ... suppose you query attribute "a"
// Create a loop
QueryStatus status;
do {
// Submit query and get status
status = query.submit();
// IMPORTANT: check if there are any results, as your buffer
// could have been too small to fit even a single result
bool has_results = query.resultBufferElements().get("a").getSecond() != 0
if (status == TILEDB_INCOMPLETE && !has_results)) {
// You need to reallocate your buffers, otherwise
// you will get an infinite loop
} else if (has_results) {
// Do something with the results
// You could set new buffers to the query here
}
} while (status == TILEDB_INCOMPLETE);
// Other statuses:
// Query::Status::{FAILED, COMPLETED, INPROGRESS, UNINITIALIZED}// ... create context ctx
// ... create query
// ... suppose you query attribute "a"
var queryStatus tiledb.QueryStatus
for {
// Submit the query
query.Submit()
queryStatus, _ = query.Status()
// IMPORTANT: check if there are any results, as your buffer
// could have been too small to fit even a single result
elements, _ := query.ResultBufferElements()
resultNum := elements["a1"][1]
if queryStatus == tiledb.TILEDB_INCOMPLETE && resultNum == 0 {
// You need to reallocate your buffers, otherwise
// you will get an infinite loop
} else {
// Do something with the results
// You could set new buffers to the query here
}
if queryStatus != tiledb.TILEDB_INCOMPLETE {
break
}
}
// Other statuses:
// Query::Status::{FAILED, COMPLETED, INPROGRESS, UNINITIALIZED}// ... create context ctx
// ... create query
// ... suppose you query attribute "a"
// Create a loop
QueryStatus status;
do
{
// Submit query and get status
query.Submit();
status = query.Status();
// IMPORTANT: check if there are any results, as your buffer
// could have been too small to fit even a single result
bool hasResults = query.GetResultDataElements("a") != 0;
if (status == QueryStatus.Incomplete
&& query.GetStatusDetails().Reason == QueryStatusDetailsReason.UserBufferSize)
{
// You need to reallocate your buffers, otherwise
// you will get an infinite loop
}
else if (hasResults)
{
// Do something with the results
// You could set new buffers to the query here
}
} while (status == QueryStatus.Incomplete);
// Other statuses:
// QueryStatus.{Failed, Completed, InProgress, Uninitialized}Last updated