TileDB is designed such that all IO to/from the storage backends is abstracted behind a Virtual Filesystem (VFS) module. This module supports simple operations, such as creating a file/directory, reading/writing to a file, etc. This abstraction enables us to easily plug in more storage backends in the future, effectively making the storage backend opaque to the user.
A nice positive “by-product” of this architecture is that it is possible to expose the basic virtual filesystem functionality via the TileDB APIs. This provides a simplified interface for file IO and directory management (i.e., not related to TileDB objects such as array) on all the storage backends that TileDB supports.
This page covers most of the TileDB VFS functionality.
// Create TileDB contexttiledb_ctx_t* ctx;tiledb_ctx_alloc(NULL, &ctx);// Create TileDB VFStiledb_vfs_t* vfs;tiledb_vfs_alloc(ctx, NULL, &vfs);// Write binary datatiledb_vfs_fh_t* fh;tiledb_vfs_open(ctx, vfs, "tiledb_vfs.bin", TILEDB_VFS_WRITE, &fh);float f1 = 153.0;const char* s1 = "abcd";tiledb_vfs_write(ctx, fh, &f1, sizeof(float));tiledb_vfs_write(ctx, fh, s1, strlen(s1));tiledb_vfs_close(ctx, fh);tiledb_vfs_fh_free(&fh);// Write binary data again - this will overwrite the previous filetiledb_vfs_open(ctx, vfs, "tiledb_vfs.bin", TILEDB_VFS_WRITE, &fh);const char* s2 = "abcdef";f1 = 153.1;tiledb_vfs_write(ctx, fh, &f1, sizeof(float));tiledb_vfs_write(ctx, fh, s2, strlen(s2));tiledb_vfs_close(ctx, fh);tiledb_vfs_fh_free(&fh);// Append binary data to existing filetiledb_vfs_open(ctx, vfs, "tiledb_vfs.bin", TILEDB_VFS_APPEND, &fh);const char* s3 = "ghijkl";tiledb_vfs_write(ctx, fh, s3, strlen(s3));tiledb_vfs_close(ctx, fh);tiledb_vfs_fh_free(&fh);// Clean uptiledb_vfs_free(&vfs);tiledb_ctx_free(&ctx);
// Create TileDB contextContext ctx;// Create TileDB VFSVFS vfs(ctx);// Create VFS file bufferVFS::filebuf fbuf(vfs);// Write binary datafbuf.open("tiledb_vfs.bin", std::ios::out);std::ostream os(&fbuf);if (!os.good()) {std::cerr << "Error opening file 'tiledb_vfs_bin'.\n";return;}float f1 = 153.0;std::string s1 = "abcd";os.write((char*)&f1, sizeof(f1));os.write(s1.data(), s1.size());// Write binary data again - this will overwrite the previous filefbuf.open("tiledb_vfs.bin", std::ios::out);if (!os.good()) {std::cerr << "Error opening file 'tiledb_vfs.bin' for write.\n";return;}f1 = 153.1;s1 = "abcdef";os.write((char*)&f1, sizeof(f1));os.write(s1.data(), s1.size());// Append binary data to existing file (this will NOT work on S3)fbuf.open("tiledb_vfs.bin", std::ios::app);if (!os.good()) {std::cerr << "Error opening file 'tiledb_vfs.bin' for append.\n";return;}s1 = "ghijkl";os.write(s1.data(), s1.size());
# Note: the Python VFS API currently supports bytes (no automatic conversion)# therefore encoding must be handled manually (see calls to 'encode'# and 'struct.pack' below)import structctx = tiledb.Ctx()vfs = tiledb.VFS(ctx=ctx)# Create and open writable buffer objectfh = vfs.open("tiledb_vfs.bin", 'w')vfs.write(fh, struct.pack('<f', 153.0))vfs.write(fh, "abcd".encode('UTF-8'))# Write data again - this will overwrite the previous filefh = vfs.open("tiledb_vfs.bin", 'w')vfs.write(fh, struct.pack('<f', 153.1))vfs.write(fh, "abcdef".encode("UTF-8"))# Append data to existing file (this will NOT work on S3)fh = vfs.open("tiledb_vfs.bin", 'a')vfs.write(fh, "ghijkl".encode("UTF-8"))# Close the handlevfs.close(fh)
# binary file to be writtenbinfile <- "tiledb_vfs.bin"# open filefhbuf <- tiledb_vfs_open(uri, "WRITE")# create a binary payload from a serialized R objectpayload <- as.integer(serialize(list(dbl=153, string="abcde"), NULL))# write it and close filetiledb_vfs_write(fhbuf, payload)tiledb_vfs_close(fhbuf)# write again overwriting previous writefhbuf <- tiledb_vfs_open(uri, "WRITE")payload <- as.integer(serialize(list(dbl=153.1, string="abcdef"), NULL))tiledb_vfs_write(fhbuf, payload)tiledb_vfs_close(fhbuf)# append to existing filefhbuf <- tiledb_vfs_open(uri, "APPEND")payload <- as.integer(serialize(c(string="ghijkl"), NULL))tiledb_vfs_write(fhbuf, payload)tiledb_vfs_close(fhbuf)
// Create TileDB context and VFStry (Context ctx = new Context(); VFS vfs = new VFS(ctx)) {String sourcePath = "tiledb_vfs.bin";// Write binary datavfs.write(sourcePath, ByteBuffer.allocate(4).putFloat(153.0).array());byte[] resultBytes = vfs.readAllBytes(sourcePath);// append bytesString s1 = "abcd";vfs.write(sourcePath, s1.getBytes(), VFSMode.TILEDB_VFS_APPEND);// Write binary data again - this will overwrite the previous filevfs.write(sourcePath, ByteBuffer.allocate(4).putFloat(153.0).array(), VFSMode.TILEDB_VFS_WRITE);s1 = "abcdef";vfs.write(sourcePath, s1.getBytes(), VFSMode.TILEDB_VFS_APPEND);// Append binary data to existing file (this will NOT work on S3)s1 = "ghijkl";vfs.write(sourcePath, s1.getBytes(), VFSMode.TILEDB_VFS_APPEND);resultBytes = vfs.readAllBytes(sourcePath);}
// Create TileDB contextconfig, _ := NewConfig()ctx, _ := NewContext(config)// Create TileDB VFSvfs, _ := NewVFS(ctx, config)// Write binary datafh1, _ := vfs.Open("tiledb_vfs.bin", tiledb.TILEDB_VFS_WRITE)var f1 float32 = 153.0s1 := "abcd"vfs.Write(fh1, float32ToBytes(f1))vfs.Write(fh1, []byte(s1))vfs.Close(fh1)// Write binary data again - this will overwrite the previous filefh2, _ := vfs.Open("tiledb_vfs.bin", tiledb.TILEDB_VFS_WRITE)var f2 float32 = 153.1s2 := "abcdef"vfs.Write(fh2, float32ToBytes(f2))vfs.Write(fh2, []byte(s2))vfs.Close(fh2)// Append binary data to existing filefh3, _ := vfs.Open("tiledb_vfs.bin", tiledb.TILEDB_VFS_APPEND)s3 := "ghijkl"vfs.Write(fh3, []byte(s3))vfs.Close(fh3)// Clean upvfs.Free()ctx.Free()func float32ToBytes(float float32) []byte {bits := math.Float32bits(float)bytes := make([]byte, 4)binary.LittleEndian.PutUint32(bytes, bits)return bytes}
// Create TileDB contexttiledb_ctx_t* ctx;tiledb_ctx_alloc(NULL, &ctx);// Create TileDB VFStiledb_vfs_t* vfs;tiledb_vfs_alloc(ctx, NULL, &vfs);// Read binary datatiledb_vfs_fh_t* fh;tiledb_vfs_open(ctx, vfs, "tiledb_vfs.bin", TILEDB_VFS_READ, &fh);float f1;char s1[13];s1[12] = '\0';tiledb_vfs_read(ctx, fh, 0, &f1, sizeof(float));tiledb_vfs_read(ctx, fh, sizeof(float), s1, 12);printf("Binary read:\n%.1f\n%s\n", f1, s1);// Clean uptiledb_vfs_fh_free(&fh);tiledb_vfs_free(&vfs);tiledb_ctx_free(&ctx);
// Create TileDB contextContext ctx;// Create TileDB VFSVFS vfs(ctx);// Read binary dataVFS::filebuf sbuf(vfs);sbuf.open("tiledb_vfs.bin", std::ios::in);std::istream is(&sbuf);if (!is.good()) {std::cerr << "Error opening file 'tiledb_vfs.bin'.\n";return;}float f1;std::string s1;auto s1_size = vfs.file_size("tiledb_vfs.bin") - sizeof(float);s1.resize(s1_size);is.read((char*)&f1, sizeof(f1));is.read((char*)s1.data(), 12);std::cout << "Binary read:\n" << f1 << '\n' << s1 << '\n';
import structctx = tiledb.Ctx()vfs = tiledb.VFS(ctx=ctx)# Create and open readable handlefh = vfs.open("tiledb_vfs.bin", "r")float_struct = struct.Struct('<f')float_data = vfs.read(fh, 0, float_struct.size)string_data = vfs.read(fh, float_struct.size, 12)print(float_struct.unpack(float_data)[0])print(string_data.decode("UTF-8"))vfs.close(fh)
# open a binary file for readingfhbuf <- tiledb_vfs_open(uri, "READ")vec <- tiledb_vfs_read(fhbuf, as.integer64(0), as.integer64(488))tiledb_vfs_close(fhbuf)print(str(unserialize(as.raw(vec))))
// Create TileDB context and VFStry (Context ctx = new Context(); VFS vfs = new VFS(ctx)) {String sourcePath = "tiledb_vfs.bin";byte[] resultBytes = vfs.readAllBytes(sourcePath);float f1 = ByteBuffer.wrap(resultBytes).order(ByteOrder.LITTLE_ENDIAN).getFloat();resultBytes = copyOfRange(resultBytes, 4, resultBytes.length);String s1 = new String( bytes, StandardCharsets.UTF_8 );System.out.printf("Binary read:%f\n,%s\n", f1, s1);}
// Create TileDB contextconfig, _ := NewConfig()ctx, _ := NewContext(config)// Create TileDB VFSvfs, _ := NewVFS(ctx, config)// Read binary datavfs.Open("tiledb_vfs.bin", tiledb.TILEDB_VFS_READ)sizeOfFile, _ := vfs.FileSize(vfsFileName)var f float32 = 0.0sizeOfFloat32 := uint64(unsafe.Sizeof(f))f1, _ := vfs.Read(fh, 0, sizeOfFloat32)s1, _ := vfs.Read(fh, sizeOfFloat32, sizeOfFile-sizeOfFloat32)fmt.Println("Binary read:")fmt.Println(float32FromBytes(f1))fmt.Println(string(s1))// Clean upvfs.Close(fh)vfs.Free()ctx.Free()func float32FromBytes(bytes []byte) float32 {bits := binary.LittleEndian.Uint32(bytes)float := math.Float32frombits(bits)return float}
// Create TileDB contexttiledb_ctx_t* ctx;tiledb_ctx_alloc(NULL, &ctx);// Create TileDB VFStiledb_vfs_t* vfs;tiledb_vfs_alloc(ctx, NULL, &vfs);// Create directoryint is_dir = 0;tiledb_vfs_is_dir(ctx, vfs, "dir_A", &is_dir);if (!is_dir) {tiledb_vfs_create_dir(ctx, vfs, "dir_A");printf("Created 'dir_A'\n");} else {printf("'dir_A' already exists\n");}// Creating an (empty) fileint is_file = 0;tiledb_vfs_is_file(ctx, vfs, "dir_A/file_A", &is_file);if (!is_file) {tiledb_vfs_touch(ctx, vfs, "dir_A/file_A");printf("Created empty file 'dir_A/file_A'\n");} else {printf("'dir_A/file_A' already exists\n");}// Getting the file sizeuint64_t file_size;tiledb_vfs_file_size(ctx, vfs, "dir_A/file_A", &file_size);// Moving files (moving directories is similar)tiledb_vfs_move_file(ctx, vfs, "dir_A/file_A", "dir_A/file_B");// Deleting files and directories. Note that, in the case of directories,// the function will delete all the contents of the directory (i.e., it// works even for non-empty directories).tiledb_vfs_remove_file(ctx, vfs, "dir_A/file_B");tiledb_vfs_remove_dir(ctx, vfs, "dir_A");// Clean uptiledb_vfs_free(&vfs);tiledb_ctx_free(&ctx);
// Create TileDB contextContext ctx;// Create TileDB VFSVFS vfs(ctx);// Create directoryif (!vfs.is_dir("dir_A")) {vfs.create_dir("dir_A");std::cout << "Created 'dir_A'\n";} else {std::cout << "'dir_A' already exists\n";}// Creating an (empty) fileif (!vfs.is_file("dir_A/file_A")) {vfs.touch("dir_A/file_A");std::cout << "Created empty file 'dir_A/file_A'\n";} else {std::cout << "'dir_A/file_A' already exists\n";}// Getting the file sizestd::cout << "Size of file 'dir_A/file_A': " << vfs.file_size("dir_A/file_A")<< "\n";// Moving files (moving directories is similar)vfs.move_file("dir_A/file_A", "dir_A/file_B");// Deleting files and directoriesvfs.remove_file("dir_A/file_B");vfs.remove_dir("dir_A");
import tiledbctx = tiledb.Ctx()vfs = tiledb.VFS(ctx=ctx)# Creating a directoryif not vfs.is_dir("dir_A"):vfs.create_dir("dir_A")print("Created 'dir_A'")else:print ("'dir_A' already exists")# Creating an (empty) fileif not vfs.is_file("dir_A/file_A"):vfs.touch("dir_A/file_A")print("Created empty file 'dir_A/file_A'")else:print("'dir_A/file_A' already exists")# Getting the file sizeprint("Size of file 'dir_A/file_A': ", vfs.file_size("dir_A/file_A"))# Moving files (moving directories is similar)vfs.move_file("dir_A/file_A", "dir_A/file_B")vfs.remove_file("dir_A/file_B")vfs.remove_dir("dir_A")
library(tiledb)ctx <- tiledb_get_context()vfs <- tiledb_vfs(ctx=ctx)## Creating a directoryif (!tiledb_vfs_is_dir(vfs, "dir_A")) {tiledb_vfs_create_dir(vfs, "dir_A")cat("Created 'dir_A'\n")} else {cat("'dir_A' already exists\n")}## Creating an (empty) fileif (!tiledb_vfs_is_file(vfs, "dir_A/file_A")) {tiledb_vfs_touch(vfs, "dir_A/file_A")cat("Created empty file 'dir_A/file_A'\n")} else {cat("File 'dir_A/file_A' already existed\n")}## Getting the file sizecat("Size of file 'dir_A/file_A': ",tiledb_vfs_file_size(vfs, "dir_A/file_A"), "\n")## Moving files (moving directories is similar)tiledb_vfs_move_file(vfs, "dir_A/file_A", "dir_A/file_B")## Cleaning uptiledb_vfs_remove_file(vfs, "dir_A/file_B")tiledb_vfs_remove_dir(vfs, "dir_A")
// Create TileDB context and VFStry (Context ctx = new Context(); VFS vfs = new VFS(ctx)) {// Create directoryif (!vfs.isDirectory("dir_A")) {vfs.createDirectory("dir_A");System.out.println( "Created 'dir_A'");} else {System.out.println("'dir_A' already exists");}// Creating an (empty) fileif (!vfs.isFile("dir_A/file_A")) {vfs.createFile("dir_A/file_A");System.out.println("Created empty file 'dir_A/file_A'");} else {System.out.println("'dir_A/file_A' already exists");}// Getting the file sizeSystem.out.printf("Size of file 'dir_A/file_A': %d\n" , vfs.fileSize("dir_A/file_A"));// Moving files (moving directories is similar)vfs.moveFile("dir_A/file_A", "dir_A/file_B");// Deleting files and directoriesvfs.removeFile("dir_A/file_B");vfs.removeDirectory("dir_A");}
// Create TileDB contextconfig, _ := NewConfig()ctx, _ := NewContext(config)// Create TileDB VFSvfs, _ := NewVFS(ctx, config)// Create directoryvfs.CreateDir("dir_A")isDir, _ := vfs.IsDir("dir_A")if !is_dir {tiledb_vfs_create_dir(ctx, vfs, "dir_A")fmt.Println("Created 'dir_A'\n")} else {fmt.Println("'dir_A' already exists\n")}// Creating an (empty) fileisFile, _ := vfs.IsFile("dir_A/file_A")if !isFile {vfs.Touch("dir_A/file_A")fmt.Println("Created empty file 'dir_A/file_A'\n")} else {fmt.Println("'dir_A/file_A' already exists\n")}// Getting the file sizefileSize, _ := vfs.FileSize("dir_A/file_A")// Moving files (moving directories is similar)vfs.MoveFile("dir_A/file_A", "dir_A/file_B")// Deleting files and directories. Note that, in the case of directories,// the function will delete all the contents of the directory (i.e., it// works even for non-empty directories).vfs.RemoveFile(ctx, vfs, "dir_A/file_B")vfs.RemoveDir(ctx, vfs, "dir_A")// Clean upvfs.Free()ctx.Free()
TileDB allows you to create/delete S3 buckets via its VFS functionality,
// ... create context ctx// ... create VFS vfstiledb_vfs_create_bucket(ctx, vfs, "s3://my_bucket");tiledb_vfs_remove_bucket(ctx, vfs, "s3://my_bucket");
// ... create context ctx// ... create VFS vfsvfs.create_bucket("s3://my_bucket");vfs.remove_bucket("s3://my_bucket");
# ... create context ctx# ... create VFS vfsvfs.create_bucket("s3://my_bucket")vfs.remove_bucket("s3://my_bucket")
# ... create context ctx# ... create VFS vfstiledb_vfs_create_bucket(vfs, "s3://my_bucket")tiledb_vfs_remove_bucket(vfs, "s3://my_bucket")
// Create TileDB context and VFStry (Context ctx = new Context(); VFS vfs = new VFS(ctx)) {vfs.createBucket("s3://my_bucket");vfs.removeBucket("s3://my_bucket");}
// ... create VFS vfsvfs.CreateBucket("s3://my_bucket")vfs.RemoveBucket("s3://my_bucket")
However, extreme care must be taken when creating/deleting buckets on AWS S3. After its creation, a bucket may take some time to “appear” in the system. This will cause problems if the user creates the bucket and immediately tries to write a file in it. Similarly, deleting a bucket may not take effect immediately and, therefore, it may continue to “exist” for some time.
You can configure VFS by passing a configuration object upon its creation.
// Create TileDB contexttiledb_ctx_t* ctx;tiledb_ctx_alloc(NULL, &ctx);// Create a configuration objecttiledb_config_t *config;tiledb_config_alloc(&config, NULL);tiledb_config_set(config, "vfs.file.max_parallel_ops", "16", NULL);// Create TileDB VFS with a config objecttiledb_vfs_t* vfs;tiledb_vfs_alloc(ctx, config, &vfs);// Clean uptiledb_config_free(&config);tiledb_vfs_free(&vfs);tiledb_ctx_free(&ctx);
// Create TileDB contextContext ctx;// Create a configuration objectConfig config;config["vfs.file.max_parallel_ops"] = "16";// Create TileDB VFS with a config objectVFS vfs(ctx, config);
ctx = tiledb.Ctx()config = tiledb.Config()config["vfs.file.max_parallel_ops"] = 16vfs = tiledb.VFS(ctx=ctx, config=config)# Or create the Config first and pass to the Ctx constructor
ctx <- tiledb_get_context()config <- tiledb_config()config["vfs.file.max_parallel_ops"] <- 16vfs <- tiledb_vfs(config, ctx)## Or create the Config first and pass to the Ctx constructor
// Create TileDB contexttry(Context ctx = new Context()) {HashMap<String, String> settings = new HashMap<>();// Set valuessettings.put("vfs.file.max_parallel_ops", "16");// Create TileDB VFS with a config objecttry (Config config = new Config(settings), VFS vfs = new VFS(ctx, config);) {}}
// Create a configuration objectconfig, _ := tiledb.NewConfig()// Create TileDB contextctx, _ := tiledb.NewContext(config)// Set vfs paramconfigCtx, err := ctx.Config()configCtx.Set("vfs.file.max_parallel_ops", "16")// Create TileDB VFS with a config objectvfs, _ := tiledb.NewVFS(ctx, config)// Clean upconfig.Free()vfs.Free()ctx.Free()
If you do not set a configuration object to VFS, then VFS will inherit the (default or set) configuration of the context. Otherwise, the set options in the passed configuration object will override those of the context's, but the rest of the options will still be inherited from the context's configuration.