Virtual Filesystem

TileDB is designed such that all IO to/from the storage backends is abstracted behind a Virtual Filesystem (VFS) module. This module supports simple operations, such as creating a file/directory, reading/writing to a file, etc. This abstraction enables us to easily plug in more storage backends in the future, effectively making the storage backend opaque to the user.
A nice positive “by-product” of this architecture is that it is possible to expose the basic virtual filesystem functionality via the TileDB APIs. This provides a simplified interface for file IO and directory management (i.e., not related to TileDB objects such as array) on all the storage backends that TileDB supports.
This page covers most of the TileDB VFS functionality.
Writing
C
C++
Python
R
Java
Go
1
// Create TileDB context
2
tiledb_ctx_t* ctx;
3
tiledb_ctx_alloc(NULL, &ctx);
4
​
5
// Create TileDB VFS
6
tiledb_vfs_t* vfs;
7
tiledb_vfs_alloc(ctx, NULL, &vfs);
8
​
9
// Write binary data
10
tiledb_vfs_fh_t* fh;
11
tiledb_vfs_open(ctx, vfs, "tiledb_vfs.bin", TILEDB_VFS_WRITE, &fh);
12
float f1 = 153.0;
13
const char* s1 = "abcd";
14
tiledb_vfs_write(ctx, fh, &f1, sizeof(float));
15
tiledb_vfs_write(ctx, fh, s1, strlen(s1));
16
tiledb_vfs_close(ctx, fh);
17
tiledb_vfs_fh_free(&fh);
18
​
19
// Write binary data again - this will overwrite the previous file
20
tiledb_vfs_open(ctx, vfs, "tiledb_vfs.bin", TILEDB_VFS_WRITE, &fh);
21
const char* s2 = "abcdef";
22
f1 = 153.1;
23
tiledb_vfs_write(ctx, fh, &f1, sizeof(float));
24
tiledb_vfs_write(ctx, fh, s2, strlen(s2));
25
tiledb_vfs_close(ctx, fh);
26
tiledb_vfs_fh_free(&fh);
27
​
28
// Append binary data to existing file
29
tiledb_vfs_open(ctx, vfs, "tiledb_vfs.bin", TILEDB_VFS_APPEND, &fh);
30
const char* s3 = "ghijkl";
31
tiledb_vfs_write(ctx, fh, s3, strlen(s3));
32
tiledb_vfs_close(ctx, fh);
33
tiledb_vfs_fh_free(&fh);
34
​
35
// Clean up
36
tiledb_vfs_free(&vfs);
37
tiledb_ctx_free(&ctx);
Copied!
1
// Create TileDB context
2
Context ctx;
3
​
4
// Create TileDB VFS
5
VFS vfs(ctx);
6
​
7
// Create VFS file buffer
8
VFS::filebuf fbuf(vfs);
9
​
10
// Write binary data
11
fbuf.open("tiledb_vfs.bin", std::ios::out);
12
std::ostream os(&fbuf);
13
if (!os.good()) {
14
  std::cerr << "Error opening file  'tiledb_vfs_bin'.\n";
15
  return;
16
}
17
float f1 = 153.0;
18
std::string s1 = "abcd";
19
os.write((char*)&f1, sizeof(f1));
20
os.write(s1.data(), s1.size());
21
​
22
// Write binary data again - this will overwrite the previous file
23
fbuf.open("tiledb_vfs.bin", std::ios::out);
24
if (!os.good()) {
25
  std::cerr << "Error opening file 'tiledb_vfs.bin' for write.\n";
26
  return;
27
}
28
f1 = 153.1;
29
s1 = "abcdef";
30
os.write((char*)&f1, sizeof(f1));
31
os.write(s1.data(), s1.size());
32
​
33
// Append binary data to existing file (this will NOT work on S3)
34
fbuf.open("tiledb_vfs.bin", std::ios::app);
35
if (!os.good()) {
36
  std::cerr << "Error opening file 'tiledb_vfs.bin' for append.\n";
37
  return;
38
}
39
s1 = "ghijkl";
40
os.write(s1.data(), s1.size());
Copied!
1
# Note: the Python VFS API currently supports bytes (no automatic conversion)
2
#       therefore encoding must be handled manually (see calls to 'encode' 
3
#       and 'struct.pack' below)
4
​
5
import struct
6
​
7
ctx = tiledb.Ctx()
8
vfs = tiledb.VFS(ctx=ctx)
9
​
10
# Create and open writable buffer object
11
fh = vfs.open("tiledb_vfs.bin", 'w')
12
vfs.write(fh, struct.pack('<f', 153.0))
13
vfs.write(fh, "abcd".encode('UTF-8'))
14
​
15
# Write data again - this will overwrite the previous file
16
fh = vfs.open("tiledb_vfs.bin", 'w')
17
vfs.write(fh, struct.pack('<f', 153.1))
18
vfs.write(fh, "abcdef".encode("UTF-8"))
19
​
20
# Append data to existing file (this will NOT work on S3)
21
fh = vfs.open("tiledb_vfs.bin", 'a')
22
vfs.write(fh, "ghijkl".encode("UTF-8"))
23
​
24
# Close the handle
25
vfs.close(fh)
Copied!
1
# binary file to be written
2
binfile <- "tiledb_vfs.bin"
3
# open file
4
fhbuf <- tiledb_vfs_open(uri, "WRITE")
5
​
6
# create a binary payload from a serialized R object
7
payload <- as.integer(serialize(list(dbl=153, string="abcde"), NULL))
8
# write it and close file
9
tiledb_vfs_write(fhbuf, payload)
10
tiledb_vfs_close(fhbuf)
11
​
12
# write again overwriting previous write
13
fhbuf <- tiledb_vfs_open(uri, "WRITE")
14
payload <- as.integer(serialize(list(dbl=153.1, string="abcdef"), NULL))
15
tiledb_vfs_write(fhbuf, payload)
16
tiledb_vfs_close(fhbuf)
17
​
18
# append to existing file
19
fhbuf <- tiledb_vfs_open(uri, "APPEND")
20
payload <- as.integer(serialize(c(string="ghijkl"), NULL))
21
tiledb_vfs_write(fhbuf, payload)
22
tiledb_vfs_close(fhbuf)
Copied!
1
// Create TileDB context and VFS
2
try (Context ctx = new Context(); VFS vfs = new VFS(ctx)) {
3
    String sourcePath = "tiledb_vfs.bin";
4
    
5
    // Write binary data
6
    vfs.write(sourcePath, ByteBuffer.allocate(4).putFloat(153.0).array());
7
​
8
    byte[] resultBytes = vfs.readAllBytes(sourcePath);
9
​
10
    // append bytes
11
    String s1 = "abcd";
12
    vfs.write(sourcePath, s1.getBytes(), VFSMode.TILEDB_VFS_APPEND);
13
​
14
    // Write binary data again - this will overwrite the previous file
15
    vfs.write(sourcePath, ByteBuffer.allocate(4).putFloat(153.0).array(), VFSMode.TILEDB_VFS_WRITE);
16
    s1 = "abcdef";
17
    vfs.write(sourcePath, s1.getBytes(), VFSMode.TILEDB_VFS_APPEND);
18
       
19
    // Append binary data to existing file (this will NOT work on S3)
20
    s1 = "ghijkl";
21
    vfs.write(sourcePath, s1.getBytes(), VFSMode.TILEDB_VFS_APPEND);
22
​
23
    resultBytes = vfs.readAllBytes(sourcePath);
24
}
Copied!
1
// Create TileDB context
2
config, _ := NewConfig()
3
ctx, _ := NewContext(config)
4
​
5
// Create TileDB VFS
6
vfs, _ := NewVFS(ctx, config)
7
​
8
// Write binary data
9
fh1, _ := vfs.Open("tiledb_vfs.bin", tiledb.TILEDB_VFS_WRITE)
10
var f1 float32 = 153.0
11
s1 := "abcd"
12
vfs.Write(fh1, float32ToBytes(f1))
13
vfs.Write(fh1, []byte(s1))
14
vfs.Close(fh1)
15
​
16
// Write binary data again - this will overwrite the previous file
17
fh2, _ := vfs.Open("tiledb_vfs.bin", tiledb.TILEDB_VFS_WRITE)
18
var f2 float32 = 153.1
19
s2 := "abcdef"
20
vfs.Write(fh2, float32ToBytes(f2))
21
vfs.Write(fh2, []byte(s2))
22
vfs.Close(fh2)
23
​
24
// Append binary data to existing file
25
fh3, _ := vfs.Open("tiledb_vfs.bin", tiledb.TILEDB_VFS_APPEND)
26
s3 := "ghijkl"
27
vfs.Write(fh3, []byte(s3))
28
vfs.Close(fh3)
29
​
30
// Clean up
31
vfs.Free()
32
ctx.Free()
33
​
34
func float32ToBytes(float float32) []byte {
35
    bits := math.Float32bits(float)
36
    bytes := make([]byte, 4)
37
    binary.LittleEndian.PutUint32(bytes, bits)
38
    return bytes
39
}
Copied!
Reading
C
C++
Python
R
Java
Go
1
// Create TileDB context
2
tiledb_ctx_t* ctx;
3
tiledb_ctx_alloc(NULL, &ctx);
4
​
5
// Create TileDB VFS
6
tiledb_vfs_t* vfs;
7
tiledb_vfs_alloc(ctx, NULL, &vfs);
8
​
9
// Read binary data
10
tiledb_vfs_fh_t* fh;
11
tiledb_vfs_open(ctx, vfs, "tiledb_vfs.bin", TILEDB_VFS_READ, &fh);
12
float f1;
13
char s1[13];
14
s1[12] = '\0';
15
tiledb_vfs_read(ctx, fh, 0, &f1, sizeof(float));
16
tiledb_vfs_read(ctx, fh, sizeof(float), s1, 12);
17
printf("Binary read:\n%.1f\n%s\n", f1, s1);
18
​
19
// Clean up
20
tiledb_vfs_fh_free(&fh);
21
tiledb_vfs_free(&vfs);
22
tiledb_ctx_free(&ctx);
Copied!
1
// Create TileDB context
2
Context ctx;
3
​
4
// Create TileDB VFS
5
VFS vfs(ctx);
6
​
7
// Read binary data
8
VFS::filebuf sbuf(vfs);
9
sbuf.open("tiledb_vfs.bin", std::ios::in);
10
std::istream is(&sbuf);
11
if (!is.good()) {
12
  std::cerr << "Error opening file 'tiledb_vfs.bin'.\n";
13
  return;
14
}
15
​
16
float f1;
17
std::string s1;
18
auto s1_size = vfs.file_size("tiledb_vfs.bin") - sizeof(float);
19
s1.resize(s1_size);
20
​
21
is.read((char*)&f1, sizeof(f1));
22
is.read((char*)s1.data(), 12);
23
std::cout << "Binary read:\n" << f1 << '\n' << s1 << '\n';
Copied!
1
import struct
2
​
3
ctx = tiledb.Ctx()
4
vfs = tiledb.VFS(ctx=ctx)
5
​
6
# Create and open readable handle 
7
fh = vfs.open("tiledb_vfs.bin", "r")
8
float_struct = struct.Struct('<f')
9
​
10
float_data = vfs.read(fh, 0, float_struct.size)
11
string_data = vfs.read(fh, float_struct.size, 12)
12
​
13
print(float_struct.unpack(float_data)[0])
14
print(string_data.decode("UTF-8"))
15
​
16
vfs.close(fh)
Copied!
1
# open a binary file for reading
2
fhbuf <- tiledb_vfs_open(uri, "READ")
3
vec <- tiledb_vfs_read(fhbuf, as.integer64(0), as.integer64(488))
4
tiledb_vfs_close(fhbuf)
5
print(str(unserialize(as.raw(vec))))
6
​
Copied!
1
// Create TileDB context and VFS
2
try (Context ctx = new Context(); VFS vfs = new VFS(ctx)) {
3
    String sourcePath = "tiledb_vfs.bin";
4
    byte[] resultBytes = vfs.readAllBytes(sourcePath);
5
    
6
    float f1 = ByteBuffer.wrap(resultBytes).order(ByteOrder.LITTLE_ENDIAN).getFloat();
7
    resultBytes = copyOfRange(resultBytes, 4, resultBytes.length);
8
    
9
    String s1 = new String( bytes, StandardCharsets.UTF_8 );
10
    
11
    System.out.printf("Binary read:%f\n,%s\n", f1, s1);
12
}
Copied!
1
// Create TileDB context
2
config, _ := NewConfig()
3
ctx, _ := NewContext(config)
4
​
5
// Create TileDB VFS
6
vfs, _ := NewVFS(ctx, config)
7
​
8
// Read binary data
9
vfs.Open("tiledb_vfs.bin", tiledb.TILEDB_VFS_READ)
10
​
11
sizeOfFile, _ := vfs.FileSize(vfsFileName)
12
    
13
var f float32 = 0.0
14
sizeOfFloat32 := uint64(unsafe.Sizeof(f))
15
f1, _ := vfs.Read(fh, 0, sizeOfFloat32)
16
s1, _ := vfs.Read(fh, sizeOfFloat32, sizeOfFile-sizeOfFloat32)
17
fmt.Println("Binary read:")
18
fmt.Println(float32FromBytes(f1))
19
fmt.Println(string(s1))
20
​
21
// Clean up
22
vfs.Close(fh)
23
vfs.Free()
24
ctx.Free()
25
​
26
func float32FromBytes(bytes []byte) float32 {
27
    bits := binary.LittleEndian.Uint32(bytes)
28
    float := math.Float32frombits(bits)
29
    return float
30
}
Copied!
Managing
C
C++
Python
R
Java
Go
1
// Create TileDB context
2
tiledb_ctx_t* ctx;
3
tiledb_ctx_alloc(NULL, &ctx);
4
​
5
// Create TileDB VFS
6
tiledb_vfs_t* vfs;
7
tiledb_vfs_alloc(ctx, NULL, &vfs);
8
​
9
// Create directory
10
int is_dir = 0;
11
tiledb_vfs_is_dir(ctx, vfs, "dir_A", &is_dir);
12
if (!is_dir) {
13
  tiledb_vfs_create_dir(ctx, vfs, "dir_A");
14
  printf("Created 'dir_A'\n");
15
} else {
16
  printf("'dir_A' already exists\n");
17
}
18
​
19
// Creating an (empty) file
20
int is_file = 0;
21
tiledb_vfs_is_file(ctx, vfs, "dir_A/file_A", &is_file);
22
if (!is_file) {
23
  tiledb_vfs_touch(ctx, vfs, "dir_A/file_A");
24
  printf("Created empty file 'dir_A/file_A'\n");
25
} else {
26
  printf("'dir_A/file_A' already exists\n");
27
}
28
​
29
// Getting the file size
30
uint64_t file_size;
31
tiledb_vfs_file_size(ctx, vfs, "dir_A/file_A", &file_size);
32
​
33
// Moving files (moving directories is similar)
34
tiledb_vfs_move_file(ctx, vfs, "dir_A/file_A", "dir_A/file_B");
35
​
36
// Deleting files and directories. Note that, in the case of directories,
37
// the function will delete all the contents of the directory (i.e., it
38
// works even for non-empty directories).
39
tiledb_vfs_remove_file(ctx, vfs, "dir_A/file_B");
40
tiledb_vfs_remove_dir(ctx, vfs, "dir_A");
41
​
42
// Clean up
43
tiledb_vfs_free(&vfs);
44
tiledb_ctx_free(&ctx);
Copied!
1
// Create TileDB context
2
Context ctx;
3
​
4
// Create TileDB VFS
5
VFS vfs(ctx);
6
​
7
// Create directory
8
if (!vfs.is_dir("dir_A")) {
9
  vfs.create_dir("dir_A");
10
  std::cout << "Created 'dir_A'\n";
11
} else {
12
  std::cout << "'dir_A' already exists\n";
13
}
14
​
15
// Creating an (empty) file
16
if (!vfs.is_file("dir_A/file_A")) {
17
  vfs.touch("dir_A/file_A");
18
  std::cout << "Created empty file 'dir_A/file_A'\n";
19
} else {
20
  std::cout << "'dir_A/file_A' already exists\n";
21
}
22
​
23
// Getting the file size
24
std::cout << "Size of file 'dir_A/file_A': " << vfs.file_size("dir_A/file_A")
25
          << "\n";
26
​
27
// Moving files (moving directories is similar)
28
vfs.move_file("dir_A/file_A", "dir_A/file_B");
29
​
30
// Deleting files and directories
31
vfs.remove_file("dir_A/file_B");
32
vfs.remove_dir("dir_A");
Copied!
1
import tiledb
2
​
3
ctx = tiledb.Ctx()
4
vfs = tiledb.VFS(ctx=ctx)
5
​
6
# Creating a directory
7
if not vfs.is_dir("dir_A"):
8
    vfs.create_dir("dir_A")
9
    print("Created 'dir_A'")
10
else:
11
    print ("'dir_A' already exists")
12
​
13
# Creating an (empty) file
14
if not vfs.is_file("dir_A/file_A"):
15
    vfs.touch("dir_A/file_A")
16
    print("Created empty file 'dir_A/file_A'")
17
else:
18
    print("'dir_A/file_A' already exists")
19
​
20
# Getting the file size
21
print("Size of file 'dir_A/file_A': ", vfs.file_size("dir_A/file_A"))
22
​
23
# Moving files (moving directories is similar)
24
vfs.move_file("dir_A/file_A", "dir_A/file_B")
25
​
26
vfs.remove_file("dir_A/file_B")
27
vfs.remove_dir("dir_A")
Copied!
1
library(tiledb)
2
​
3
ctx <- tiledb_get_context() 
4
vfs <- tiledb_vfs(ctx=ctx)
5
​
6
## Creating a directory
7
if (!tiledb_vfs_is_dir(vfs, "dir_A")) {
8
    tiledb_vfs_create_dir(vfs, "dir_A")
9
    cat("Created 'dir_A'\n")
10
} else {
11
    cat("'dir_A' already exists\n")
12
}
13
​
14
## Creating an (empty) file
15
if (!tiledb_vfs_is_file(vfs, "dir_A/file_A")) {
16
    tiledb_vfs_touch(vfs, "dir_A/file_A")
17
    cat("Created empty file 'dir_A/file_A'\n")
18
} else {
19
    cat("File 'dir_A/file_A' already existed\n")
20
}
21
​
22
## Getting the file size
23
cat("Size of file 'dir_A/file_A': ", 
24
    tiledb_vfs_file_size(vfs, "dir_A/file_A"), "\n")
25
​
26
## Moving files (moving directories is similar)
27
tiledb_vfs_move_file(vfs, "dir_A/file_A", "dir_A/file_B")
28
​
29
## Cleaning up
30
tiledb_vfs_remove_file(vfs, "dir_A/file_B")
31
tiledb_vfs_remove_dir(vfs, "dir_A")
Copied!
1
// Create TileDB context and VFS
2
try (Context ctx = new Context(); VFS vfs = new VFS(ctx)) {
3
    // Create directory
4
    if (!vfs.isDirectory("dir_A")) {
5
      vfs.createDirectory("dir_A");
6
      System.out.println( "Created 'dir_A'");
7
    } else {
8
      System.out.println("'dir_A' already exists");
9
    }
10
    
11
    // Creating an (empty) file
12
    if (!vfs.isFile("dir_A/file_A")) {
13
      vfs.createFile("dir_A/file_A");
14
      System.out.println("Created empty file 'dir_A/file_A'");
15
    } else {
16
      System.out.println("'dir_A/file_A' already exists");
17
    }
18
    
19
    // Getting the file size
20
    System.out.printf("Size of file 'dir_A/file_A': %d\n" , vfs.fileSize("dir_A/file_A"));
21
    
22
    // Moving files (moving directories is similar)
23
    vfs.moveFile("dir_A/file_A", "dir_A/file_B");
24
    
25
    // Deleting files and directories
26
    vfs.removeFile("dir_A/file_B");
27
    vfs.removeDirectory("dir_A");
28
}
Copied!
1
// Create TileDB context
2
config, _ := NewConfig()
3
ctx, _ := NewContext(config)
4
​
5
// Create TileDB VFS
6
vfs, _ := NewVFS(ctx, config)
7
    
8
// Create directory
9
vfs.CreateDir("dir_A")
10
isDir, _ := vfs.IsDir("dir_A")
11
if !is_dir {
12
  tiledb_vfs_create_dir(ctx, vfs, "dir_A")
13
  fmt.Println("Created 'dir_A'\n")
14
} else {
15
  fmt.Println("'dir_A' already exists\n")
16
}
17
​
18
// Creating an (empty) file
19
isFile, _ := vfs.IsFile("dir_A/file_A")
20
if !isFile {
21
  vfs.Touch("dir_A/file_A")
22
  fmt.Println("Created empty file 'dir_A/file_A'\n")
23
} else {
24
  fmt.Println("'dir_A/file_A' already exists\n")
25
}
26
​
27
// Getting the file size
28
fileSize, _ := vfs.FileSize("dir_A/file_A")
29
​
30
// Moving files (moving directories is similar)
31
vfs.MoveFile("dir_A/file_A", "dir_A/file_B")
32
​
33
// Deleting files and directories. Note that, in the case of directories,
34
// the function will delete all the contents of the directory (i.e., it
35
// works even for non-empty directories).
36
vfs.RemoveFile(ctx, vfs, "dir_A/file_B")
37
vfs.RemoveDir(ctx, vfs, "dir_A")
38
​
39
// Clean up
40
vfs.Free()
41
ctx.Free()
Copied!
TileDB allows you to create/delete S3 buckets via its VFS functionality,
C
C++