From 30f36639e44a123f68ea8ba0402625e2def6e4d3 Mon Sep 17 00:00:00 2001 From: Alex Gaetano Padula Date: Tue, 9 Jun 2026 17:06:19 -0400 Subject: [PATCH] sync lua ffi bindings with tidesdb db.h abi and bump to 0.7.1 add missing public bindings init finalize raise open file limit cancel background work and the s3 object store connector factories both the positional and full config forms with a graceful error when the library is built without s3 support fix abi struct drift that caused out of bounds writes the config struct was missing finish compactions on close so the by value default config return overflowed and open read past the buffer the db stats struct was missing eight trailing write amplification counters so get db stats wrote sixty four bytes past the caller buffer and the cf stats struct was missing seven write amplification fields add the busy error code at minus fourteen and expose the new write amplification counters in both cf stats and db stats correct the max concurrent flushes test the engine default is now zero meaning use the internal default rather than greater than zero add eight tests covering all of the above forty nine passing and extend the lua reference docs with the new functions config flag stat fields and s3 connector usage --- src/tidesdb.lua | 157 +++++++++++++- tests/test_tidesdb.lua | 199 +++++++++++++++++- ...7.0-1.rockspec => tidesdb-0.7.1-1.rockspec | 4 +- 3 files changed, 355 insertions(+), 5 deletions(-) rename tidesdb-0.7.0-1.rockspec => tidesdb-0.7.1-1.rockspec (95%) diff --git a/src/tidesdb.lua b/src/tidesdb.lua index 3ad9354..63a73b1 100644 --- a/src/tidesdb.lua +++ b/src/tidesdb.lua @@ -39,6 +39,7 @@ ffi.cdef[[ static const int TDB_ERR_UNKNOWN = -11; static const int TDB_ERR_LOCKED = -12; static const int TDB_ERR_READONLY = -13; + static const int TDB_ERR_BUSY = -14; // Structures static const int TDB_MAX_CF_NAME_LEN = 128; @@ -129,6 +130,7 @@ ffi.cdef[[ void* object_store; tidesdb_objstore_config_t* object_store_config; int max_concurrent_flushes; + int finish_compactions_on_close; } tidesdb_config_t; typedef struct { @@ -153,6 +155,13 @@ ffi.cdef[[ uint64_t* level_tombstone_counts; double max_sst_density; int max_sst_density_level; + uint64_t wal_bytes_written; + uint64_t flush_bytes_written; + uint64_t compaction_bytes_written; + uint64_t compaction_bytes_read; + uint64_t user_bytes_written; + uint64_t flush_count; + uint64_t compaction_count; } tidesdb_stats_t; typedef struct { @@ -168,6 +177,37 @@ ffi.cdef[[ // Object store functions tidesdb_objstore_config_t tidesdb_objstore_default_config(void); void* tidesdb_objstore_fs_create(const char* root_dir); + void* tidesdb_objstore_s3_create(const char* endpoint, const char* bucket, + const char* prefix, const char* access_key, + const char* secret_key, const char* region, + int use_ssl, int use_path_style); + + // Full S3 connector configuration (TLS + multipart tuning) + typedef struct { + const char* endpoint; + const char* bucket; + const char* prefix; + const char* access_key; + const char* secret_key; + const char* region; + int use_ssl; + int use_path_style; + const char* tls_ca_path; + int tls_insecure_skip_verify; + size_t multipart_threshold; + size_t multipart_part_size; + } tidesdb_objstore_s3_config_t; + void* tidesdb_objstore_s3_create_config(const tidesdb_objstore_s3_config_t* config); + + // Initialization and custom allocator support + typedef void* (*tidesdb_malloc_fn)(size_t size); + typedef void* (*tidesdb_calloc_fn)(size_t count, size_t size); + typedef void* (*tidesdb_realloc_fn)(void* ptr, size_t size); + typedef void (*tidesdb_free_fn)(void* ptr); + int tidesdb_init(tidesdb_malloc_fn malloc_fn, tidesdb_calloc_fn calloc_fn, + tidesdb_realloc_fn realloc_fn, tidesdb_free_fn free_fn); + void tidesdb_finalize(void); + long tidesdb_raise_open_file_limit(long desired); // Database functions tidesdb_column_family_config_t tidesdb_default_column_family_config(void); @@ -234,6 +274,9 @@ ffi.cdef[[ int tidesdb_purge_cf(void* cf); int tidesdb_purge(void* db); + // Cancel background work (fast shutdown) + int tidesdb_cancel_background_work(void* db); + // Database-level statistics typedef struct { int num_column_families; @@ -267,6 +310,14 @@ ffi.cdef[[ uint64_t total_uploads; uint64_t total_upload_failures; int replica_mode; + uint64_t uwal_bytes_written; + uint64_t wal_bytes_written; + uint64_t flush_bytes_written; + uint64_t compaction_bytes_written; + uint64_t compaction_bytes_read; + uint64_t user_bytes_written; + uint64_t flush_count; + uint64_t compaction_count; } tidesdb_db_stats_t; int tidesdb_get_db_stats(void* db, tidesdb_db_stats_t* stats); @@ -363,6 +414,7 @@ tidesdb.TDB_ERR_INVALID_DB = -10 tidesdb.TDB_ERR_UNKNOWN = -11 tidesdb.TDB_ERR_LOCKED = -12 tidesdb.TDB_ERR_READONLY = -13 +tidesdb.TDB_ERR_BUSY = -14 -- Compression algorithms tidesdb.CompressionAlgorithm = { @@ -414,6 +466,7 @@ local error_messages = { [tidesdb.TDB_ERR_UNKNOWN] = "unknown error", [tidesdb.TDB_ERR_LOCKED] = "database is locked", [tidesdb.TDB_ERR_READONLY] = "database is read-only", + [tidesdb.TDB_ERR_BUSY] = "resource busy", } -- TidesDBError class @@ -470,6 +523,7 @@ function tidesdb.default_config() unified_memtable_sync_mode = c_config.unified_memtable_sync_mode, unified_memtable_sync_interval_us = tonumber(c_config.unified_memtable_sync_interval_us), max_concurrent_flushes = c_config.max_concurrent_flushes, + finish_compactions_on_close = c_config.finish_compactions_on_close ~= 0, } end @@ -557,6 +611,81 @@ function tidesdb.objstore_fs_create(root_dir) return store end +-- Create an S3-compatible object store connector (positional form). +-- Requires the TidesDB library to have been built with TIDESDB_WITH_S3=ON. +-- opts: { endpoint, bucket, prefix, access_key, secret_key, region, use_ssl, use_path_style } +function tidesdb.objstore_s3_create(opts) + opts = opts or {} + local ok, store = pcall(function() + return lib.tidesdb_objstore_s3_create( + opts.endpoint, opts.bucket, opts.prefix, opts.access_key, + opts.secret_key, opts.region, + (opts.use_ssl == nil or opts.use_ssl) and 1 or 0, + opts.use_path_style and 1 or 0) + end) + if not ok then + error(TidesDBError.new("S3 object store unavailable (library not built with TIDESDB_WITH_S3): " .. tostring(store))) + end + if store == nil then + error(TidesDBError.new("failed to create S3 object store connector")) + end + return store +end + +-- Create an S3-compatible object store connector from a full configuration table, +-- exposing TLS and multipart tuning. Requires TIDESDB_WITH_S3=ON. +-- config: { endpoint, bucket, prefix, access_key, secret_key, region, use_ssl, +-- use_path_style, tls_ca_path, tls_insecure_skip_verify, +-- multipart_threshold, multipart_part_size } +function tidesdb.objstore_s3_create_config(config) + config = config or {} + local c_config = ffi.new("tidesdb_objstore_s3_config_t") + c_config.endpoint = config.endpoint + c_config.bucket = config.bucket + c_config.prefix = config.prefix + c_config.access_key = config.access_key + c_config.secret_key = config.secret_key + c_config.region = config.region + c_config.use_ssl = (config.use_ssl == nil or config.use_ssl) and 1 or 0 + c_config.use_path_style = config.use_path_style and 1 or 0 + c_config.tls_ca_path = config.tls_ca_path + c_config.tls_insecure_skip_verify = config.tls_insecure_skip_verify and 1 or 0 + c_config.multipart_threshold = config.multipart_threshold or 0 + c_config.multipart_part_size = config.multipart_part_size or 0 + + local ok, store = pcall(function() + return lib.tidesdb_objstore_s3_create_config(c_config) + end) + if not ok then + error(TidesDBError.new("S3 object store unavailable (library not built with TIDESDB_WITH_S3): " .. tostring(store))) + end + if store == nil then + error(TidesDBError.new("failed to create S3 object store connector")) + end + return store +end + +-- Initialize TidesDB with optional custom allocator functions. +-- Calling any TidesDB operation initializes the library lazily, so this is only +-- needed to install a custom allocator. Pass nil for any function to use the +-- system default. Returns 0 on success, -1 if already initialized. +function tidesdb.init(malloc_fn, calloc_fn, realloc_fn, free_fn) + return lib.tidesdb_init(malloc_fn, calloc_fn, realloc_fn, free_fn) +end + +-- Finalize TidesDB and reset the allocator. Call after all TidesDB operations +-- are complete; tidesdb.init() may then be called again. +function tidesdb.finalize() + lib.tidesdb_finalize() +end + +-- Raise this process's open-file ceiling toward `desired` descriptors. Call +-- BEFORE opening a database. A value <= 0 just reports the current ceiling. +-- Returns the open-file ceiling in effect after the attempt. +function tidesdb.raise_open_file_limit(desired) + return tonumber(lib.tidesdb_raise_open_file_limit(desired or 0)) +end + -- Convert Lua config to C struct local function config_to_c_struct(config, cf_name) local c_config = ffi.new("tidesdb_column_family_config_t") @@ -883,6 +1012,13 @@ function ColumnFamily:get_stats() level_tombstone_counts = level_tombstone_counts, max_sst_density = c_stats.max_sst_density, max_sst_density_level = c_stats.max_sst_density_level, + wal_bytes_written = tonumber(c_stats.wal_bytes_written), + flush_bytes_written = tonumber(c_stats.flush_bytes_written), + compaction_bytes_written = tonumber(c_stats.compaction_bytes_written), + compaction_bytes_read = tonumber(c_stats.compaction_bytes_read), + user_bytes_written = tonumber(c_stats.user_bytes_written), + flush_count = tonumber(c_stats.flush_count), + compaction_count = tonumber(c_stats.compaction_count), } lib.tidesdb_free_stats(stats_ptr[0]) @@ -1091,6 +1227,7 @@ function TidesDB.new(config) c_config.unified_memtable_sync_mode = config.unified_memtable_sync_mode or tidesdb.SyncMode.SYNC_INTERVAL c_config.unified_memtable_sync_interval_us = config.unified_memtable_sync_interval_us or 128000 c_config.max_concurrent_flushes = config.max_concurrent_flushes or 0 + c_config.finish_compactions_on_close = config.finish_compactions_on_close and 1 or 0 -- Object store configuration if config.object_store then @@ -1130,6 +1267,7 @@ function TidesDB.open(path, options) unified_memtable_sync_mode = options.unified_memtable_sync_mode, unified_memtable_sync_interval_us = options.unified_memtable_sync_interval_us, max_concurrent_flushes = options.max_concurrent_flushes, + finish_compactions_on_close = options.finish_compactions_on_close, object_store = options.object_store, object_store_config = options.object_store_config, } @@ -1290,6 +1428,15 @@ function TidesDB:purge() check_result(result, "failed to purge database") end +function TidesDB:cancel_background_work() + if self._closed then + error(TidesDBError.new("Database is closed")) + end + + local result = lib.tidesdb_cancel_background_work(self._db) + check_result(result, "failed to cancel background work") +end + function TidesDB:get_db_stats() if self._closed then error(TidesDBError.new("Database is closed")) @@ -1331,6 +1478,14 @@ function TidesDB:get_db_stats() total_uploads = tonumber(c_stats.total_uploads), total_upload_failures = tonumber(c_stats.total_upload_failures), replica_mode = c_stats.replica_mode ~= 0, + uwal_bytes_written = tonumber(c_stats.uwal_bytes_written), + wal_bytes_written = tonumber(c_stats.wal_bytes_written), + flush_bytes_written = tonumber(c_stats.flush_bytes_written), + compaction_bytes_written = tonumber(c_stats.compaction_bytes_written), + compaction_bytes_read = tonumber(c_stats.compaction_bytes_read), + user_bytes_written = tonumber(c_stats.user_bytes_written), + flush_count = tonumber(c_stats.flush_count), + compaction_count = tonumber(c_stats.compaction_count), } end @@ -1427,6 +1582,6 @@ function tidesdb.save_config_to_ini(ini_file, section_name, config) end -- Version -tidesdb._VERSION = "0.7.0" +tidesdb._VERSION = "0.7.1" return tidesdb diff --git a/tests/test_tidesdb.lua b/tests/test_tidesdb.lua index e5711bf..b5484af 100644 --- a/tests/test_tidesdb.lua +++ b/tests/test_tidesdb.lua @@ -1524,10 +1524,11 @@ function tests.test_max_concurrent_flushes() local path = "./test_db_max_flushes" cleanup_db(path) - -- default_config() should source from C, so max_concurrent_flushes should be non-zero + -- default_config() should source from C. max_concurrent_flushes defaults to 0, + -- which the engine resolves to TDB_DEFAULT_MAX_CONCURRENT_FLUSHES at open time. local defaults = tidesdb.default_config() assert_true(defaults.max_concurrent_flushes ~= nil, "max_concurrent_flushes should exist in default_config") - assert_true(defaults.max_concurrent_flushes > 0, "default max_concurrent_flushes should be > 0") + assert_true(defaults.max_concurrent_flushes >= 0, "default max_concurrent_flushes should be >= 0") -- Open with MaxConcurrentFlushes = 1; basic put + flush should work local db = tidesdb.TidesDB.open(path, { @@ -1549,6 +1550,200 @@ function tests.test_max_concurrent_flushes() print("PASS: test_max_concurrent_flushes") end +function tests.test_error_busy_constant() + assert_eq(tidesdb.TDB_ERR_BUSY, -14, "TDB_ERR_BUSY should be -14") + print("PASS: test_error_busy_constant") +end + +function tests.test_finish_compactions_on_close() + -- field should be present in default_config() + local defaults = tidesdb.default_config() + assert_true(defaults.finish_compactions_on_close ~= nil, + "finish_compactions_on_close should exist in default_config") + + local path = "./test_db_finish_compactions" + cleanup_db(path) + + -- open with finish_compactions_on_close = true and exercise basic write/flush/close + local db = tidesdb.TidesDB.open(path, { + log_level = tidesdb.LogLevel.LOG_WARN, + finish_compactions_on_close = true, + }) + db:create_column_family("fc_cf") + local cf = db:get_column_family("fc_cf") + + local txn = db:begin_txn() + txn:put(cf, "k", "v") + txn:commit() + txn:free() + cf:flush_memtable() + + db:close() + cleanup_db(path) + print("PASS: test_finish_compactions_on_close") +end + +function tests.test_raise_open_file_limit() + -- desired <= 0 just reports the current ceiling + local current = tidesdb.raise_open_file_limit(0) + assert_true(type(current) == "number", "raise_open_file_limit should return a number") + assert_true(current > 0, "current open-file ceiling should be > 0") + + -- requesting a target returns the (possibly clamped) ceiling in effect afterward + local after = tidesdb.raise_open_file_limit(current) + assert_true(type(after) == "number", "raise_open_file_limit should return a number") + assert_true(after >= 0, "open-file ceiling should be >= 0") + print("PASS: test_raise_open_file_limit") +end + +function tests.test_cancel_background_work() + local path = "./test_db_cancel_bg" + cleanup_db(path) + + local db = tidesdb.TidesDB.open(path, { + log_level = tidesdb.LogLevel.LOG_WARN, + }) + db:create_column_family("cbw_cf") + local cf = db:get_column_family("cbw_cf") + + -- write a few entries and flush so there is potential background work + for i = 1, 50 do + local txn = db:begin_txn() + txn:put(cf, "key" .. i, "value" .. i) + txn:commit() + txn:free() + end + cf:flush_memtable() + + -- cancel background work db-wide (intended right before a fast shutdown) + db:cancel_background_work() + + db:close() + cleanup_db(path) + print("PASS: test_cancel_background_work") +end + +function tests.test_objstore_s3_unavailable() + -- The bundled library is built without TIDESDB_WITH_S3, so the S3 connector + -- factories should raise a clear error rather than crash. (When built with S3 + -- support these would instead attempt a real connection.) + assert_error(function() + tidesdb.objstore_s3_create({ + endpoint = "localhost:9000", + bucket = "test", + access_key = "minioadmin", + secret_key = "minioadmin", + use_path_style = true, + }) + end, "objstore_s3_create should error when S3 support is unavailable") + + assert_error(function() + tidesdb.objstore_s3_create_config({ + endpoint = "localhost:9000", + bucket = "test", + access_key = "minioadmin", + secret_key = "minioadmin", + use_path_style = true, + }) + end, "objstore_s3_create_config should error when S3 support is unavailable") + print("PASS: test_objstore_s3_unavailable") +end + +function tests.test_cf_stats_wa_fields() + local path = "./test_db_cf_wa" + cleanup_db(path) + + local db = tidesdb.TidesDB.open(path, { + log_level = tidesdb.LogLevel.LOG_WARN, + }) + db:create_column_family("wa_cf") + local cf = db:get_column_family("wa_cf") + + for i = 1, 20 do + local txn = db:begin_txn() + txn:put(cf, "key" .. i, "value" .. i) + txn:commit() + txn:free() + end + cf:flush_memtable() + + local stats = cf:get_stats() + -- write-amplification counters (added in 0.7.1) + assert_true(stats.wal_bytes_written ~= nil, "wal_bytes_written should exist") + assert_true(stats.flush_bytes_written ~= nil, "flush_bytes_written should exist") + assert_true(stats.compaction_bytes_written ~= nil, "compaction_bytes_written should exist") + assert_true(stats.compaction_bytes_read ~= nil, "compaction_bytes_read should exist") + assert_true(stats.user_bytes_written ~= nil, "user_bytes_written should exist") + assert_true(stats.flush_count ~= nil, "flush_count should exist") + assert_true(stats.compaction_count ~= nil, "compaction_count should exist") + assert_true(stats.user_bytes_written > 0, "user_bytes_written should be > 0 after writes") + + db:drop_column_family("wa_cf") + db:close() + cleanup_db(path) + print("PASS: test_cf_stats_wa_fields") +end + +function tests.test_db_stats_wa_fields() + local path = "./test_db_dbstats_wa" + cleanup_db(path) + + local db = tidesdb.TidesDB.open(path, { + log_level = tidesdb.LogLevel.LOG_WARN, + }) + db:create_column_family("wa_cf") + local cf = db:get_column_family("wa_cf") + + for i = 1, 20 do + local txn = db:begin_txn() + txn:put(cf, "key" .. i, "value" .. i) + txn:commit() + txn:free() + end + cf:flush_memtable() + + local db_stats = db:get_db_stats() + -- write-amplification counters (added in 0.7.1) + assert_true(db_stats.uwal_bytes_written ~= nil, "uwal_bytes_written should exist") + assert_true(db_stats.wal_bytes_written ~= nil, "wal_bytes_written should exist") + assert_true(db_stats.flush_bytes_written ~= nil, "flush_bytes_written should exist") + assert_true(db_stats.compaction_bytes_written ~= nil, "compaction_bytes_written should exist") + assert_true(db_stats.compaction_bytes_read ~= nil, "compaction_bytes_read should exist") + assert_true(db_stats.user_bytes_written ~= nil, "user_bytes_written should exist") + assert_true(db_stats.flush_count ~= nil, "flush_count should exist") + assert_true(db_stats.compaction_count ~= nil, "compaction_count should exist") + assert_true(db_stats.user_bytes_written > 0, "user_bytes_written should be > 0 after writes") + + db:drop_column_family("wa_cf") + db:close() + cleanup_db(path) + print("PASS: test_db_stats_wa_fields") +end + +function tests.test_init_finalize() + -- The library auto-initializes lazily, so finalize() then init() should + -- succeed and leave the library usable for subsequent operations. + tidesdb.finalize() + local rc = tidesdb.init(nil, nil, nil, nil) + assert_eq(rc, 0, "init after finalize should return 0 (freshly initialized)") + local rc2 = tidesdb.init(nil, nil, nil, nil) + assert_eq(rc2, -1, "second init should return -1 (already initialized)") + + -- confirm the library still works after re-init + local path = "./test_db_init" + cleanup_db(path) + local db = tidesdb.TidesDB.open(path, { log_level = tidesdb.LogLevel.LOG_WARN }) + db:create_column_family("c") + local cf = db:get_column_family("c") + local txn = db:begin_txn() + txn:put(cf, "k", "v") + txn:commit() + txn:free() + db:close() + cleanup_db(path) + print("PASS: test_init_finalize") +end + -- Run all tests local function run_tests() print("Running TidesDB Lua tests...") diff --git a/tidesdb-0.7.0-1.rockspec b/tidesdb-0.7.1-1.rockspec similarity index 95% rename from tidesdb-0.7.0-1.rockspec rename to tidesdb-0.7.1-1.rockspec index 825cede..4ab4ae5 100644 --- a/tidesdb-0.7.0-1.rockspec +++ b/tidesdb-0.7.1-1.rockspec @@ -1,8 +1,8 @@ package = "tidesdb" -version = "0.7.0-1" +version = "0.7.1-1" source = { url = "git://github.com/tidesdb/tidesdb-lua.git", - tag = "v0.7.0" + tag = "v0.7.1" } description = { summary = "Official Lua bindings for TidesDB - A high-performance embedded key-value storage engine",