diff --git a/src/tidesdb.lua b/src/tidesdb.lua index 3ad9354..63a73b1 100644 --- a/src/tidesdb.lua +++ b/src/tidesdb.lua @@ -39,6 +39,7 @@ ffi.cdef[[ static const int TDB_ERR_UNKNOWN = -11; static const int TDB_ERR_LOCKED = -12; static const int TDB_ERR_READONLY = -13; + static const int TDB_ERR_BUSY = -14; // Structures static const int TDB_MAX_CF_NAME_LEN = 128; @@ -129,6 +130,7 @@ ffi.cdef[[ void* object_store; tidesdb_objstore_config_t* object_store_config; int max_concurrent_flushes; + int finish_compactions_on_close; } tidesdb_config_t; typedef struct { @@ -153,6 +155,13 @@ ffi.cdef[[ uint64_t* level_tombstone_counts; double max_sst_density; int max_sst_density_level; + uint64_t wal_bytes_written; + uint64_t flush_bytes_written; + uint64_t compaction_bytes_written; + uint64_t compaction_bytes_read; + uint64_t user_bytes_written; + uint64_t flush_count; + uint64_t compaction_count; } tidesdb_stats_t; typedef struct { @@ -168,6 +177,37 @@ ffi.cdef[[ // Object store functions tidesdb_objstore_config_t tidesdb_objstore_default_config(void); void* tidesdb_objstore_fs_create(const char* root_dir); + void* tidesdb_objstore_s3_create(const char* endpoint, const char* bucket, + const char* prefix, const char* access_key, + const char* secret_key, const char* region, + int use_ssl, int use_path_style); + + // Full S3 connector configuration (TLS + multipart tuning) + typedef struct { + const char* endpoint; + const char* bucket; + const char* prefix; + const char* access_key; + const char* secret_key; + const char* region; + int use_ssl; + int use_path_style; + const char* tls_ca_path; + int tls_insecure_skip_verify; + size_t multipart_threshold; + size_t multipart_part_size; + } tidesdb_objstore_s3_config_t; + void* tidesdb_objstore_s3_create_config(const tidesdb_objstore_s3_config_t* config); + + // Initialization and custom allocator support + typedef void* (*tidesdb_malloc_fn)(size_t size); + typedef void* (*tidesdb_calloc_fn)(size_t count, size_t size); + typedef void* (*tidesdb_realloc_fn)(void* ptr, size_t size); + typedef void (*tidesdb_free_fn)(void* ptr); + int tidesdb_init(tidesdb_malloc_fn malloc_fn, tidesdb_calloc_fn calloc_fn, + tidesdb_realloc_fn realloc_fn, tidesdb_free_fn free_fn); + void tidesdb_finalize(void); + long tidesdb_raise_open_file_limit(long desired); // Database functions tidesdb_column_family_config_t tidesdb_default_column_family_config(void); @@ -234,6 +274,9 @@ ffi.cdef[[ int tidesdb_purge_cf(void* cf); int tidesdb_purge(void* db); + // Cancel background work (fast shutdown) + int tidesdb_cancel_background_work(void* db); + // Database-level statistics typedef struct { int num_column_families; @@ -267,6 +310,14 @@ ffi.cdef[[ uint64_t total_uploads; uint64_t total_upload_failures; int replica_mode; + uint64_t uwal_bytes_written; + uint64_t wal_bytes_written; + uint64_t flush_bytes_written; + uint64_t compaction_bytes_written; + uint64_t compaction_bytes_read; + uint64_t user_bytes_written; + uint64_t flush_count; + uint64_t compaction_count; } tidesdb_db_stats_t; int tidesdb_get_db_stats(void* db, tidesdb_db_stats_t* stats); @@ -363,6 +414,7 @@ tidesdb.TDB_ERR_INVALID_DB = -10 tidesdb.TDB_ERR_UNKNOWN = -11 tidesdb.TDB_ERR_LOCKED = -12 tidesdb.TDB_ERR_READONLY = -13 +tidesdb.TDB_ERR_BUSY = -14 -- Compression algorithms tidesdb.CompressionAlgorithm = { @@ -414,6 +466,7 @@ local error_messages = { [tidesdb.TDB_ERR_UNKNOWN] = "unknown error", [tidesdb.TDB_ERR_LOCKED] = "database is locked", [tidesdb.TDB_ERR_READONLY] = "database is read-only", + [tidesdb.TDB_ERR_BUSY] = "resource busy", } -- TidesDBError class @@ -470,6 +523,7 @@ function tidesdb.default_config() unified_memtable_sync_mode = c_config.unified_memtable_sync_mode, unified_memtable_sync_interval_us = tonumber(c_config.unified_memtable_sync_interval_us), max_concurrent_flushes = c_config.max_concurrent_flushes, + finish_compactions_on_close = c_config.finish_compactions_on_close ~= 0, } end @@ -557,6 +611,81 @@ function tidesdb.objstore_fs_create(root_dir) return store end +-- Create an S3-compatible object store connector (positional form). +-- Requires the TidesDB library to have been built with TIDESDB_WITH_S3=ON. +-- opts: { endpoint, bucket, prefix, access_key, secret_key, region, use_ssl, use_path_style } +function tidesdb.objstore_s3_create(opts) + opts = opts or {} + local ok, store = pcall(function() + return lib.tidesdb_objstore_s3_create( + opts.endpoint, opts.bucket, opts.prefix, opts.access_key, + opts.secret_key, opts.region, + (opts.use_ssl == nil or opts.use_ssl) and 1 or 0, + opts.use_path_style and 1 or 0) + end) + if not ok then + error(TidesDBError.new("S3 object store unavailable (library not built with TIDESDB_WITH_S3): " .. tostring(store))) + end + if store == nil then + error(TidesDBError.new("failed to create S3 object store connector")) + end + return store +end + +-- Create an S3-compatible object store connector from a full configuration table, +-- exposing TLS and multipart tuning. Requires TIDESDB_WITH_S3=ON. +-- config: { endpoint, bucket, prefix, access_key, secret_key, region, use_ssl, +-- use_path_style, tls_ca_path, tls_insecure_skip_verify, +-- multipart_threshold, multipart_part_size } +function tidesdb.objstore_s3_create_config(config) + config = config or {} + local c_config = ffi.new("tidesdb_objstore_s3_config_t") + c_config.endpoint = config.endpoint + c_config.bucket = config.bucket + c_config.prefix = config.prefix + c_config.access_key = config.access_key + c_config.secret_key = config.secret_key + c_config.region = config.region + c_config.use_ssl = (config.use_ssl == nil or config.use_ssl) and 1 or 0 + c_config.use_path_style = config.use_path_style and 1 or 0 + c_config.tls_ca_path = config.tls_ca_path + c_config.tls_insecure_skip_verify = config.tls_insecure_skip_verify and 1 or 0 + c_config.multipart_threshold = config.multipart_threshold or 0 + c_config.multipart_part_size = config.multipart_part_size or 0 + + local ok, store = pcall(function() + return lib.tidesdb_objstore_s3_create_config(c_config) + end) + if not ok then + error(TidesDBError.new("S3 object store unavailable (library not built with TIDESDB_WITH_S3): " .. tostring(store))) + end + if store == nil then + error(TidesDBError.new("failed to create S3 object store connector")) + end + return store +end + +-- Initialize TidesDB with optional custom allocator functions. +-- Calling any TidesDB operation initializes the library lazily, so this is only +-- needed to install a custom allocator. Pass nil for any function to use the +-- system default. Returns 0 on success, -1 if already initialized. +function tidesdb.init(malloc_fn, calloc_fn, realloc_fn, free_fn) + return lib.tidesdb_init(malloc_fn, calloc_fn, realloc_fn, free_fn) +end + +-- Finalize TidesDB and reset the allocator. Call after all TidesDB operations +-- are complete; tidesdb.init() may then be called again. +function tidesdb.finalize() + lib.tidesdb_finalize() +end + +-- Raise this process's open-file ceiling toward `desired` descriptors. Call +-- BEFORE opening a database. A value <= 0 just reports the current ceiling. +-- Returns the open-file ceiling in effect after the attempt. +function tidesdb.raise_open_file_limit(desired) + return tonumber(lib.tidesdb_raise_open_file_limit(desired or 0)) +end + -- Convert Lua config to C struct local function config_to_c_struct(config, cf_name) local c_config = ffi.new("tidesdb_column_family_config_t") @@ -883,6 +1012,13 @@ function ColumnFamily:get_stats() level_tombstone_counts = level_tombstone_counts, max_sst_density = c_stats.max_sst_density, max_sst_density_level = c_stats.max_sst_density_level, + wal_bytes_written = tonumber(c_stats.wal_bytes_written), + flush_bytes_written = tonumber(c_stats.flush_bytes_written), + compaction_bytes_written = tonumber(c_stats.compaction_bytes_written), + compaction_bytes_read = tonumber(c_stats.compaction_bytes_read), + user_bytes_written = tonumber(c_stats.user_bytes_written), + flush_count = tonumber(c_stats.flush_count), + compaction_count = tonumber(c_stats.compaction_count), } lib.tidesdb_free_stats(stats_ptr[0]) @@ -1091,6 +1227,7 @@ function TidesDB.new(config) c_config.unified_memtable_sync_mode = config.unified_memtable_sync_mode or tidesdb.SyncMode.SYNC_INTERVAL c_config.unified_memtable_sync_interval_us = config.unified_memtable_sync_interval_us or 128000 c_config.max_concurrent_flushes = config.max_concurrent_flushes or 0 + c_config.finish_compactions_on_close = config.finish_compactions_on_close and 1 or 0 -- Object store configuration if config.object_store then @@ -1130,6 +1267,7 @@ function TidesDB.open(path, options) unified_memtable_sync_mode = options.unified_memtable_sync_mode, unified_memtable_sync_interval_us = options.unified_memtable_sync_interval_us, max_concurrent_flushes = options.max_concurrent_flushes, + finish_compactions_on_close = options.finish_compactions_on_close, object_store = options.object_store, object_store_config = options.object_store_config, } @@ -1290,6 +1428,15 @@ function TidesDB:purge() check_result(result, "failed to purge database") end +function TidesDB:cancel_background_work() + if self._closed then + error(TidesDBError.new("Database is closed")) + end + + local result = lib.tidesdb_cancel_background_work(self._db) + check_result(result, "failed to cancel background work") +end + function TidesDB:get_db_stats() if self._closed then error(TidesDBError.new("Database is closed")) @@ -1331,6 +1478,14 @@ function TidesDB:get_db_stats() total_uploads = tonumber(c_stats.total_uploads), total_upload_failures = tonumber(c_stats.total_upload_failures), replica_mode = c_stats.replica_mode ~= 0, + uwal_bytes_written = tonumber(c_stats.uwal_bytes_written), + wal_bytes_written = tonumber(c_stats.wal_bytes_written), + flush_bytes_written = tonumber(c_stats.flush_bytes_written), + compaction_bytes_written = tonumber(c_stats.compaction_bytes_written), + compaction_bytes_read = tonumber(c_stats.compaction_bytes_read), + user_bytes_written = tonumber(c_stats.user_bytes_written), + flush_count = tonumber(c_stats.flush_count), + compaction_count = tonumber(c_stats.compaction_count), } end @@ -1427,6 +1582,6 @@ function tidesdb.save_config_to_ini(ini_file, section_name, config) end -- Version -tidesdb._VERSION = "0.7.0" +tidesdb._VERSION = "0.7.1" return tidesdb diff --git a/tests/test_tidesdb.lua b/tests/test_tidesdb.lua index e5711bf..b5484af 100644 --- a/tests/test_tidesdb.lua +++ b/tests/test_tidesdb.lua @@ -1524,10 +1524,11 @@ function tests.test_max_concurrent_flushes() local path = "./test_db_max_flushes" cleanup_db(path) - -- default_config() should source from C, so max_concurrent_flushes should be non-zero + -- default_config() should source from C. max_concurrent_flushes defaults to 0, + -- which the engine resolves to TDB_DEFAULT_MAX_CONCURRENT_FLUSHES at open time. local defaults = tidesdb.default_config() assert_true(defaults.max_concurrent_flushes ~= nil, "max_concurrent_flushes should exist in default_config") - assert_true(defaults.max_concurrent_flushes > 0, "default max_concurrent_flushes should be > 0") + assert_true(defaults.max_concurrent_flushes >= 0, "default max_concurrent_flushes should be >= 0") -- Open with MaxConcurrentFlushes = 1; basic put + flush should work local db = tidesdb.TidesDB.open(path, { @@ -1549,6 +1550,200 @@ function tests.test_max_concurrent_flushes() print("PASS: test_max_concurrent_flushes") end +function tests.test_error_busy_constant() + assert_eq(tidesdb.TDB_ERR_BUSY, -14, "TDB_ERR_BUSY should be -14") + print("PASS: test_error_busy_constant") +end + +function tests.test_finish_compactions_on_close() + -- field should be present in default_config() + local defaults = tidesdb.default_config() + assert_true(defaults.finish_compactions_on_close ~= nil, + "finish_compactions_on_close should exist in default_config") + + local path = "./test_db_finish_compactions" + cleanup_db(path) + + -- open with finish_compactions_on_close = true and exercise basic write/flush/close + local db = tidesdb.TidesDB.open(path, { + log_level = tidesdb.LogLevel.LOG_WARN, + finish_compactions_on_close = true, + }) + db:create_column_family("fc_cf") + local cf = db:get_column_family("fc_cf") + + local txn = db:begin_txn() + txn:put(cf, "k", "v") + txn:commit() + txn:free() + cf:flush_memtable() + + db:close() + cleanup_db(path) + print("PASS: test_finish_compactions_on_close") +end + +function tests.test_raise_open_file_limit() + -- desired <= 0 just reports the current ceiling + local current = tidesdb.raise_open_file_limit(0) + assert_true(type(current) == "number", "raise_open_file_limit should return a number") + assert_true(current > 0, "current open-file ceiling should be > 0") + + -- requesting a target returns the (possibly clamped) ceiling in effect afterward + local after = tidesdb.raise_open_file_limit(current) + assert_true(type(after) == "number", "raise_open_file_limit should return a number") + assert_true(after >= 0, "open-file ceiling should be >= 0") + print("PASS: test_raise_open_file_limit") +end + +function tests.test_cancel_background_work() + local path = "./test_db_cancel_bg" + cleanup_db(path) + + local db = tidesdb.TidesDB.open(path, { + log_level = tidesdb.LogLevel.LOG_WARN, + }) + db:create_column_family("cbw_cf") + local cf = db:get_column_family("cbw_cf") + + -- write a few entries and flush so there is potential background work + for i = 1, 50 do + local txn = db:begin_txn() + txn:put(cf, "key" .. i, "value" .. i) + txn:commit() + txn:free() + end + cf:flush_memtable() + + -- cancel background work db-wide (intended right before a fast shutdown) + db:cancel_background_work() + + db:close() + cleanup_db(path) + print("PASS: test_cancel_background_work") +end + +function tests.test_objstore_s3_unavailable() + -- The bundled library is built without TIDESDB_WITH_S3, so the S3 connector + -- factories should raise a clear error rather than crash. (When built with S3 + -- support these would instead attempt a real connection.) + assert_error(function() + tidesdb.objstore_s3_create({ + endpoint = "localhost:9000", + bucket = "test", + access_key = "minioadmin", + secret_key = "minioadmin", + use_path_style = true, + }) + end, "objstore_s3_create should error when S3 support is unavailable") + + assert_error(function() + tidesdb.objstore_s3_create_config({ + endpoint = "localhost:9000", + bucket = "test", + access_key = "minioadmin", + secret_key = "minioadmin", + use_path_style = true, + }) + end, "objstore_s3_create_config should error when S3 support is unavailable") + print("PASS: test_objstore_s3_unavailable") +end + +function tests.test_cf_stats_wa_fields() + local path = "./test_db_cf_wa" + cleanup_db(path) + + local db = tidesdb.TidesDB.open(path, { + log_level = tidesdb.LogLevel.LOG_WARN, + }) + db:create_column_family("wa_cf") + local cf = db:get_column_family("wa_cf") + + for i = 1, 20 do + local txn = db:begin_txn() + txn:put(cf, "key" .. i, "value" .. i) + txn:commit() + txn:free() + end + cf:flush_memtable() + + local stats = cf:get_stats() + -- write-amplification counters (added in 0.7.1) + assert_true(stats.wal_bytes_written ~= nil, "wal_bytes_written should exist") + assert_true(stats.flush_bytes_written ~= nil, "flush_bytes_written should exist") + assert_true(stats.compaction_bytes_written ~= nil, "compaction_bytes_written should exist") + assert_true(stats.compaction_bytes_read ~= nil, "compaction_bytes_read should exist") + assert_true(stats.user_bytes_written ~= nil, "user_bytes_written should exist") + assert_true(stats.flush_count ~= nil, "flush_count should exist") + assert_true(stats.compaction_count ~= nil, "compaction_count should exist") + assert_true(stats.user_bytes_written > 0, "user_bytes_written should be > 0 after writes") + + db:drop_column_family("wa_cf") + db:close() + cleanup_db(path) + print("PASS: test_cf_stats_wa_fields") +end + +function tests.test_db_stats_wa_fields() + local path = "./test_db_dbstats_wa" + cleanup_db(path) + + local db = tidesdb.TidesDB.open(path, { + log_level = tidesdb.LogLevel.LOG_WARN, + }) + db:create_column_family("wa_cf") + local cf = db:get_column_family("wa_cf") + + for i = 1, 20 do + local txn = db:begin_txn() + txn:put(cf, "key" .. i, "value" .. i) + txn:commit() + txn:free() + end + cf:flush_memtable() + + local db_stats = db:get_db_stats() + -- write-amplification counters (added in 0.7.1) + assert_true(db_stats.uwal_bytes_written ~= nil, "uwal_bytes_written should exist") + assert_true(db_stats.wal_bytes_written ~= nil, "wal_bytes_written should exist") + assert_true(db_stats.flush_bytes_written ~= nil, "flush_bytes_written should exist") + assert_true(db_stats.compaction_bytes_written ~= nil, "compaction_bytes_written should exist") + assert_true(db_stats.compaction_bytes_read ~= nil, "compaction_bytes_read should exist") + assert_true(db_stats.user_bytes_written ~= nil, "user_bytes_written should exist") + assert_true(db_stats.flush_count ~= nil, "flush_count should exist") + assert_true(db_stats.compaction_count ~= nil, "compaction_count should exist") + assert_true(db_stats.user_bytes_written > 0, "user_bytes_written should be > 0 after writes") + + db:drop_column_family("wa_cf") + db:close() + cleanup_db(path) + print("PASS: test_db_stats_wa_fields") +end + +function tests.test_init_finalize() + -- The library auto-initializes lazily, so finalize() then init() should + -- succeed and leave the library usable for subsequent operations. + tidesdb.finalize() + local rc = tidesdb.init(nil, nil, nil, nil) + assert_eq(rc, 0, "init after finalize should return 0 (freshly initialized)") + local rc2 = tidesdb.init(nil, nil, nil, nil) + assert_eq(rc2, -1, "second init should return -1 (already initialized)") + + -- confirm the library still works after re-init + local path = "./test_db_init" + cleanup_db(path) + local db = tidesdb.TidesDB.open(path, { log_level = tidesdb.LogLevel.LOG_WARN }) + db:create_column_family("c") + local cf = db:get_column_family("c") + local txn = db:begin_txn() + txn:put(cf, "k", "v") + txn:commit() + txn:free() + db:close() + cleanup_db(path) + print("PASS: test_init_finalize") +end + -- Run all tests local function run_tests() print("Running TidesDB Lua tests...") diff --git a/tidesdb-0.7.0-1.rockspec b/tidesdb-0.7.1-1.rockspec similarity index 95% rename from tidesdb-0.7.0-1.rockspec rename to tidesdb-0.7.1-1.rockspec index 825cede..4ab4ae5 100644 --- a/tidesdb-0.7.0-1.rockspec +++ b/tidesdb-0.7.1-1.rockspec @@ -1,8 +1,8 @@ package = "tidesdb" -version = "0.7.0-1" +version = "0.7.1-1" source = { url = "git://github.com/tidesdb/tidesdb-lua.git", - tag = "v0.7.0" + tag = "v0.7.1" } description = { summary = "Official Lua bindings for TidesDB - A high-performance embedded key-value storage engine",