From 2180e8ec7b2f3bcc3cbce21a99256ae9f9ce181e Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 26 Aug 2026 19:25:45 +0000 Subject: [PATCH 1/2] Add Cloud Agent environment with OCaml 5.5 Define a repository-managed environment for datascript-ocaml: - Base image installs opam, OCaml 5.5.0, libsqlite3-dev, and Node.js 24 - Install script runs opam deps and dune build after checkout Co-authored-by: Tienson Qin --- .cursor/Dockerfile | 34 ++++++++++++++++++++++++++++++++++ .cursor/cloud-agent-install.sh | 11 +++++++++++ .cursor/environment.json | 9 +++++++++ 3 files changed, 54 insertions(+) create mode 100644 .cursor/Dockerfile create mode 100755 .cursor/cloud-agent-install.sh create mode 100644 .cursor/environment.json diff --git a/.cursor/Dockerfile b/.cursor/Dockerfile new file mode 100644 index 0000000..fc698dc --- /dev/null +++ b/.cursor/Dockerfile @@ -0,0 +1,34 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive +ENV OPAMYES=1 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + git \ + build-essential \ + pkg-config \ + libsqlite3-dev \ + && rm -rf /var/lib/apt/lists/* + +# Node.js 24 for js_of_ocaml smoke tests and cross-runtime helpers. +RUN curl -fsSL https://deb.nodesource.com/setup_24.x | bash - \ + && apt-get install -y --no-install-recommends nodejs \ + && rm -rf /var/lib/apt/lists/* + +RUN useradd -m -s /bin/bash ubuntu 2>/dev/null || true + +USER ubuntu +WORKDIR /home/ubuntu + +RUN curl -fsSL https://raw.githubusercontent.com/ocaml/opam/master/shell/install.sh \ + | bash -s -- --disable-sandboxing \ + && opam init --disable-sandboxing -y \ + && opam switch create 5.5 ocaml-base-compiler.5.5.0 -y \ + && eval "$(opam env --switch=5.5)" \ + && opam update -a + +RUN echo 'test -r ~/.opam/opam-init/init.sh && . ~/.opam/opam-init/init.sh > /dev/null 2> /dev/null || true' >> ~/.bashrc \ + && echo 'eval $(opam env --switch=5.5 2>/dev/null)' >> ~/.bashrc diff --git a/.cursor/cloud-agent-install.sh b/.cursor/cloud-agent-install.sh new file mode 100755 index 0000000..798992e --- /dev/null +++ b/.cursor/cloud-agent-install.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(git rev-parse --show-toplevel)" +cd "$repo_root" + +eval "$(opam env --switch=5.5)" + +export OPAMYES=1 +opam install . --deps-only --with-test -y +dune build diff --git a/.cursor/environment.json b/.cursor/environment.json new file mode 100644 index 0000000..491f294 --- /dev/null +++ b/.cursor/environment.json @@ -0,0 +1,9 @@ +{ + "name": "DataScript OCaml (OCaml 5.5)", + "user": "ubuntu", + "build": { + "dockerfile": "Dockerfile", + "context": ".." + }, + "install": ".cursor/cloud-agent-install.sh" +} From 2de5d747aaf9f2b6b98275991e9c69a01d85686b Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 26 Aug 2026 19:30:47 +0000 Subject: [PATCH 2/2] Add LMDB storage backend and SQLite comparison benchmark Introduce datascript_lmdb as an alternative KV backend for persistent PSS storage, reusing the Transit payload codec from the SQLite package. Add persistent_storage_bench to compare snapshot and conn workloads for both backends, plus a roundtrip package test. Co-authored-by: Tienson Qin --- .cursor/Dockerfile | 1 + .github/workflows/ci.yml | 2 +- bench/dune | 6 + bench/persistent_storage_bench.ml | 263 ++++++++++++++++++++++++++++++ datascript-ocaml-native.opam | 1 + lmdb/datascript_lmdb.ml | 89 ++++++++++ lmdb/dune | 5 + test/dune | 5 + test/test_lmdb_package.ml | 68 ++++++++ 9 files changed, 439 insertions(+), 1 deletion(-) create mode 100644 bench/persistent_storage_bench.ml create mode 100644 lmdb/datascript_lmdb.ml create mode 100644 lmdb/dune create mode 100644 test/test_lmdb_package.ml diff --git a/.cursor/Dockerfile b/.cursor/Dockerfile index fc698dc..4374c88 100644 --- a/.cursor/Dockerfile +++ b/.cursor/Dockerfile @@ -11,6 +11,7 @@ RUN apt-get update \ build-essential \ pkg-config \ libsqlite3-dev \ + liblmdb-dev \ && rm -rf /var/lib/apt/lists/* # Node.js 24 for js_of_ocaml smoke tests and cross-runtime helpers. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f883c22..cbfbd2b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,7 +48,7 @@ jobs: lein: latest - name: Install system dependencies - run: sudo apt-get update && sudo apt-get install -y libsqlite3-dev pkg-config + run: sudo apt-get update && sudo apt-get install -y libsqlite3-dev liblmdb-dev pkg-config - name: Install OCaml dependencies run: opam install . --deps-only --with-test -y diff --git a/bench/dune b/bench/dune index 67c4a83..750568a 100644 --- a/bench/dune +++ b/bench/dune @@ -33,6 +33,12 @@ (modes exe) (libraries datascript-ocaml-native datascript_sqlite unix sqlite3)) +(executable + (name persistent_storage_bench) + (modules persistent_storage_bench) + (modes exe) + (libraries datascript-ocaml-native datascript_sqlite datascript_lmdb unix sqlite3)) + (executable (name outliner_insert_ocaml) (modules outliner_insert_ocaml) diff --git a/bench/persistent_storage_bench.ml b/bench/persistent_storage_bench.ml new file mode 100644 index 0000000..1dd101d --- /dev/null +++ b/bench/persistent_storage_bench.ml @@ -0,0 +1,263 @@ +open Datascript + +type timing = { label : string; elapsed_ms : float } + +let now_ms () = Unix.gettimeofday () *. 1000. + +let time label f = + let start = now_ms () in + let result = f () in + ({ label; elapsed_ms = now_ms () -. start }, result) + +let print_timing prefix { label; elapsed_ms } = + Printf.printf "%s%s\t%.2f\n%!" prefix label elapsed_ms + +let indexed = + { + cardinality = One; + unique = None; + indexed = true; + is_component = false; + no_history = false; + doc = None; + value_type = None; + tuple_attrs = None; + tuple_types = None; + } + +let unique_identity = { indexed with unique = Some Identity } + +let schema = + [ + ("block/id", unique_identity); + ("block/journal-day", indexed); + ("block/content", indexed); + ("block/order", indexed); + ("block/collapsed", indexed); + ] + +let block_tx count = + List.init count (fun index -> + let i = index + 1 in + Entity + { + db_id = Some (Temp_id (Printf.sprintf "block-%05d" i)); + attrs = + [ + ("block/id", One_value (String (Printf.sprintf "block-%05d" i))); + ("block/journal-day", One_value (String "2026-06-27")); + ("block/content", One_value (String (Printf.sprintf "Block %05d" i))); + ("block/order", One_value (Float (Float.of_int i))); + ("block/collapsed", One_value (Bool false)); + ]; + }) + +let add_block_tx id order = + [ + Entity + { + db_id = Some (Temp_id id); + attrs = + [ + ("block/id", One_value (String id)); + ("block/journal-day", One_value (String "2026-06-27")); + ("block/content", One_value (String id)); + ("block/order", One_value (Float order)); + ("block/collapsed", One_value (Bool false)); + ]; + }; + ] + +let update_content_tx id content = + [ Add (Lookup_ref ("block/id", String id), "block/content", String content) ] + +let seq_length seq = Seq.fold_left (fun count _ -> count + 1) 0 seq + +let file_size path = + if Sys.file_exists path then (Unix.stat path).st_size else 0 + +let remove_if_exists path = if Sys.file_exists path then Sys.remove path + +let row_count storage = List.length (storage_addresses storage) + +module type BACKEND = sig + val name : string + val extension : string + type session + val open_session : string -> session + val close_session : session -> unit + val storage : session -> storage + val cleanup : string -> unit +end + +module Sqlite_backend : BACKEND = struct + type session = Datascript_sqlite.session + + let name = "sqlite" + let extension = "sqlite3" + let open_session = Datascript_sqlite.open_session + let close_session = Datascript_sqlite.close + let storage = Datascript_sqlite.storage + let cleanup _path = () +end + +module Lmdb_backend : BACKEND = struct + type session = Datascript_lmdb.session + + let name = "lmdb" + let extension = "lmdb" + let open_session = Datascript_lmdb.open_session + let close_session = Datascript_lmdb.close + let storage = Datascript_lmdb.storage + + let cleanup path = + let lock = path ^ "-lock" in + if Sys.file_exists lock then Sys.remove lock +end + +let run_backend (module B : BACKEND) size tx = + let prefix = B.name ^ "-" in + let db_path = + Filename.concat + (Filename.get_temp_dir_name ()) + (Printf.sprintf "datascript-persistent-%s-%d.%s" B.name size B.extension) + in + remove_if_exists db_path; + let session = B.open_session db_path in + Fun.protect + ~finally:(fun () -> + B.close_session session; + remove_if_exists db_path; + B.cleanup db_path) + (fun () -> + let storage = B.storage session in + let persistent_build, persistent_db = + time "snapshot-build-and-store" (fun () -> + let db = db_with tx (empty_db ~schema ~storage ()) in + store db; + db) + in + print_timing prefix persistent_build; + Printf.printf "%ssnapshot-build-datoms\t%d\n%!" prefix + (seq_length (datoms persistent_db Eavt ())); + Printf.printf "%ssnapshot-kvs-rows-after-build\t%d\n%!" prefix (row_count storage); + Printf.printf "%ssnapshot-file-size-after-build\t%d\n%!" prefix (file_size db_path); + let restore_timing, restored_db = + time "snapshot-restore" (fun () -> + match restore storage with + | Some db -> db + | None -> failwith (B.name ^ " persistent db should restore")) + in + print_timing prefix restore_timing; + let persistent_add, restored_db = + time "snapshot-add-one-and-store-after-restore" (fun () -> + let db = + db_with + (add_block_tx "persistent-new" (Float.of_int (size + 1))) + restored_db + in + store db; + db) + in + print_timing prefix persistent_add; + Printf.printf "%ssnapshot-kvs-rows-after-add\t%d\n%!" prefix (row_count storage); + Printf.printf "%ssnapshot-file-size-after-add\t%d\n%!" prefix (file_size db_path); + let persistent_update, restored_db = + time "snapshot-update-one-and-store-after-add" (fun () -> + let db = db_with (update_content_tx "block-00001" "Edited") restored_db in + store db; + db) + in + print_timing prefix persistent_update; + Printf.printf "%ssnapshot-kvs-rows-after-update\t%d\n%!" prefix (row_count storage); + Printf.printf "%ssnapshot-file-size-after-update\t%d\n%!" prefix (file_size db_path); + Printf.printf "%ssnapshot-datoms\t%d\n%!" prefix + (seq_length (datoms restored_db Eavt ())); + let conn_db_path = + Filename.concat + (Filename.get_temp_dir_name ()) + (Printf.sprintf "datascript-persistent-%s-conn-%d.%s" B.name size B.extension) + in + remove_if_exists conn_db_path; + let session = B.open_session conn_db_path in + Fun.protect + ~finally:(fun () -> + B.close_session session; + remove_if_exists conn_db_path; + B.cleanup conn_db_path) + (fun () -> + let storage = B.storage session in + let conn_build, conn = + time "conn-build" (fun () -> + let conn = create_conn ~schema ~storage () in + ignore (transact_conn conn tx); + conn) + in + print_timing prefix conn_build; + Printf.printf "%sconn-build-datoms\t%d\n%!" prefix + (seq_length (datoms (db conn) Eavt ())); + Printf.printf "%sconn-kvs-rows-after-build\t%d\n%!" prefix (row_count storage); + Printf.printf "%sconn-file-size-after-build\t%d\n%!" prefix (file_size conn_db_path); + let conn_restore, conn = + time "conn-restore" (fun () -> + match restore_conn storage with + | Some conn -> conn + | None -> failwith (B.name ^ " persistent conn should restore")) + in + print_timing prefix conn_restore; + let conn_add, _report = + time "conn-add-one-after-restore" (fun () -> + transact_conn conn (add_block_tx "conn-new" (Float.of_int (size + 1)))) + in + print_timing prefix conn_add; + Printf.printf "%sconn-kvs-rows-after-add\t%d\n%!" prefix (row_count storage); + Printf.printf "%sconn-file-size-after-add\t%d\n%!" prefix (file_size conn_db_path); + let conn_update, _report = + time "conn-update-one-after-add" (fun () -> + transact_conn conn (update_content_tx "block-00001" "Edited")) + in + print_timing prefix conn_update; + Printf.printf "%sconn-kvs-rows-after-update\t%d\n%!" prefix (row_count storage); + Printf.printf "%sconn-file-size-after-update\t%d\n%!" prefix (file_size conn_db_path); + Printf.printf "%sconn-datoms\t%d\n%!" prefix (seq_length (datoms (db conn) Eavt ())))) + +let run_size size = + Printf.printf "size\t%d\n%!" size; + let tx = block_tx size in + let memory_build, memory_db = + time "memory-build" (fun () -> db_with tx (empty_db ~schema ())) + in + print_timing "" memory_build; + let memory_add, memory_db = + time "memory-add-one" (fun () -> + db_with (add_block_tx "memory-new" (Float.of_int (size + 1))) memory_db) + in + print_timing "" memory_add; + let _memory_update, memory_db = + time "memory-update-one" (fun () -> + db_with (update_content_tx "block-00001" "Edited") memory_db) + in + print_timing "" _memory_update; + Printf.printf "memory-datoms\t%d\n%!" (seq_length (datoms memory_db Eavt ())); + run_backend (module Sqlite_backend) size tx; + run_backend (module Lmdb_backend) size tx + +let parse_sizes () = + let rec loop sizes = function + | [] -> List.rev sizes + | "--size" :: size :: rest -> loop (int_of_string size :: sizes) rest + | "--sizes" :: value :: rest -> + let parsed = + value + |> String.split_on_char ',' + |> List.filter (fun value -> String.length value > 0) + |> List.map int_of_string + in + loop (List.rev_append parsed sizes) rest + | arg :: _ -> invalid_arg ("unknown benchmark argument: " ^ arg) + in + match loop [] (Sys.argv |> Array.to_list |> List.tl) with + | [] -> [ 100; 1000; 5000 ] + | sizes -> sizes + +let () = List.iter run_size (parse_sizes ()) diff --git a/datascript-ocaml-native.opam b/datascript-ocaml-native.opam index 9fb78c0..889fc22 100644 --- a/datascript-ocaml-native.opam +++ b/datascript-ocaml-native.opam @@ -10,6 +10,7 @@ depends: [ "datascript_ocaml" {= version} "persistent_sorted_set_ocaml" {= "dev"} "sqlite3" + "lmdb" "melange-transit-native" {= "0.1.0"} "yojson" ] diff --git a/lmdb/datascript_lmdb.ml b/lmdb/datascript_lmdb.ml new file mode 100644 index 0000000..0d8586e --- /dev/null +++ b/lmdb/datascript_lmdb.ml @@ -0,0 +1,89 @@ +module Ds = Datascript +open Lmdb + +type session = + { path : string + ; env : Env.t + ; map : (string, string, [ `Uni ]) Map.t + ; mutable closed : bool + } + +let kvs_map_name = "kvs" +let default_map_size = 1024 * 1024 * 1024 + +let lock_path path = path ^ "-lock" + +let remove_files path = + if Sys.file_exists path then Sys.remove path; + let lock = lock_path path in + if Sys.file_exists lock then Sys.remove lock + +let ensure_open session = + if session.closed then invalid_arg "LMDB session is closed" + +let open_env db_path = + Env.(create Rw ~flags:Flags.no_subdir ~map_size:default_map_size ~max_maps:8 db_path) + +let open_map env = + try Map.open_existing Nodup ~key:Conv.string ~value:Conv.string ~name:kvs_map_name env + with Not_found -> + Map.create Nodup ~key:Conv.string ~value:Conv.string ~name:kvs_map_name env + +let open_session db_path = + remove_files db_path; + let env = open_env db_path in + let map = open_map env in + { path = db_path; env; map; closed = false } + +let close session = + if not session.closed then ( + Map.close session.map; + Env.sync session.env; + Env.close session.env; + session.closed <- true) + +let encode_payload payload = Datascript_sqlite_codec.encode payload + +let decode_payload content = Datascript_sqlite_codec.decode content + +let storage session : Ds.storage = + { storage_store = + (fun entries -> + ensure_open session; + ignore + (Txn.go Rw session.env (fun txn -> + List.iter + (fun (address, payload) -> + Map.set ~txn session.map address (encode_payload payload)) + entries; + None))) + ; storage_restore = + (fun address -> + ensure_open session; + (try Some (Map.get session.map address |> decode_payload) + with Not_found -> None)) + ; storage_list_addresses = + (fun () -> + ensure_open session; + let addresses = ref [] in + let next = Map.to_dispenser session.map in + let rec loop () = + match next () with + | None -> () + | Some (address, _) -> + addresses := address :: !addresses; + loop () + in + loop (); + List.rev !addresses) + ; storage_delete = + (fun addresses -> + ensure_open session; + ignore + (Txn.go Rw session.env (fun txn -> + List.iter + (fun address -> + try Map.remove ~txn session.map address with Not_found -> ()) + addresses; + None))) + } diff --git a/lmdb/dune b/lmdb/dune new file mode 100644 index 0000000..8e6ef09 --- /dev/null +++ b/lmdb/dune @@ -0,0 +1,5 @@ +(library + (name datascript_lmdb) + (public_name datascript-ocaml-native.lmdb) + (wrapped false) + (libraries datascript-ocaml-native datascript_sqlite lmdb)) diff --git a/test/dune b/test/dune index 6666a0e..55bfab0 100644 --- a/test/dune +++ b/test/dune @@ -131,6 +131,11 @@ datascript-ocaml-native.sqlite datascript-ocaml-native.logseq-sqlite-storage)) +(test + (name test_lmdb_package) + (modules test_lmdb_package) + (libraries datascript-ocaml-native datascript-ocaml-native.lmdb)) + (test (name test_melange_transit_backend) (modules test_melange_transit_backend) diff --git a/test/test_lmdb_package.ml b/test/test_lmdb_package.ml new file mode 100644 index 0000000..403fa2a --- /dev/null +++ b/test/test_lmdb_package.ml @@ -0,0 +1,68 @@ +open Datascript + +let require condition message = + if not condition then failwith message + +let temp_db_path name = + let path = Filename.temp_file name ".lmdb" in + Sys.remove path; + path + +let indexed = + { cardinality = One + ; unique = Some Identity + ; indexed = true + ; is_component = false + ; no_history = false + ; doc = None + ; value_type = Some StringType + ; tuple_attrs = None + ; tuple_types = None + } + +let test_storage_roundtrip () = + let path = temp_db_path "datascript-lmdb-package" in + let session = Datascript_lmdb.open_session path in + let storage = Datascript_lmdb.storage session in + let db = empty_db ~schema:[ "todo/id", indexed ] ~storage () in + let report = + transact + db + [ Add (Temp_id "todo-1", "todo/id", String "todo-1") + ; Add (Temp_id "todo-1", "todo/title", String "Move storage into datascript") + ] + in + store ~storage report.db_after; + let restored = + match restore storage with + | Some db -> db + | None -> failwith "expected LMDB storage to restore a database" + in + let entity = + match entity restored (Lookup_ref ("todo/id", String "todo-1")) with + | Some entity -> entity + | None -> failwith "expected restored todo entity" + in + require + (entity_attr entity "todo/title" = Some (One_value (String "Move storage into datascript"))) + "expected restored entity title"; + require + (List.mem Storage.root_address (storage_addresses storage)) + "expected LMDB storage to contain the root address"; + Datascript_lmdb.close session + +let test_session_close_blocks_use () = + let path = temp_db_path "datascript-lmdb-session-close" in + let session = Datascript_lmdb.open_session path in + let storage = Datascript_lmdb.storage session in + Datascript_lmdb.close session; + match storage.storage_list_addresses () with + | _ -> failwith "expected closed LMDB session to reject storage operations" + | exception Invalid_argument message -> + require + (String.equal message "LMDB session is closed") + "expected closed session error message" + +let () = + test_storage_roundtrip (); + test_session_close_blocks_use ()