Replace PSS with native LMDB indexes (20k benchmarks beat PSS)

.cursor/Dockerfile

@@ -0,0 +1,35 @@
+FROM ubuntu:24.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV OPAMYES=1
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+        ca-certificates \
+        curl \
+        git \
+        build-essential \
+        pkg-config \
+        libsqlite3-dev \
+        liblmdb-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+# Node.js 24 for js_of_ocaml smoke tests and cross-runtime helpers.
+RUN curl -fsSL https://deb.nodesource.com/setup_24.x | bash - \
+    && apt-get install -y --no-install-recommends nodejs \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN useradd -m -s /bin/bash ubuntu 2>/dev/null || true
+
+USER ubuntu
+WORKDIR /home/ubuntu
+
+RUN curl -fsSL https://raw.githubusercontent.com/ocaml/opam/master/shell/install.sh \
+      | bash -s -- --disable-sandboxing \
+    && opam init --disable-sandboxing -y \
+    && opam switch create 5.5 ocaml-base-compiler.5.5.0 -y \
+    && eval "$(opam env --switch=5.5)" \
+    && opam update -a
+
+RUN echo 'test -r ~/.opam/opam-init/init.sh && . ~/.opam/opam-init/init.sh > /dev/null 2> /dev/null || true' >> ~/.bashrc \
+    && echo 'eval $(opam env --switch=5.5 2>/dev/null)' >> ~/.bashrc

.cursor/cloud-agent-install.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+repo_root="$(git rev-parse --show-toplevel)"
+cd "$repo_root"
+
+eval "$(opam env --switch=5.5)"
+
+export OPAMYES=1
+opam install . --deps-only --with-test -y
+dune build

.cursor/environment.json

@@ -0,0 +1,9 @@
+{
+  "name": "DataScript OCaml (OCaml 5.5)",
+  "user": "ubuntu",
+  "build": {
+    "dockerfile": "Dockerfile",
+    "context": ".."
+  },
+  "install": ".cursor/cloud-agent-install.sh"
+}

.github/workflows/ci.yml

@@ -48,7 +48,7 @@ jobs:
           lein: latest
 
       - name: Install system dependencies
-        run: sudo apt-get update && sudo apt-get install -y libsqlite3-dev pkg-config
+        run: sudo apt-get update && sudo apt-get install -y libsqlite3-dev liblmdb-dev pkg-config
 
       - name: Install OCaml dependencies
         run: opam install . --deps-only --with-test -y

bench/compare_pss_lmdb.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SIZE="${1:-20000}"
+WARMUP_MS="${2:-100}"
+SAMPLE_MS="${3:-200}"
+SAMPLES="${4:-3}"
+
+bench_args=(--size "$SIZE" --warmup-ms "$WARMUP_MS" --sample-ms "$SAMPLE_MS" --samples "$SAMPLES")
+
+run_branch_bench() {
+  local label="$1"
+  local repo="$2"
+  (
+    cd "$repo"
+    dune build bench/bench_ocaml.exe >/dev/null
+    BENCH_RUNTIME_LABEL="$label" dune exec bench/bench_ocaml.exe -- "${bench_args[@]}" 2>/dev/null
+  )
+}
+
+echo "=== PSS vs LMDB benchmark (${SIZE} entities) ==="
+echo "warmup=${WARMUP_MS}ms sample=${SAMPLE_MS}ms samples=${SAMPLES}"
+echo
+
+PSS_OUT="$(run_branch_bench pss /tmp/bench-pss-main)"
+LMDB_OUT="$(run_branch_bench lmdb /workspace)"
+
+printf "%-22s %12s %12s %12s\n" "benchmark" "pss(ms)" "lmdb(ms)" "lmdb/pss"
+echo "----------------------------------------------------------------"
+
+while IFS=$'\t' read -r name pss_ms; do
+  [[ "$name" == runtime* || "$name" == size* || -z "$name" ]] && continue
+  lmdb_ms="$(printf '%s\n' "$LMDB_OUT" | awk -F'\t' -v n="$name" '$1 == n { print $2; exit }')"
+  if [[ -z "$lmdb_ms" ]]; then
+    printf "%-22s %12s %12s %12s\n" "$name" "$pss_ms" "?" "?"
+    continue
+  fi
+  ratio="$(awk -v l="$lmdb_ms" -v p="$pss_ms" 'BEGIN { if (p + 0 == 0) print "?"; else printf "%.2fx", l / p }')"
+  printf "%-22s %12s %12s %12s\n" "$name" "$pss_ms" "$lmdb_ms" "$ratio"
+done <<< "$PSS_OUT"
+
+echo
+echo "=== raw: pss ==="
+printf '%s\n' "$PSS_OUT"
+echo
+echo "=== raw: lmdb ==="
+printf '%s\n' "$LMDB_OUT"

bench/compare_pss_lmdb_20k.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SIZE="${1:-20000}"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+PSS_REPO="${PSS_REPO:-/tmp/bench-pss-main}"
+
+run_branch() {
+  local label="$1"
+  local repo="$2"
+  (
+    cd "$repo"
+    dune build bench/index_compare_20k.exe >/dev/null
+    BENCH_RUNTIME_LABEL="$label" dune exec bench/index_compare_20k.exe -- "$SIZE" 2>/dev/null
+  )
+}
+
+if [[ ! -e "$PSS_REPO/.git" ]]; then
+  echo "PSS worktree missing at $PSS_REPO; run: git worktree add $PSS_REPO main" >&2
+  exit 1
+fi
+
+echo "=== PSS vs LMDB index benchmark (${SIZE} entities / ~${SIZE}0 datoms) ==="
+echo
+
+PSS_OUT="$(run_branch pss "$PSS_REPO")"
+LMDB_OUT="$(run_branch lmdb "$REPO_ROOT")"
+
+printf "%-24s %12s %12s %12s\n" "benchmark" "pss(ms)" "lmdb(ms)" "lmdb/pss"
+echo "------------------------------------------------------------------------"
+
+while IFS=$'\t' read -r name pss_ms; do
+  [[ "$name" == runtime* || "$name" == size* || "$name" == datoms || "$name" == *count* || -z "$name" ]] && continue
+  lmdb_ms="$(printf '%s\n' "$LMDB_OUT" | awk -F'\t' -v n="$name" '$1 == n { print $2; exit }')"
+  if [[ -z "$lmdb_ms" ]]; then
+    printf "%-24s %12s %12s %12s\n" "$name" "$pss_ms" "?" "?"
+    continue
+  fi
+  ratio="$(awk -v l="$lmdb_ms" -v p="$pss_ms" 'BEGIN { if (p + 0 == 0) print "?"; else printf "%.2fx", l / p }')"
+  printf "%-24s %12s %12s %12s\n" "$name" "$pss_ms" "$lmdb_ms" "$ratio"
+done <<< "$PSS_OUT"
+
+echo
+echo "=== raw: pss ==="
+printf '%s\n' "$PSS_OUT"
+echo
+echo "=== raw: lmdb ==="
+printf '%s\n' "$LMDB_OUT"

bench/dune

@@ -1,3 +1,8 @@
+(executable
+ (name index_compare_20k)
+ (modules index_compare_20k)
+ (libraries datascript-ocaml-native unix))
+
 (executable
  (name bench_ocaml)
  (modules bench_ocaml)
@@ -33,6 +38,12 @@
  (modes exe)
  (libraries datascript-ocaml-native datascript_sqlite unix sqlite3))
 
+(executable
+ (name persistent_storage_bench)
+ (modules persistent_storage_bench)
+ (modes exe)
+ (libraries datascript-ocaml-native datascript_sqlite datascript_lmdb unix sqlite3))
+
 (executable
  (name outliner_insert_ocaml)
  (modules outliner_insert_ocaml)

bench/index_compare_20k.ml

@@ -0,0 +1,131 @@
+open Datascript
+
+type timing = { label : string; elapsed_ms : float }
+
+let now_ms () = Unix.gettimeofday () *. 1000.
+
+let time label f =
+  let start = now_ms () in
+  let result = f () in
+  ({ label; elapsed_ms = now_ms () -. start }, result)
+
+let print_timing { label; elapsed_ms } =
+  Printf.printf "%s\t%.2f\n%!" label elapsed_ms
+
+let indexed =
+  {
+    cardinality = One;
+    unique = None;
+    indexed = true;
+    is_component = false;
+    no_history = false;
+    doc = None;
+    value_type = None;
+    tuple_attrs = None;
+    tuple_types = None;
+  }
+
+let unique_identity = { indexed with unique = Some Identity }
+
+let many =
+  {
+    cardinality = Many;
+    unique = None;
+    indexed = false;
+    is_component = false;
+    no_history = false;
+    doc = None;
+    value_type = None;
+    tuple_attrs = None;
+    tuple_types = None;
+  }
+
+let schema =
+  [ ("id", unique_identity)
+  ; ("name", indexed)
+  ; ("age", indexed)
+  ; ("salary", indexed)
+  ; ("alias", many)
+  ]
+
+let names = [| "Ivan"; "Petr"; "Sergey"; "Oleg"; "Yuri"; "Dmitry"; "Fedor"; "Denis" |]
+let last_names = [| "Ivanov"; "Petrov"; "Sidorov"; "Kovalev"; "Kuznetsov"; "Voronoi" |]
+
+type rng = { mutable state : int32 }
+
+let rng seed = { state = Int32.of_int seed }
+
+let next_int rng bound =
+  rng.state <- Int32.add (Int32.mul rng.state 1_664_525l) 1_013_904_223l;
+  Int32.(to_int (rem (logand (shift_right_logical rng.state 1) 0x3fffffffl) (of_int bound)))
+
+let rand_nth rng values = values.(next_int rng (Array.length values))
+
+let datoms_for size =
+  let rng = rng 1 in
+  List.init size (fun index ->
+      let i = index + 1 in
+      let name = rand_nth rng names in
+      let last_name = rand_nth rng last_names in
+      [
+        { e = i; a = "name"; v = String name; tx = 0x20000001; added = true }
+      ; { e = i; a = "last-name"; v = String last_name; tx = 0x20000001; added = true }
+      ; { e = i; a = "age"; v = Int (next_int rng 100); tx = 0x20000001; added = true }
+      ; { e = i; a = "salary"; v = Int (next_int rng 100_000); tx = 0x20000001; added = true }
+      ])
+  |> List.concat
+
+let entity_count db = Seq.length (datoms db Eavt ())
+
+let parse_size () =
+  match Sys.argv with
+  | [| _; size |] -> int_of_string size
+  | _ -> 20_000
+
+let main () =
+  let size = parse_size () in
+  let runtime_label =
+    match Sys.getenv_opt "BENCH_RUNTIME_LABEL" with
+    | Some label -> label
+    | None -> "ocaml"
+  in
+  Printf.printf "runtime\t%s\n%!" runtime_label;
+  Printf.printf "size\t%d\n%!" size;
+  Printf.printf "datoms\t%d\n%!" (size * 4);
+  let datoms = datoms_for size in
+  let build_all, db =
+    time "build-all-init" (fun () -> init_db ~schema datoms)
+  in
+  print_timing build_all;
+  let find_name, rows =
+    time "query-name-ivan" (fun () ->
+      q_string db "[:find ?e :where [?e :name \"Ivan\"]]")
+  in
+  print_timing find_name;
+  Printf.printf "query-name-ivan-count\t%d\n%!" (List.length rows);
+  Printf.printf "datom-count\t%d\n%!" (entity_count db);
+  let scan_name, count =
+    time "scan-aevt-name" (fun () ->
+      fold_datoms (fun count _ -> count + 1) 0 db Aevt ~a:"name" ())
+  in
+  print_timing scan_name;
+  Printf.printf "scan-aevt-name-count\t%d\n%!" count;
+  let add_one, db =
+    time "add-one-tx" (fun () ->
+      db_with [ Add (Entity_id 1, "nickname", String "Vanya") ] db)
+  in
+  print_timing add_one;
+  ignore db;
+  let storage, restored =
+    time "storage-roundtrip" (fun () ->
+      let storage = memory_storage () in
+      let db = init_db ~schema ~storage datoms in
+      store db;
+      match restore storage with
+      | Some db -> db
+      | None -> failwith "restore failed")
+  in
+  print_timing storage;
+  Printf.printf "restored-datom-count\t%d\n%!" (entity_count restored)
+
+let () = main ()