From 2180e8ec7b2f3bcc3cbce21a99256ae9f9ce181e Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 26 Aug 2026 19:25:45 +0000
Subject: [PATCH 01/10] Add Cloud Agent environment with OCaml 5.5

Define a repository-managed environment for datascript-ocaml:
- Base image installs opam, OCaml 5.5.0, libsqlite3-dev, and Node.js 24
- Install script runs opam deps and dune build after checkout

Co-authored-by: Tienson Qin <tiensonqin@gmail.com>
---
 .cursor/Dockerfile             | 34 ++++++++++++++++++++++++++++++++++
 .cursor/cloud-agent-install.sh | 11 +++++++++++
 .cursor/environment.json       |  9 +++++++++
 3 files changed, 54 insertions(+)
 create mode 100644 .cursor/Dockerfile
 create mode 100755 .cursor/cloud-agent-install.sh
 create mode 100644 .cursor/environment.json

diff --git a/.cursor/Dockerfile b/.cursor/Dockerfile
new file mode 100644
index 0000000..fc698dc
--- /dev/null
+++ b/.cursor/Dockerfile
@@ -0,0 +1,34 @@
+FROM ubuntu:24.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV OPAMYES=1
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+        ca-certificates \
+        curl \
+        git \
+        build-essential \
+        pkg-config \
+        libsqlite3-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+# Node.js 24 for js_of_ocaml smoke tests and cross-runtime helpers.
+RUN curl -fsSL https://deb.nodesource.com/setup_24.x | bash - \
+    && apt-get install -y --no-install-recommends nodejs \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN useradd -m -s /bin/bash ubuntu 2>/dev/null || true
+
+USER ubuntu
+WORKDIR /home/ubuntu
+
+RUN curl -fsSL https://raw.githubusercontent.com/ocaml/opam/master/shell/install.sh \
+      | bash -s -- --disable-sandboxing \
+    && opam init --disable-sandboxing -y \
+    && opam switch create 5.5 ocaml-base-compiler.5.5.0 -y \
+    && eval "$(opam env --switch=5.5)" \
+    && opam update -a
+
+RUN echo 'test -r ~/.opam/opam-init/init.sh && . ~/.opam/opam-init/init.sh > /dev/null 2> /dev/null || true' >> ~/.bashrc \
+    && echo 'eval $(opam env --switch=5.5 2>/dev/null)' >> ~/.bashrc
diff --git a/.cursor/cloud-agent-install.sh b/.cursor/cloud-agent-install.sh
new file mode 100755
index 0000000..798992e
--- /dev/null
+++ b/.cursor/cloud-agent-install.sh
@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+repo_root="$(git rev-parse --show-toplevel)"
+cd "$repo_root"
+
+eval "$(opam env --switch=5.5)"
+
+export OPAMYES=1
+opam install . --deps-only --with-test -y
+dune build
diff --git a/.cursor/environment.json b/.cursor/environment.json
new file mode 100644
index 0000000..491f294
--- /dev/null
+++ b/.cursor/environment.json
@@ -0,0 +1,9 @@
+{
+  "name": "DataScript OCaml (OCaml 5.5)",
+  "user": "ubuntu",
+  "build": {
+    "dockerfile": "Dockerfile",
+    "context": ".."
+  },
+  "install": ".cursor/cloud-agent-install.sh"
+}

From 2de5d747aaf9f2b6b98275991e9c69a01d85686b Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 26 Aug 2026 19:30:47 +0000
Subject: [PATCH 02/10] Add LMDB storage backend and SQLite comparison
 benchmark

Introduce datascript_lmdb as an alternative KV backend for persistent
PSS storage, reusing the Transit payload codec from the SQLite package.
Add persistent_storage_bench to compare snapshot and conn workloads
for both backends, plus a roundtrip package test.

Co-authored-by: Tienson Qin <tiensonqin@gmail.com>
---
 .cursor/Dockerfile                |   1 +
 .github/workflows/ci.yml          |   2 +-
 bench/dune                        |   6 +
 bench/persistent_storage_bench.ml | 263 ++++++++++++++++++++++++++++++
 datascript-ocaml-native.opam      |   1 +
 lmdb/datascript_lmdb.ml           |  89 ++++++++++
 lmdb/dune                         |   5 +
 test/dune                         |   5 +
 test/test_lmdb_package.ml         |  68 ++++++++
 9 files changed, 439 insertions(+), 1 deletion(-)
 create mode 100644 bench/persistent_storage_bench.ml
 create mode 100644 lmdb/datascript_lmdb.ml
 create mode 100644 lmdb/dune
 create mode 100644 test/test_lmdb_package.ml

diff --git a/.cursor/Dockerfile b/.cursor/Dockerfile
index fc698dc..4374c88 100644
--- a/.cursor/Dockerfile
+++ b/.cursor/Dockerfile
@@ -11,6 +11,7 @@ RUN apt-get update \
         build-essential \
         pkg-config \
         libsqlite3-dev \
+        liblmdb-dev \
     && rm -rf /var/lib/apt/lists/*
 
 # Node.js 24 for js_of_ocaml smoke tests and cross-runtime helpers.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index f883c22..cbfbd2b 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -48,7 +48,7 @@ jobs:
           lein: latest
 
       - name: Install system dependencies
-        run: sudo apt-get update && sudo apt-get install -y libsqlite3-dev pkg-config
+        run: sudo apt-get update && sudo apt-get install -y libsqlite3-dev liblmdb-dev pkg-config
 
       - name: Install OCaml dependencies
         run: opam install . --deps-only --with-test -y
diff --git a/bench/dune b/bench/dune
index 67c4a83..750568a 100644
--- a/bench/dune
+++ b/bench/dune
@@ -33,6 +33,12 @@
  (modes exe)
  (libraries datascript-ocaml-native datascript_sqlite unix sqlite3))
 
+(executable
+ (name persistent_storage_bench)
+ (modules persistent_storage_bench)
+ (modes exe)
+ (libraries datascript-ocaml-native datascript_sqlite datascript_lmdb unix sqlite3))
+
 (executable
  (name outliner_insert_ocaml)
  (modules outliner_insert_ocaml)
diff --git a/bench/persistent_storage_bench.ml b/bench/persistent_storage_bench.ml
new file mode 100644
index 0000000..1dd101d
--- /dev/null
+++ b/bench/persistent_storage_bench.ml
@@ -0,0 +1,263 @@
+open Datascript
+
+type timing = { label : string; elapsed_ms : float }
+
+let now_ms () = Unix.gettimeofday () *. 1000.
+
+let time label f =
+  let start = now_ms () in
+  let result = f () in
+  ({ label; elapsed_ms = now_ms () -. start }, result)
+
+let print_timing prefix { label; elapsed_ms } =
+  Printf.printf "%s%s\t%.2f\n%!" prefix label elapsed_ms
+
+let indexed =
+  {
+    cardinality = One;
+    unique = None;
+    indexed = true;
+    is_component = false;
+    no_history = false;
+    doc = None;
+    value_type = None;
+    tuple_attrs = None;
+    tuple_types = None;
+  }
+
+let unique_identity = { indexed with unique = Some Identity }
+
+let schema =
+  [
+    ("block/id", unique_identity);
+    ("block/journal-day", indexed);
+    ("block/content", indexed);
+    ("block/order", indexed);
+    ("block/collapsed", indexed);
+  ]
+
+let block_tx count =
+  List.init count (fun index ->
+      let i = index + 1 in
+      Entity
+        {
+          db_id = Some (Temp_id (Printf.sprintf "block-%05d" i));
+          attrs =
+            [
+              ("block/id", One_value (String (Printf.sprintf "block-%05d" i)));
+              ("block/journal-day", One_value (String "2026-06-27"));
+              ("block/content", One_value (String (Printf.sprintf "Block %05d" i)));
+              ("block/order", One_value (Float (Float.of_int i)));
+              ("block/collapsed", One_value (Bool false));
+            ];
+        })
+
+let add_block_tx id order =
+  [
+    Entity
+      {
+        db_id = Some (Temp_id id);
+        attrs =
+          [
+            ("block/id", One_value (String id));
+            ("block/journal-day", One_value (String "2026-06-27"));
+            ("block/content", One_value (String id));
+            ("block/order", One_value (Float order));
+            ("block/collapsed", One_value (Bool false));
+          ];
+      };
+  ]
+
+let update_content_tx id content =
+  [ Add (Lookup_ref ("block/id", String id), "block/content", String content) ]
+
+let seq_length seq = Seq.fold_left (fun count _ -> count + 1) 0 seq
+
+let file_size path =
+  if Sys.file_exists path then (Unix.stat path).st_size else 0
+
+let remove_if_exists path = if Sys.file_exists path then Sys.remove path
+
+let row_count storage = List.length (storage_addresses storage)
+
+module type BACKEND = sig
+  val name : string
+  val extension : string
+  type session
+  val open_session : string -> session
+  val close_session : session -> unit
+  val storage : session -> storage
+  val cleanup : string -> unit
+end
+
+module Sqlite_backend : BACKEND = struct
+  type session = Datascript_sqlite.session
+
+  let name = "sqlite"
+  let extension = "sqlite3"
+  let open_session = Datascript_sqlite.open_session
+  let close_session = Datascript_sqlite.close
+  let storage = Datascript_sqlite.storage
+  let cleanup _path = ()
+end
+
+module Lmdb_backend : BACKEND = struct
+  type session = Datascript_lmdb.session
+
+  let name = "lmdb"
+  let extension = "lmdb"
+  let open_session = Datascript_lmdb.open_session
+  let close_session = Datascript_lmdb.close
+  let storage = Datascript_lmdb.storage
+
+  let cleanup path =
+    let lock = path ^ "-lock" in
+    if Sys.file_exists lock then Sys.remove lock
+end
+
+let run_backend (module B : BACKEND) size tx =
+  let prefix = B.name ^ "-" in
+  let db_path =
+    Filename.concat
+      (Filename.get_temp_dir_name ())
+      (Printf.sprintf "datascript-persistent-%s-%d.%s" B.name size B.extension)
+  in
+  remove_if_exists db_path;
+  let session = B.open_session db_path in
+  Fun.protect
+    ~finally:(fun () ->
+      B.close_session session;
+      remove_if_exists db_path;
+      B.cleanup db_path)
+    (fun () ->
+      let storage = B.storage session in
+      let persistent_build, persistent_db =
+        time "snapshot-build-and-store" (fun () ->
+            let db = db_with tx (empty_db ~schema ~storage ()) in
+            store db;
+            db)
+      in
+      print_timing prefix persistent_build;
+      Printf.printf "%ssnapshot-build-datoms\t%d\n%!" prefix
+        (seq_length (datoms persistent_db Eavt ()));
+      Printf.printf "%ssnapshot-kvs-rows-after-build\t%d\n%!" prefix (row_count storage);
+      Printf.printf "%ssnapshot-file-size-after-build\t%d\n%!" prefix (file_size db_path);
+      let restore_timing, restored_db =
+        time "snapshot-restore" (fun () ->
+            match restore storage with
+            | Some db -> db
+            | None -> failwith (B.name ^ " persistent db should restore"))
+      in
+      print_timing prefix restore_timing;
+      let persistent_add, restored_db =
+        time "snapshot-add-one-and-store-after-restore" (fun () ->
+            let db =
+              db_with
+                (add_block_tx "persistent-new" (Float.of_int (size + 1)))
+                restored_db
+            in
+            store db;
+            db)
+      in
+      print_timing prefix persistent_add;
+      Printf.printf "%ssnapshot-kvs-rows-after-add\t%d\n%!" prefix (row_count storage);
+      Printf.printf "%ssnapshot-file-size-after-add\t%d\n%!" prefix (file_size db_path);
+      let persistent_update, restored_db =
+        time "snapshot-update-one-and-store-after-add" (fun () ->
+            let db = db_with (update_content_tx "block-00001" "Edited") restored_db in
+            store db;
+            db)
+      in
+      print_timing prefix persistent_update;
+      Printf.printf "%ssnapshot-kvs-rows-after-update\t%d\n%!" prefix (row_count storage);
+      Printf.printf "%ssnapshot-file-size-after-update\t%d\n%!" prefix (file_size db_path);
+      Printf.printf "%ssnapshot-datoms\t%d\n%!" prefix
+        (seq_length (datoms restored_db Eavt ()));
+      let conn_db_path =
+        Filename.concat
+          (Filename.get_temp_dir_name ())
+          (Printf.sprintf "datascript-persistent-%s-conn-%d.%s" B.name size B.extension)
+      in
+      remove_if_exists conn_db_path;
+      let session = B.open_session conn_db_path in
+      Fun.protect
+        ~finally:(fun () ->
+          B.close_session session;
+          remove_if_exists conn_db_path;
+          B.cleanup conn_db_path)
+        (fun () ->
+          let storage = B.storage session in
+          let conn_build, conn =
+            time "conn-build" (fun () ->
+                let conn = create_conn ~schema ~storage () in
+                ignore (transact_conn conn tx);
+                conn)
+          in
+          print_timing prefix conn_build;
+          Printf.printf "%sconn-build-datoms\t%d\n%!" prefix
+            (seq_length (datoms (db conn) Eavt ()));
+          Printf.printf "%sconn-kvs-rows-after-build\t%d\n%!" prefix (row_count storage);
+          Printf.printf "%sconn-file-size-after-build\t%d\n%!" prefix (file_size conn_db_path);
+          let conn_restore, conn =
+            time "conn-restore" (fun () ->
+                match restore_conn storage with
+                | Some conn -> conn
+                | None -> failwith (B.name ^ " persistent conn should restore"))
+          in
+          print_timing prefix conn_restore;
+          let conn_add, _report =
+            time "conn-add-one-after-restore" (fun () ->
+                transact_conn conn (add_block_tx "conn-new" (Float.of_int (size + 1))))
+          in
+          print_timing prefix conn_add;
+          Printf.printf "%sconn-kvs-rows-after-add\t%d\n%!" prefix (row_count storage);
+          Printf.printf "%sconn-file-size-after-add\t%d\n%!" prefix (file_size conn_db_path);
+          let conn_update, _report =
+            time "conn-update-one-after-add" (fun () ->
+                transact_conn conn (update_content_tx "block-00001" "Edited"))
+          in
+          print_timing prefix conn_update;
+          Printf.printf "%sconn-kvs-rows-after-update\t%d\n%!" prefix (row_count storage);
+          Printf.printf "%sconn-file-size-after-update\t%d\n%!" prefix (file_size conn_db_path);
+          Printf.printf "%sconn-datoms\t%d\n%!" prefix (seq_length (datoms (db conn) Eavt ()))))
+
+let run_size size =
+  Printf.printf "size\t%d\n%!" size;
+  let tx = block_tx size in
+  let memory_build, memory_db =
+    time "memory-build" (fun () -> db_with tx (empty_db ~schema ()))
+  in
+  print_timing "" memory_build;
+  let memory_add, memory_db =
+    time "memory-add-one" (fun () ->
+        db_with (add_block_tx "memory-new" (Float.of_int (size + 1))) memory_db)
+  in
+  print_timing "" memory_add;
+  let _memory_update, memory_db =
+    time "memory-update-one" (fun () ->
+        db_with (update_content_tx "block-00001" "Edited") memory_db)
+  in
+  print_timing "" _memory_update;
+  Printf.printf "memory-datoms\t%d\n%!" (seq_length (datoms memory_db Eavt ()));
+  run_backend (module Sqlite_backend) size tx;
+  run_backend (module Lmdb_backend) size tx
+
+let parse_sizes () =
+  let rec loop sizes = function
+    | [] -> List.rev sizes
+    | "--size" :: size :: rest -> loop (int_of_string size :: sizes) rest
+    | "--sizes" :: value :: rest ->
+        let parsed =
+          value
+          |> String.split_on_char ','
+          |> List.filter (fun value -> String.length value > 0)
+          |> List.map int_of_string
+        in
+        loop (List.rev_append parsed sizes) rest
+    | arg :: _ -> invalid_arg ("unknown benchmark argument: " ^ arg)
+  in
+  match loop [] (Sys.argv |> Array.to_list |> List.tl) with
+  | [] -> [ 100; 1000; 5000 ]
+  | sizes -> sizes
+
+let () = List.iter run_size (parse_sizes ())
diff --git a/datascript-ocaml-native.opam b/datascript-ocaml-native.opam
index 9fb78c0..889fc22 100644
--- a/datascript-ocaml-native.opam
+++ b/datascript-ocaml-native.opam
@@ -10,6 +10,7 @@ depends: [
   "datascript_ocaml" {= version}
   "persistent_sorted_set_ocaml" {= "dev"}
   "sqlite3"
+  "lmdb"
   "melange-transit-native" {= "0.1.0"}
   "yojson"
 ]
diff --git a/lmdb/datascript_lmdb.ml b/lmdb/datascript_lmdb.ml
new file mode 100644
index 0000000..0d8586e
--- /dev/null
+++ b/lmdb/datascript_lmdb.ml
@@ -0,0 +1,89 @@
+module Ds = Datascript
+open Lmdb
+
+type session =
+  { path : string
+  ; env : Env.t
+  ; map : (string, string, [ `Uni ]) Map.t
+  ; mutable closed : bool
+  }
+
+let kvs_map_name = "kvs"
+let default_map_size = 1024 * 1024 * 1024
+
+let lock_path path = path ^ "-lock"
+
+let remove_files path =
+  if Sys.file_exists path then Sys.remove path;
+  let lock = lock_path path in
+  if Sys.file_exists lock then Sys.remove lock
+
+let ensure_open session =
+  if session.closed then invalid_arg "LMDB session is closed"
+
+let open_env db_path =
+  Env.(create Rw ~flags:Flags.no_subdir ~map_size:default_map_size ~max_maps:8 db_path)
+
+let open_map env =
+  try Map.open_existing Nodup ~key:Conv.string ~value:Conv.string ~name:kvs_map_name env
+  with Not_found ->
+    Map.create Nodup ~key:Conv.string ~value:Conv.string ~name:kvs_map_name env
+
+let open_session db_path =
+  remove_files db_path;
+  let env = open_env db_path in
+  let map = open_map env in
+  { path = db_path; env; map; closed = false }
+
+let close session =
+  if not session.closed then (
+    Map.close session.map;
+    Env.sync session.env;
+    Env.close session.env;
+    session.closed <- true)
+
+let encode_payload payload = Datascript_sqlite_codec.encode payload
+
+let decode_payload content = Datascript_sqlite_codec.decode content
+
+let storage session : Ds.storage =
+  { storage_store =
+      (fun entries ->
+        ensure_open session;
+        ignore
+          (Txn.go Rw session.env (fun txn ->
+             List.iter
+               (fun (address, payload) ->
+                 Map.set ~txn session.map address (encode_payload payload))
+               entries;
+             None)))
+  ; storage_restore =
+      (fun address ->
+        ensure_open session;
+        (try Some (Map.get session.map address |> decode_payload)
+         with Not_found -> None))
+  ; storage_list_addresses =
+      (fun () ->
+        ensure_open session;
+        let addresses = ref [] in
+        let next = Map.to_dispenser session.map in
+        let rec loop () =
+          match next () with
+          | None -> ()
+          | Some (address, _) ->
+              addresses := address :: !addresses;
+              loop ()
+        in
+        loop ();
+        List.rev !addresses)
+  ; storage_delete =
+      (fun addresses ->
+        ensure_open session;
+        ignore
+          (Txn.go Rw session.env (fun txn ->
+             List.iter
+               (fun address ->
+                 try Map.remove ~txn session.map address with Not_found -> ())
+               addresses;
+             None)))
+  }
diff --git a/lmdb/dune b/lmdb/dune
new file mode 100644
index 0000000..8e6ef09
--- /dev/null
+++ b/lmdb/dune
@@ -0,0 +1,5 @@
+(library
+ (name datascript_lmdb)
+ (public_name datascript-ocaml-native.lmdb)
+ (wrapped false)
+ (libraries datascript-ocaml-native datascript_sqlite lmdb))
diff --git a/test/dune b/test/dune
index 6666a0e..55bfab0 100644
--- a/test/dune
+++ b/test/dune
@@ -131,6 +131,11 @@
   datascript-ocaml-native.sqlite
   datascript-ocaml-native.logseq-sqlite-storage))
 
+(test
+ (name test_lmdb_package)
+ (modules test_lmdb_package)
+ (libraries datascript-ocaml-native datascript-ocaml-native.lmdb))
+
 (test
  (name test_melange_transit_backend)
  (modules test_melange_transit_backend)
diff --git a/test/test_lmdb_package.ml b/test/test_lmdb_package.ml
new file mode 100644
index 0000000..403fa2a
--- /dev/null
+++ b/test/test_lmdb_package.ml
@@ -0,0 +1,68 @@
+open Datascript
+
+let require condition message =
+  if not condition then failwith message
+
+let temp_db_path name =
+  let path = Filename.temp_file name ".lmdb" in
+  Sys.remove path;
+  path
+
+let indexed =
+  { cardinality = One
+  ; unique = Some Identity
+  ; indexed = true
+  ; is_component = false
+  ; no_history = false
+  ; doc = None
+  ; value_type = Some StringType
+  ; tuple_attrs = None
+  ; tuple_types = None
+  }
+
+let test_storage_roundtrip () =
+  let path = temp_db_path "datascript-lmdb-package" in
+  let session = Datascript_lmdb.open_session path in
+  let storage = Datascript_lmdb.storage session in
+  let db = empty_db ~schema:[ "todo/id", indexed ] ~storage () in
+  let report =
+    transact
+      db
+      [ Add (Temp_id "todo-1", "todo/id", String "todo-1")
+      ; Add (Temp_id "todo-1", "todo/title", String "Move storage into datascript")
+      ]
+  in
+  store ~storage report.db_after;
+  let restored =
+    match restore storage with
+    | Some db -> db
+    | None -> failwith "expected LMDB storage to restore a database"
+  in
+  let entity =
+    match entity restored (Lookup_ref ("todo/id", String "todo-1")) with
+    | Some entity -> entity
+    | None -> failwith "expected restored todo entity"
+  in
+  require
+    (entity_attr entity "todo/title" = Some (One_value (String "Move storage into datascript")))
+    "expected restored entity title";
+  require
+    (List.mem Storage.root_address (storage_addresses storage))
+    "expected LMDB storage to contain the root address";
+  Datascript_lmdb.close session
+
+let test_session_close_blocks_use () =
+  let path = temp_db_path "datascript-lmdb-session-close" in
+  let session = Datascript_lmdb.open_session path in
+  let storage = Datascript_lmdb.storage session in
+  Datascript_lmdb.close session;
+  match storage.storage_list_addresses () with
+  | _ -> failwith "expected closed LMDB session to reject storage operations"
+  | exception Invalid_argument message ->
+      require
+        (String.equal message "LMDB session is closed")
+        "expected closed session error message"
+
+let () =
+  test_storage_roundtrip ();
+  test_session_close_blocks_use ()

From cdc61f47a25953cc6947c3c22fee148cc185a015 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 26 Aug 2026 20:01:32 +0000
Subject: [PATCH 03/10] Document non-PSS LMDB design with Scheme A index types

Co-authored-by: Tienson Qin <tiensonqin@gmail.com>
---
 docs/design-non-pss.md | 153 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 153 insertions(+)
 create mode 100644 docs/design-non-pss.md

diff --git a/docs/design-non-pss.md b/docs/design-non-pss.md
new file mode 100644
index 0000000..c9ef052
--- /dev/null
+++ b/docs/design-non-pss.md
@@ -0,0 +1,153 @@
+# Non-PSS LMDB Index Design
+
+Branch: `feat/non-pss`
+
+This document records the architecture for replacing `Persistent_sorted_set` indexes
+with native LMDB indexes. Queries and writes go directly to LMDB. Datoms keep `tx`,
+`added`, and DataScript `value` types (not Datalevin AVG/aid encoding).
+
+## Type strategy: Scheme A
+
+Replace PSS field types in `db` with an explicit `Index.t`. Do not alias
+`Persistent_sorted_set` to LMDB.
+
+### New public types (`type/datascript_types.ml`)
+
+```ocaml
+module Index : sig
+  type t
+  type 'a seq
+  val to_seq : 'a seq -> 'a Seq.t
+end
+
+type index = Index.t
+
+and db = {
+  db_uid : int;
+  schema : schema;
+  eavt_index : index;
+  aevt_index : index;
+  avet_index : index;
+  (* caches and duplicate side tables unchanged *)
+  ...
+  storage_ref : storage option;
+  ...
+}
+```
+
+`Index.t` is abstract at the type level. Native builds use LMDB; the type does not
+mention PSS or LMDB in `datascript_types`.
+
+### Index module API (`lmdb/datascript_lmdb_index.mli`)
+
+Surface area required by `impl/db.ml` (PSS-free subset):
+
+```ocaml
+type t
+type 'a seq
+
+val empty : index -> t
+val of_sorted_list : index -> datom list -> t
+
+val add : datom -> t -> t
+val remove : datom -> t -> t
+
+val to_list : t -> datom list
+val fold : (acc -> datom -> acc) -> acc -> t -> acc
+
+val slice : ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom list
+val slice_seq : ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom seq
+val rslice_seq : ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom seq
+val seq : t -> datom seq
+val seq_to_list : datom seq -> datom list
+val fold_seq : (acc -> datom -> acc) -> acc -> datom seq -> acc
+val to_seq : datom seq -> datom Seq.t
+```
+
+Notes:
+
+- `index` parameter is `Eavt | Aevt | Avet` (`Datascript_types.index`).
+- Default comparator is `Util.compare_datom index`; bound slices pass custom `cmp`
+  (same as PSS today).
+- `t` holds a reference to the shared LMDB environment and the DBI name for that
+  logical index (`ds/eavt`, `ds/aevt`, `ds/avet`).
+
+### Shared LMDB database handle
+
+Introduce `Lmdb_db.t` (one env per storage session):
+
+```ocaml
+type t = {
+  env : Lmdb.Env.t;
+  eavt : (bytes, bytes, [`Uni]) Lmdb.Map.t;
+  aevt : (bytes, bytes, [`Uni]) Lmdb.Map.t;
+  avet : (bytes, bytes, [`Uni]) Lmdb.Map.t;
+  meta : (string, bytes, [`Uni]) Lmdb.Map.t;
+  path : string;
+}
+```
+
+Each `Index.t` is `{ db : Lmdb_db.t; which : index }` or three dedicated handles
+created at `empty_db` / `restore`.
+
+`storage` on `feat/non-pss` simplifies to wrapping `Lmdb_db.t`:
+
+```ocaml
+type storage = Lmdb_db.t
+```
+
+Remove PSS snapshot payloads (`Storage_root`, `Storage_node`, tail groups) from the
+native non-PSS path. Logseq SQLite reader code stays on other branches.
+
+## LMDB key/value encoding
+
+Keys are order-preserving binary tuples matching `Util.compare_datom`:
+
+| Index | Key field order |
+| --- | --- |
+| EAVT | e, a, v, tx |
+| AEVT | a, e, v, tx |
+| AVET | a, v, e, tx |
+
+- `v` uses a typed order-preserving encoder aligned with `Util.compare_value`.
+- `tx` is included in the key (unlike Datalevin).
+- Value payload stores at least `added : bool` and may mirror `v` for simpler decode.
+
+Duplicate numeric facts (`Int 1` vs `Float 1.0` comparator-equal but distinct) stay
+in side tables (`duplicate_*` fields on `db`) with merge at read time, same as today.
+
+## Code migration map (Scheme A)
+
+| Area | Change |
+| --- | --- |
+| `type/datascript_types.ml` | `eavt_index` etc. become `Index.t`; drop PSS from `db` |
+| `impl/db.ml` | `module PSet = ...` removed; call `Lmdb_index.*` |
+| `impl/datascript.ml` | Replace direct `PSet.*` on `db.*_index` |
+| `impl/serialize.ml` | Iterate LMDB or `to_list`; no PSS builders |
+| `impl/storage.ml` | Open/sync `Lmdb_db`; delete PSS store/restore/tail |
+| `impl/conn.ml` | Commit LMDB txn per transact; no tail compaction |
+| `lmdb/*` | `datascript_lmdb_codec.ml`, `datascript_lmdb_index.ml`, `datascript_lmdb_db.ml` |
+| `impl/dune` | Native links `datascript_lmdb_index`; drop `persistent_sorted_set_ocaml` on this branch |
+| `test/test_db.ml` | Replace `assert_uses_persistent_sorted_set` with LMDB index checks |
+| `bench/*` | Update labels; expect no snapshot full-tree rewrite |
+
+## What stays unchanged
+
+- Public query semantics: lazy `datoms`, bound slices, filtered DB order.
+- `datom` record including `tx` and `added`.
+- `value` algebra and `Util.compare_value` rules.
+- Three indexes: EAVT, AEVT, AVET.
+
+## Implementation phases
+
+1. **Codec + Index unit tests** — key order matches `compare_datom`.
+2. **`Lmdb_db` lifecycle** — temp env for `empty_db ()`, path-backed for conn.
+3. **Rewire `db.ml`** — Scheme A types throughout.
+4. **Storage + conn** — remove PSS snapshot/tail.
+5. **Tests + bench** — parity vs PSS branch on small fixtures; benchmark tables.
+
+## Explicit non-goals (this branch)
+
+- js_of_ocaml / melange non-PSS.
+- Logseq PSS `db.sqlite` compatibility.
+- Datalevin-style aid/AVG encoding or dropping `tx` from storage.

From 2f53f7dd9eaeb269b8699b36a30e71520025d1d1 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 26 Aug 2026 20:31:08 +0000
Subject: [PATCH 04/10] Remove file storage and transaction tail from LMDB
 storage

Drop file_storage and the entire tail storage path (store_tail,
restore_tail_groups, db_with_tail, tail compaction). Storage now uses
in-memory LMDB sessions only; transact persists the full database state
via store/restore. Update public APIs, platform storage modules, and
tests accordingly.

Co-authored-by: Tienson Qin <tiensonqin@gmail.com>
---
 datascript-ocaml-melange.opam           |   2 -
 datascript-ocaml-native.opam            |   2 -
 datascript_ocaml.opam                   |   4 -
 impl/conn.ml                            |  39 +--
 impl/conn.mli                           |  12 +-
 impl/datascript.ml                      |  87 +------
 impl/datascript.mli                     |  28 +-
 impl/db.ml                              | 107 ++++----
 impl/dune                               |   5 +-
 impl/index.mli                          |  24 ++
 impl/platform.mli                       |   5 -
 impl/platform/jsoo/dune                 |   6 +-
 impl/platform/jsoo/index.ml             |  36 +++
 impl/platform/jsoo/platform.ml          |   3 -
 impl/platform/jsoo/storage.ml           | 105 ++++++++
 impl/platform/melange/dune              |   6 +-
 impl/platform/melange/index.ml          |  36 +++
 impl/platform/melange/platform.ml       |   3 -
 impl/platform/melange/storage.ml        | 105 ++++++++
 impl/platform/native/dune               |   7 +-
 impl/platform/native/index.ml           |  36 +++
 impl/platform/native/platform.ml        |  81 ------
 impl/platform/native/storage.ml         | 105 ++++++++
 impl/serialize.ml                       |  31 +--
 impl/storage.mli                        |  17 +-
 impl/storage_lmdb_impl.ml               | 122 +++++++++
 impl/{storage.ml => storage_pss.ml}     |   0
 impl/util.ml                            | 302 +---------------------
 impl/util.mli                           |   1 +
 lmdb/datascript_lmdb.ml                 |   4 +-
 lmdb/datascript_lmdb_codec.ml           | 289 +++++++++++++++++++++
 lmdb/datascript_lmdb_codec.mli          |  13 +
 lmdb/datascript_lmdb_db.mli             |  16 ++
 lmdb/datascript_lmdb_db_melange.ml      |  88 +++++++
 lmdb/datascript_lmdb_index.ml           |  94 +++++++
 lmdb/datascript_lmdb_index.mli          |  30 +++
 lmdb/datascript_lmdb_node.js            |  51 ++++
 lmdb/datascript_storage_lmdb.ml         |  80 ++++++
 lmdb/dune                               |  20 +-
 lmdb/melange/datascript_lmdb_db.ml      | 108 ++++++++
 lmdb/melange/datascript_lmdb_db.mli     |  16 ++
 lmdb/melange/datascript_lmdb_index.ml   |  94 +++++++
 lmdb/melange/datascript_lmdb_index.mli  |  30 +++
 lmdb/melange/datascript_lmdb_node.js    |  51 ++++
 lmdb/melange/datascript_storage_lmdb.ml |  80 ++++++
 lmdb/melange/dune                       |  25 ++
 lmdb/native/datascript_lmdb_db.ml       | 104 ++++++++
 lmdb/native/datascript_lmdb_db.mli      |  16 ++
 lmdb/native/datascript_lmdb_index.ml    |  94 +++++++
 lmdb/native/datascript_lmdb_index.mli   |  30 +++
 lmdb/native/datascript_storage_lmdb.ml  |  80 ++++++
 lmdb/native/dune                        |  25 ++
 melange/datascript_melange_storage.ml   |  72 ++++--
 melange/dune                            |   3 +-
 sqlite/datascript_sqlite_codec.ml       |  72 ++++--
 sqlite/dune                             |   2 +-
 test/dune                               |   2 +-
 test/test_db.ml                         |  12 +-
 test/test_storage.ml                    | 315 +----------------------
 type/datascript_types.ml                | 326 ++++++++++++++++++++++--
 type/dune                               |   2 +-
 61 files changed, 2554 insertions(+), 1007 deletions(-)
 create mode 100644 impl/index.mli
 create mode 100644 impl/platform/jsoo/index.ml
 create mode 100644 impl/platform/jsoo/storage.ml
 create mode 100644 impl/platform/melange/index.ml
 create mode 100644 impl/platform/melange/storage.ml
 create mode 100644 impl/platform/native/index.ml
 create mode 100644 impl/platform/native/storage.ml
 create mode 100644 impl/storage_lmdb_impl.ml
 rename impl/{storage.ml => storage_pss.ml} (100%)
 create mode 100644 lmdb/datascript_lmdb_codec.ml
 create mode 100644 lmdb/datascript_lmdb_codec.mli
 create mode 100644 lmdb/datascript_lmdb_db.mli
 create mode 100644 lmdb/datascript_lmdb_db_melange.ml
 create mode 100644 lmdb/datascript_lmdb_index.ml
 create mode 100644 lmdb/datascript_lmdb_index.mli
 create mode 100644 lmdb/datascript_lmdb_node.js
 create mode 100644 lmdb/datascript_storage_lmdb.ml
 create mode 100644 lmdb/melange/datascript_lmdb_db.ml
 create mode 100644 lmdb/melange/datascript_lmdb_db.mli
 create mode 100644 lmdb/melange/datascript_lmdb_index.ml
 create mode 100644 lmdb/melange/datascript_lmdb_index.mli
 create mode 100644 lmdb/melange/datascript_lmdb_node.js
 create mode 100644 lmdb/melange/datascript_storage_lmdb.ml
 create mode 100644 lmdb/melange/dune
 create mode 100644 lmdb/native/datascript_lmdb_db.ml
 create mode 100644 lmdb/native/datascript_lmdb_db.mli
 create mode 100644 lmdb/native/datascript_lmdb_index.ml
 create mode 100644 lmdb/native/datascript_lmdb_index.mli
 create mode 100644 lmdb/native/datascript_storage_lmdb.ml
 create mode 100644 lmdb/native/dune

diff --git a/datascript-ocaml-melange.opam b/datascript-ocaml-melange.opam
index 3f574a1..7d8532e 100644
--- a/datascript-ocaml-melange.opam
+++ b/datascript-ocaml-melange.opam
@@ -8,12 +8,10 @@ depends: [
   "ocaml" {>= "5.1.1"}
   "dune" {>= "3.17"}
   "datascript_ocaml" {= version}
-  "persistent_sorted_set_ocaml" {= "dev"}
   "melange"
   "melange-transit-melange" {= "0.1.0"}
 ]
 pin-depends: [
-  ["persistent_sorted_set_ocaml.dev" "git+https://github.com/logseq/persistent-sorted-set-ocaml.git#main"]
   ["melange-edn-core.0.5.0" "git+https://github.com/RCmerci/melange-edn.git#main"]
   ["melange-edn-melange.0.5.0" "git+https://github.com/RCmerci/melange-edn.git#main"]
   ["melange-transit-core.0.1.0" "git+https://github.com/RCmerci/melange-transit.git#main"]
diff --git a/datascript-ocaml-native.opam b/datascript-ocaml-native.opam
index 889fc22..3858445 100644
--- a/datascript-ocaml-native.opam
+++ b/datascript-ocaml-native.opam
@@ -8,14 +8,12 @@ depends: [
   "ocaml" {>= "5.1.1"}
   "dune" {>= "3.17"}
   "datascript_ocaml" {= version}
-  "persistent_sorted_set_ocaml" {= "dev"}
   "sqlite3"
   "lmdb"
   "melange-transit-native" {= "0.1.0"}
   "yojson"
 ]
 pin-depends: [
-  ["persistent_sorted_set_ocaml.dev" "git+https://github.com/logseq/persistent-sorted-set-ocaml.git#main"]
   ["melange-edn-core.0.5.0" "git+https://github.com/RCmerci/melange-edn.git#main"]
   ["melange-edn-native.0.5.0" "git+https://github.com/RCmerci/melange-edn.git#main"]
   ["melange-transit-core.0.1.0" "git+https://github.com/RCmerci/melange-transit.git#main"]
diff --git a/datascript_ocaml.opam b/datascript_ocaml.opam
index 72cf185..4130cd0 100644
--- a/datascript_ocaml.opam
+++ b/datascript_ocaml.opam
@@ -7,12 +7,8 @@ license: "MIT"
 depends: [
   "ocaml" {>= "5.1.1"}
   "dune" {>= "3.17"}
-  "persistent_sorted_set_ocaml" {= "dev"}
   "melange"
 ]
-pin-depends: [
-  ["persistent_sorted_set_ocaml.dev" "git+https://github.com/logseq/persistent-sorted-set-ocaml.git#main"]
-]
 build: [
   ["dune" "build" "-p" name "-j" jobs]
 ]
diff --git a/impl/conn.ml b/impl/conn.ml
index cac0400..9a68d7b 100644
--- a/impl/conn.ml
+++ b/impl/conn.ml
@@ -5,7 +5,6 @@ type t =
   ; mutable listeners : (string * (tx_report -> unit)) list
   ; mutable next_listener_id : int
   ; storage : storage option
-  ; mutable storage_tail : datom list list
   }
 
 type creation_context =
@@ -19,16 +18,10 @@ type schema_context =
   ; with_schema : db -> schema -> db
   }
 
-type restore_context =
-  { restore : storage -> db option
-  ; restore_tail_groups : storage -> datom list list
-  }
+type restore_context = { restore : storage -> db option }
 
 type transact_context =
   { store : ?storage:storage -> db -> unit
-  ; store_tail : storage -> datom list list -> unit
-  ; storage_tail_datom_count : datom list list -> int
-  ; storage_tail_compaction_threshold : int
   ; transact : tx_meta:tx_meta -> db -> tx_op list -> tx_report
   }
 
@@ -41,11 +34,7 @@ type context =
   { empty_db : ?schema:schema -> ?storage:storage -> unit -> db
   ; init_db : ?schema:schema -> ?storage:storage -> datom list -> db
   ; store : ?storage:storage -> db -> unit
-  ; store_tail : storage -> datom list list -> unit
   ; restore : storage -> db option
-  ; restore_tail_groups : storage -> datom list list
-  ; storage_tail_datom_count : datom list list -> int
-  ; storage_tail_compaction_threshold : int
   ; transact : tx_meta:tx_meta -> db -> tx_op list -> tx_report
   ; datoms : db -> datom list
   ; with_schema : db -> schema -> db
@@ -64,13 +53,13 @@ let tx_meta_without_store_control tx_meta =
       | _ -> true)
     tx_meta
 
-let make ?storage ?(storage_tail = []) db =
+let make ?storage db =
   let db =
     match storage with
     | None -> db
     | Some _ -> { db with storage_ref = storage }
   in
-  { db; listeners = []; next_listener_id = 0; storage; storage_tail }
+  { db; listeners = []; next_listener_id = 0; storage }
 
 let create (context : creation_context) ?schema ?storage () =
   let db = context.empty_db ?schema ?storage () in
@@ -122,15 +111,13 @@ let reset_schema (context : schema_context) conn schema =
   conn.db <- db;
   (match conn.storage with
    | None -> ()
-   | Some storage ->
-     context.store ~storage db;
-     conn.storage_tail <- []);
+   | Some storage -> context.store ~storage db);
   db
 
 let restore (context : restore_context) storage =
   match context.restore storage with
   | None -> None
-  | Some db -> Some (make ~storage ~storage_tail:(context.restore_tail_groups storage) db)
+  | Some db -> Some (make ~storage db)
 
 let transact (context : transact_context) ?(tx_meta = []) conn tx_data =
   let skip_store = tx_meta_skips_store tx_meta in
@@ -139,17 +126,7 @@ let transact (context : transact_context) ?(tx_meta = []) conn tx_data =
   if not skip_store then
     (match conn.storage with
      | None -> ()
-     | Some storage ->
-       if report.tx_data <> [] then begin
-         let tail = conn.storage_tail @ [ report.tx_data ] in
-         if context.storage_tail_datom_count tail > context.storage_tail_compaction_threshold then begin
-           context.store ~storage report.db_after;
-           conn.storage_tail <- []
-         end else begin
-           conn.storage_tail <- tail;
-           context.store_tail storage conn.storage_tail
-         end
-       end);
+     | Some storage -> context.store ~storage report.db_after);
   notify_listeners conn report;
   report
 
@@ -167,8 +144,6 @@ let reset (context : reset_context) ?(tx_meta = []) conn db =
   conn.db <- db;
   (match conn.storage with
    | None -> ()
-   | Some storage ->
-     context.store ~storage db;
-     conn.storage_tail <- []);
+   | Some storage -> context.store ~storage db);
   notify_listeners conn report;
   db
diff --git a/impl/conn.mli b/impl/conn.mli
index f7cda87..c8b37d7 100644
--- a/impl/conn.mli
+++ b/impl/conn.mli
@@ -13,16 +13,10 @@ type schema_context =
   ; with_schema : db -> schema -> db
   }
 
-type restore_context =
-  { restore : storage -> db option
-  ; restore_tail_groups : storage -> datom list list
-  }
+type restore_context = { restore : storage -> db option }
 
 type transact_context =
   { store : ?storage:storage -> db -> unit
-  ; store_tail : storage -> datom list list -> unit
-  ; storage_tail_datom_count : datom list list -> int
-  ; storage_tail_compaction_threshold : int
   ; transact : tx_meta:tx_meta -> db -> tx_op list -> tx_report
   }
 
@@ -35,11 +29,7 @@ type context =
   { empty_db : ?schema:schema -> ?storage:storage -> unit -> db
   ; init_db : ?schema:schema -> ?storage:storage -> datom list -> db
   ; store : ?storage:storage -> db -> unit
-  ; store_tail : storage -> datom list list -> unit
   ; restore : storage -> db option
-  ; restore_tail_groups : storage -> datom list list
-  ; storage_tail_datom_count : datom list list -> int
-  ; storage_tail_compaction_threshold : int
   ; transact : tx_meta:tx_meta -> db -> tx_op list -> tx_report
   ; datoms : db -> datom list
   ; with_schema : db -> schema -> db
diff --git a/impl/datascript.ml b/impl/datascript.ml
index 4f7976c..f0f403b 100644
--- a/impl/datascript.ml
+++ b/impl/datascript.ml
@@ -29,7 +29,7 @@ module Serialize = Serialize
 module Storage = Storage
 module Util = Util
 module Upsert = Upsert
-module PSet = Persistent_sorted_set
+module Index = Index
 
 let validate_entity_id = Db_impl.validate_entity_id
 
@@ -87,11 +87,6 @@ let store ?storage db =
   Storage.store ?storage db
 
 let memory_storage = Storage.memory_storage
-let file_storage = Storage.file_storage
-let store_tail = Storage.store_tail
-let storage_tail_compaction_threshold = Storage.tail_compaction_threshold
-let storage_tail_datom_count = Storage.tail_datom_count
-let restore_tail_groups = Storage.restore_tail_groups
 let storage_addresses = Storage.storage_addresses
 let storage = Storage.storage
 let addresses = Storage.addresses
@@ -246,7 +241,7 @@ let find_avet_exact db attr value =
     else Util.compare_datom Avet left right
   in
   match
-    PSet.slice ~from_:bound ~to_:bound ~cmp db.avet_index
+    Index.slice ~from_:bound ~to_:bound ~cmp db.avet_index
     @ List.filter
         (fun datom -> datom.a = attr && value_equal datom.v value)
         (Option.value (Hashtbl.find_opt db.duplicate_avet_by_attr attr) ~default:[])
@@ -270,7 +265,7 @@ let find_eavt_exact db entity_id attr value =
     else Util.compare_datom Eavt left right
   in
   match
-    PSet.slice ~from_:bound ~to_:bound ~cmp db.eavt_index
+    Index.slice ~from_:bound ~to_:bound ~cmp db.eavt_index
     @ List.filter
         (fun datom -> datom.e = entity_id && datom.a = attr && value_equal datom.v value)
         (Option.value (Hashtbl.find_opt db.duplicate_eavt_by_entity entity_id) ~default:[])
@@ -736,61 +731,13 @@ let db_with tx_ops db =
   let db_after, _, _ = apply_tx tx_ops db in
   db_after
 
-let apply_tail_group db group =
-  List.iter
-    (fun datom ->
-      if datom.added && is_unique db datom.a then
-        match Db_access_impl.find_datom db Avet ~a:datom.a ~v:datom.v () with
-        | Some existing when existing.e <> datom.e ->
-          invalid_arg "tail group conflicts with an existing unique value"
-        | Some _ | None -> ())
-    group;
-  let group =
-    List.fold_left
-      (fun tx_data datom ->
-        if datom.added && cardinality db datom.a = One then
-          let existing =
-            Db_access_impl.datoms db Eavt ~e:datom.e ~a:datom.a ()
-            |> Seq.filter (fun existing -> not (value_equal existing.v datom.v))
-            |> Seq.map (fun existing ->
-              { existing with tx = datom.tx; added = false })
-            |> List.of_seq
-          in
-          List.rev_append existing (datom :: tx_data)
-        else
-          datom :: tx_data)
-      []
-      group
-    |> List.rev
-  in
-  let max_eid =
-    List.fold_left
-      (fun max_eid datom ->
-        let max_eid =
-          if datom.e <= max_allocatable_entity_id then max max_eid datom.e
-          else max_eid
-        in
-        max_eid_in_value max_eid datom.v)
-      db.max_eid
-      group
-  in
-  let db = refresh_db_indexes_with_tx_data db group in
-  { db with max_eid }
-
-let storage_tail_context : Storage.tail_context =
-  { apply_group = apply_tail_group }
-
-let db_with_tail db tail =
-  Storage.db_with_tail storage_tail_context db tail
-
-let storage_restore_context : Storage.restore_context =
-  { next_db_uid; db_with_tail }
+let storage_restore_context : Storage.restore_context = { next_db_uid }
 
 let restore storage =
   Storage.restore storage_restore_context storage
 
 let restore_conn storage =
-  let context : Conn.restore_context = { restore; restore_tail_groups } in
+  let context : Conn.restore_context = { restore } in
   Conn.restore context storage
 
 let tx_meta_skips_store tx_meta =
@@ -800,16 +747,11 @@ let tx_meta_skips_store tx_meta =
       | _ -> false)
     tx_meta
 
-let persist_transact_tail ~tx_meta db tx_data =
-  if tx_data <> [] && not (tx_meta_skips_store tx_meta) then
+let persist_transact ~tx_meta db =
+  if not (tx_meta_skips_store tx_meta) then
     match db.storage_ref with
     | None -> ()
-    | Some storage ->
-      let tail = restore_tail_groups storage @ [ tx_data ] in
-      if storage_tail_datom_count tail > storage_tail_compaction_threshold then
-        store ~storage db
-      else
-        store_tail storage tail
+    | Some storage -> store ~storage db
 
 let transact_report ?(tx_meta = []) db tx_ops =
   let db_after, tempids, tx_data = apply_tx tx_ops db in
@@ -817,19 +759,14 @@ let transact_report ?(tx_meta = []) db tx_ops =
 
 let transact ?(tx_meta = []) db tx_ops =
   let report = transact_report ~tx_meta db tx_ops in
-  persist_transact_tail ~tx_meta report.db_after report.tx_data;
+  persist_transact ~tx_meta report.db_after;
   report
 
 let with_tx ?tx_meta db tx_ops = transact ?tx_meta db tx_ops
 
 let transact_conn ?(tx_meta = []) conn tx_data =
   let context : Conn.transact_context =
-    { store
-    ; store_tail
-    ; storage_tail_datom_count
-    ; storage_tail_compaction_threshold
-    ; transact = (fun ~tx_meta db tx_data -> transact_report ~tx_meta db tx_data)
-    }
+    { store; transact = (fun ~tx_meta db tx_data -> transact_report ~tx_meta db tx_data) }
   in
   Conn.transact context ~tx_meta conn tx_data
 
@@ -1163,7 +1100,7 @@ let primary_attr_datoms db index attr =
       else if left == bound then -compare_prefix right left
       else Util.compare_datom index left right
     in
-    PSet.slice ~from_:bound ~to_:bound ~cmp index_set
+    Index.slice ~from_:bound ~to_:bound ~cmp index_set
   in
   match index with
   | Aevt ->
@@ -1180,7 +1117,7 @@ let primary_attr_datoms db index attr =
        let datoms = attr_prefix_datoms Avet db.avet_index in
        Hashtbl.replace db.avet_by_attr attr datoms;
        datoms)
-  | Eavt -> PSet.to_list db.eavt_index
+  | Eavt -> Index.to_list db.eavt_index
 
 let primary_attr_datoms_seq db index ?e ~a ?v ?tx () =
   let datoms = primary_attr_datoms db index a in
diff --git a/impl/datascript.mli b/impl/datascript.mli
index fbf1e4f..245da29 100644
--- a/impl/datascript.mli
+++ b/impl/datascript.mli
@@ -88,16 +88,10 @@ module Conn : sig
     ; with_schema : db -> schema -> db
     }
 
-  type restore_context =
-    { restore : storage -> db option
-    ; restore_tail_groups : storage -> datom list list
-    }
+  type restore_context = { restore : storage -> db option }
 
   type transact_context =
     { store : ?storage:storage -> db -> unit
-    ; store_tail : storage -> datom list list -> unit
-    ; storage_tail_datom_count : datom list list -> int
-    ; storage_tail_compaction_threshold : int
     ; transact : tx_meta:tx_meta -> db -> tx_op list -> tx_report
     }
 
@@ -231,26 +225,11 @@ module Serialize : sig
 end
 
 module Storage : sig
-  type tail_context =
-    { apply_group : db -> datom list -> db
-    }
-
-  type restore_context =
-    { next_db_uid : unit -> int
-    ; db_with_tail : db -> datom list list -> db
-    }
+  type restore_context = { next_db_uid : unit -> int }
 
-  val root_address : storage_address
-  val tail_address : storage_address
   val memory_storage : unit -> storage
-  val file_storage : string -> storage
   val store : ?storage:storage -> db -> unit
-  val store_tail : storage -> datom list list -> unit
-  val tail_compaction_threshold : int
-  val tail_datom_count : datom list list -> int
   val restore_root_snapshot : storage -> serializable_db option
-  val restore_tail_groups : storage -> datom list list
-  val db_with_tail : tail_context -> db -> datom list list -> db
   val restore : restore_context -> storage -> db option
   val storage_addresses : storage -> storage_address list
   val storage : db -> storage option
@@ -404,11 +383,8 @@ val serializable : db -> serializable_db
 val from_serializable : serializable_db -> db
 val db_from_reader_string : string -> db
 val memory_storage : unit -> storage
-val file_storage : string -> storage
 val store : ?storage:storage -> db -> unit
-val store_tail : storage -> datom list list -> unit
 val restore : storage -> db option
-val db_with_tail : db -> datom list list -> db
 val storage : db -> storage option
 val addresses : db list -> storage_address list
 val settings : db -> (attr * value) list
diff --git a/impl/db.ml b/impl/db.ml
index 87579d1..c78d90c 100644
--- a/impl/db.ml
+++ b/impl/db.ml
@@ -1,6 +1,6 @@
 open Datascript_types
 
-module PSet = Persistent_sorted_set
+module Index = Index
 
 let tx0 = 0x20000000
 
@@ -62,14 +62,10 @@ let normalize_datom_for_schema schema d =
   ignore schema;
   Util.normalize_datom_value d
 
-let empty_index index =
-  PSet.empty_by ~cmp:(Util.compare_datom index) ()
+let empty_index index lmdb = Index.empty index lmdb
 
-let build_index index datoms =
-  let cmp = Util.compare_datom index in
-  let items = Array.of_list datoms in
-  Array.sort cmp items;
-  PSet.of_sorted_array_by ~cmp items
+let build_index index lmdb datoms =
+  Index.of_sorted_list index datoms lmdb
 
 let duplicate_datoms datoms =
   let datoms = List.sort (Util.compare_datom Eavt) datoms in
@@ -103,10 +99,6 @@ let duplicate_datoms_by_attr duplicate_datoms =
   Hashtbl.iter (fun attr datoms -> Hashtbl.replace table attr (List.rev datoms)) table;
   table
 
-let build_avet_index schema datoms =
-  datoms
-  |> List.filter (fun d -> Schema.schema_attr_is_avet_accessible schema d.a)
-  |> build_index Avet
 
 let datoms_by_attr datoms =
   let table = Hashtbl.create 1024 in
@@ -124,10 +116,21 @@ let invalidate_attr_tables db =
   else
     { db with aevt_by_attr = Hashtbl.create 0; avet_by_attr = Hashtbl.create 0 }
 
+let lmdb_of_db db =
+  try Index.lmdb_of (Index.db_of db.eavt_index)
+  with Invalid_argument _ ->
+    let lmdb, _ = Index.create_lmdb db.storage_ref in
+    lmdb
+
 let set_indexes_from_datoms db datoms =
-  let eavt_index = build_index Eavt datoms in
-  let aevt_index = build_index Aevt datoms in
-  let avet_index = build_avet_index db.schema datoms in
+  let lmdb = lmdb_of_db db in
+  let eavt_index = build_index Eavt lmdb datoms in
+  let aevt_index = build_index Aevt lmdb datoms in
+  let avet_index =
+    datoms
+    |> List.filter (fun d -> Schema.schema_attr_is_avet_accessible db.schema d.a)
+    |> build_index Avet lmdb
+  in
   let duplicate_datoms = duplicate_datoms datoms in
   let duplicate_aevt_datoms = List.sort (Util.compare_datom Aevt) duplicate_datoms in
   let duplicate_avet_datoms =
@@ -143,8 +146,8 @@ let set_indexes_from_datoms db datoms =
     eavt_index
   ; aevt_index
   ; avet_index
-  ; aevt_by_attr = datoms_by_attr (PSet.to_list aevt_index)
-  ; avet_by_attr = datoms_by_attr (PSet.to_list avet_index)
+  ; aevt_by_attr = datoms_by_attr (Index.to_list aevt_index)
+  ; avet_by_attr = datoms_by_attr (Index.to_list avet_index)
   ; duplicate_datoms
   ; duplicate_aevt_datoms
   ; duplicate_avet_datoms
@@ -155,7 +158,7 @@ let set_indexes_from_datoms db datoms =
   }
 
 let eavt_datoms db =
-  PSet.to_list db.eavt_index @ db.duplicate_datoms |> List.sort (Util.compare_datom Eavt)
+  Index.to_list db.eavt_index @ db.duplicate_datoms |> List.sort (Util.compare_datom Eavt)
 
 let refresh_indexes db =
   set_indexes_from_datoms db (eavt_datoms db)
@@ -163,7 +166,7 @@ let refresh_indexes db =
 let add_datoms_to_index include_datom datoms index_set =
   List.fold_left
     (fun index_set datom ->
-      if include_datom datom then PSet.add datom index_set else index_set)
+      if include_datom datom then Index.add datom index_set else index_set)
     index_set
     datoms
 
@@ -206,17 +209,17 @@ let find_active_datom_by_fact db datom =
     Option.value (Hashtbl.find_opt db.duplicate_eavt_by_entity datom.e) ~default:[]
     |> List.filter (fun active -> active.a = datom.a && value_equal active.v datom.v)
   in
-  match PSet.slice ~from_:bound ~to_:bound ~cmp db.eavt_index @ duplicate_matches with
+  match Index.slice ~from_:bound ~to_:bound ~cmp db.eavt_index @ duplicate_matches with
   | [] -> None
   | matches -> Some (matches |> List.sort (Util.compare_datom Eavt) |> List.hd)
 
 let add_datom_to_indexes db datom =
   { db with
-    eavt_index = PSet.add datom db.eavt_index
-  ; aevt_index = PSet.add datom db.aevt_index
+    eavt_index = Index.add datom db.eavt_index
+  ; aevt_index = Index.add datom db.aevt_index
   ; avet_index =
       if Schema.schema_attr_is_avet_accessible db.schema datom.a then
-        PSet.add datom db.avet_index
+        Index.add datom db.avet_index
       else
         db.avet_index
   ; max_datom_e = max db.max_datom_e datom.e
@@ -233,9 +236,9 @@ let refresh_indexes_with_tx_data db tx_data =
           | None -> db
           | Some active ->
             { db with
-              eavt_index = PSet.remove active db.eavt_index
-            ; aevt_index = PSet.remove active db.aevt_index
-            ; avet_index = PSet.remove active db.avet_index
+              eavt_index = Index.remove active db.eavt_index
+            ; aevt_index = Index.remove active db.aevt_index
+            ; avet_index = Index.remove active db.avet_index
             })
       db
       tx_data
@@ -247,11 +250,12 @@ let with_datoms db datoms =
 
 let empty_db context ?(schema = []) ?storage () =
   let schema = Schema.validate_schema schema in
+  let lmdb, storage_ref = Index.create_lmdb storage in
   { db_uid = context.next_db_uid ()
   ; schema
-  ; eavt_index = empty_index Eavt
-  ; aevt_index = empty_index Aevt
-  ; avet_index = empty_index Avet
+  ; eavt_index = empty_index Eavt lmdb
+  ; aevt_index = empty_index Aevt lmdb
+  ; avet_index = empty_index Avet lmdb
   ; aevt_by_attr = Hashtbl.create 0
   ; avet_by_attr = Hashtbl.create 0
   ; duplicate_datoms = []
@@ -264,7 +268,7 @@ let empty_db context ?(schema = []) ?storage () =
   ; max_datom_e = 0
   ; max_tx = tx0
   ; filter_pred = None
-  ; storage_ref = storage
+  ; storage_ref
   ; tx_fns = []
   }
 
@@ -277,11 +281,12 @@ let init_db context ?(schema = []) ?storage datoms =
     List.fold_left (fun max_eid d -> max_eid_in_value (max_eid_with_entity_id max_eid d.e) d.v) 0 datoms
   in
   let max_tx = List.fold_left (fun max_tx d -> max max_tx d.tx) tx0 datoms in
+  let lmdb, storage_ref = Index.create_lmdb storage in
   { db_uid = context.next_db_uid ()
   ; schema
-  ; eavt_index = empty_index Eavt
-  ; aevt_index = empty_index Aevt
-  ; avet_index = empty_index Avet
+  ; eavt_index = empty_index Eavt lmdb
+  ; aevt_index = empty_index Aevt lmdb
+  ; avet_index = empty_index Avet lmdb
   ; aevt_by_attr = Hashtbl.create 0
   ; avet_by_attr = Hashtbl.create 0
   ; duplicate_datoms = []
@@ -294,7 +299,7 @@ let init_db context ?(schema = []) ?storage datoms =
   ; max_datom_e = 0
   ; max_tx
   ; filter_pred = None
-  ; storage_ref = storage
+  ; storage_ref
   ; tx_fns = []
   }
   |> fun db -> with_datoms db datoms
@@ -398,7 +403,7 @@ let primary_attr_datoms db index attr =
       else if left == bound then -compare_prefix right left
       else Util.compare_datom index left right
     in
-    PSet.slice ~from_:bound ~to_:bound ~cmp index_set
+    Index.slice ~from_:bound ~to_:bound ~cmp index_set
   in
   match index with
   | Aevt ->
@@ -415,7 +420,7 @@ let primary_attr_datoms db index attr =
        let datoms = attr_prefix_datoms Avet db.avet_index in
        Hashtbl.replace db.avet_by_attr attr datoms;
        datoms)
-  | Eavt -> PSet.to_list db.eavt_index
+  | Eavt -> Index.to_list db.eavt_index
 
 let duplicate_prefix_datoms db index e a =
   match index, e, a with
@@ -434,7 +439,7 @@ let exact_sorted_slice cmp bound datoms =
   drop_before datoms
 
 let raw_index_datoms_list db index =
-  merge_sorted_datoms index (stored_index db index |> PSet.to_list) (duplicate_index_datoms db index)
+  merge_sorted_datoms index (stored_index db index |> Index.to_list) (duplicate_index_datoms db index)
 
 let visible_index_datoms db index =
   let datoms = raw_index_datoms_list db index in
@@ -444,14 +449,14 @@ let visible_index_datoms db index =
 
 let index_datoms_seq db index =
   match db.duplicate_datoms with
-  | [] -> stored_index db index |> PSet.seq |> PSet.to_seq
+  | [] -> stored_index db index |> Index.seq |> Index.to_seq
   | _ -> raw_index_datoms_list db index |> List.to_seq
 
 let reverse_index_datoms_seq db index =
   match db.duplicate_datoms with
-  | [] -> stored_index db index |> PSet.rslice_seq |> PSet.to_seq
+  | [] -> stored_index db index |> Index.rslice_seq |> Index.to_seq
   | _ ->
-    let indexed = stored_index db index |> PSet.rslice_seq |> PSet.to_seq in
+    let indexed = stored_index db index |> Index.rslice_seq |> Index.to_seq in
     let duplicates = duplicate_index_datoms db index |> List.rev |> List.to_seq in
     merge_sorted_datom_seqs
       (fun left right -> Util.compare_datom index right left)
@@ -643,9 +648,9 @@ let exact_prefix_datoms context db index e a v tx =
           Some (merge_sorted_datom_seqs (Util.compare_datom index) (List.to_seq indexed) (List.to_seq duplicates))
         | _ ->
           (match db.duplicate_datoms with
-           | [] -> Some (PSet.slice_seq ~from_:bound ~to_:bound ~cmp (stored_index db index) |> PSet.to_seq)
+           | [] -> Some (Index.slice_seq ~from_:bound ~to_:bound ~cmp (stored_index db index) |> Index.to_seq)
            | _ ->
-             let indexed = PSet.slice_seq ~from_:bound ~to_:bound ~cmp (stored_index db index) |> PSet.to_seq in
+             let indexed = Index.slice_seq ~from_:bound ~to_:bound ~cmp (stored_index db index) |> Index.to_seq in
              let duplicates = duplicate_prefix_datoms db index e a |> exact_sorted_slice cmp bound in
              Some (merge_sorted_datom_seqs (Util.compare_datom index) indexed (List.to_seq duplicates)))))
 
@@ -657,8 +662,8 @@ let exact_prefix_datoms_list context db index e a v tx =
     (match db.duplicate_datoms with
      | [] ->
        Some
-         (PSet.slice_seq ~from_:bound ~to_:bound ~cmp (stored_index db index)
-          |> PSet.seq_to_list)
+         (Index.slice_seq ~from_:bound ~to_:bound ~cmp (stored_index db index)
+          |> Index.seq_to_list)
      | _ ->
        exact_prefix_datoms context db index e a v tx
        |> Option.map List.of_seq)
@@ -674,7 +679,7 @@ let lower_prefix_datoms context db index e a v tx =
         primary_attr_datoms db index attr
         |> List.filter (fun datom -> cmp datom bound >= 0)
         |> List.to_seq
-      | _ -> PSet.slice_seq ~from_:bound ~cmp (stored_index db index) |> PSet.to_seq
+      | _ -> Index.slice_seq ~from_:bound ~cmp (stored_index db index) |> Index.to_seq
     in
     (match db.duplicate_datoms with
      | [] -> Some indexed
@@ -694,7 +699,7 @@ let reverse_upper_prefix_datoms context db index e a v tx =
         |> List.filter (fun datom -> cmp datom bound <= 0)
         |> List.rev
         |> List.to_seq
-      | _ -> PSet.rslice_seq ~from_:bound ~cmp (stored_index db index) |> PSet.to_seq
+      | _ -> Index.rslice_seq ~from_:bound ~cmp (stored_index db index) |> Index.to_seq
     in
     (match db.duplicate_datoms with
      | [] -> Some indexed
@@ -741,7 +746,7 @@ let avet_range_datoms context db attr start stop =
     match db.duplicate_datoms with
     | [] ->
       let cmp = slice_cmp context Avet from_bound from_fields to_bound to_fields in
-      PSet.slice_seq ~from_:from_bound ~to_:to_bound ~cmp db.avet_index |> PSet.to_seq
+      Index.slice_seq ~from_:from_bound ~to_:to_bound ~cmp db.avet_index |> Index.to_seq
     | _ ->
       primary_attr_datoms db Avet attr
       |> List.filter (fun datom -> lower_matches datom && upper_matches datom)
@@ -829,7 +834,7 @@ let fold_datoms f init context db index ?e ?a ?v ?tx () =
   match db.duplicate_datoms, exact_prefix_bound index e a prefix_v prefix_tx with
   | [], Some (bound, bound_fields) ->
     let cmp = exact_prefix_slice_cmp context index bound bound_fields in
-    let seq = PSet.slice_seq ~from_:bound ~to_:bound ~cmp (stored_index db index) in
+    let seq = Index.slice_seq ~from_:bound ~to_:bound ~cmp (stored_index db index) in
     let fold =
       match exact_attr_prefix || (e, a, v, tx) = (None, None, None, None), db.filter_pred with
       | true, None -> f
@@ -837,12 +842,12 @@ let fold_datoms f init context db index ?e ?a ?v ?tx () =
       | false, None -> fold_filter
       | false, Some _ -> fold_filter_and_pred
     in
-    PSet.fold_seq fold init seq
+    Index.fold_seq fold init seq
   | [], None when (e, a, v, tx) = (None, None, None, None) ->
     (match db.filter_pred with
-     | None -> PSet.fold f init (stored_index db index)
+     | None -> Index.fold f init (stored_index db index)
      | Some pred ->
-       PSet.fold (fun acc datom -> if pred datom then f acc datom else acc) init (stored_index db index))
+       Index.fold (fun acc datom -> if pred datom then f acc datom else acc) init (stored_index db index))
   | _ ->
     datoms context db index ?e ?a ?v ?tx () |> Seq.fold_left f init
 
diff --git a/impl/dune b/impl/dune
index e0bf38a..e1e6971 100644
--- a/impl/dune
+++ b/impl/dune
@@ -1,6 +1,7 @@
 (library
  (name datascript)
  (public_name datascript_ocaml)
- (virtual_modules platform)
+ (virtual_modules platform index storage)
  (modes native byte melange)
- (libraries datascript_types persistent_sorted_set_ocaml))
+ (modules (:standard \ storage_pss storage_lmdb_impl))
+ (libraries datascript_types))
diff --git a/impl/index.mli b/impl/index.mli
new file mode 100644
index 0000000..8550923
--- /dev/null
+++ b/impl/index.mli
@@ -0,0 +1,24 @@
+open Datascript_types
+
+type t = index_set
+type 'a seq
+type lmdb
+
+val create_lmdb : storage option -> lmdb * storage option
+val lmdb_of : lmdb -> lmdb
+val db_of : t -> lmdb
+
+val empty : index -> lmdb -> t
+val of_sorted_list : index -> datom list -> lmdb -> t
+val add : datom -> t -> t
+val remove : datom -> t -> t
+val to_list : t -> datom list
+val fold : ('acc -> datom -> 'acc) -> 'acc -> t -> 'acc
+val slice : ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom list
+val slice_seq : ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom seq
+val rslice_seq : ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom seq
+val seq : t -> datom seq
+val seq_to_list : datom seq -> datom list
+val fold_seq : ('acc -> datom -> 'acc) -> 'acc -> datom seq -> 'acc
+val to_seq : datom seq -> datom Seq.t
+val seek : datom -> datom seq -> datom seq
diff --git a/impl/platform.mli b/impl/platform.mli
index a639eeb..1d7071a 100644
--- a/impl/platform.mli
+++ b/impl/platform.mli
@@ -1,13 +1,8 @@
 type regex
 
-open Datascript_types
-
 (** Return the current wall-clock time as seconds since the Unix epoch. *)
 val now_seconds : unit -> float
 
-(** Create a file-backed storage instance rooted at the given path. *)
-val file_storage : string -> storage
-
 (** Compile a platform-specific regular expression from a pattern string. *)
 val compile_regex : string -> regex
 
diff --git a/impl/platform/jsoo/dune b/impl/platform/jsoo/dune
index 3909a7a..efd469d 100644
--- a/impl/platform/jsoo/dune
+++ b/impl/platform/jsoo/dune
@@ -3,4 +3,8 @@
  (public_name datascript-ocaml-jsoo)
  (implements datascript)
  (modes byte)
- (libraries js_of_ocaml persistent_sorted_set_ocaml.native))
+ (libraries
+  js_of_ocaml
+  lmdb_db_native
+  lmdb_index_native
+  storage_lmdb_native))
diff --git a/impl/platform/jsoo/index.ml b/impl/platform/jsoo/index.ml
new file mode 100644
index 0000000..1749a76
--- /dev/null
+++ b/impl/platform/jsoo/index.ml
@@ -0,0 +1,36 @@
+open Datascript_types
+
+(* Native LMDB indexes use identity coercions because [index_set] stays abstract in
+   [Datascript_types] while this module owns the concrete LMDB representation. *)
+external inject : Datascript_lmdb_index.t -> index_set = "%identity"
+external project : index_set -> Datascript_lmdb_index.t = "%identity"
+
+type t = index_set
+type 'a seq = 'a Datascript_lmdb_index.seq
+type lmdb = Datascript_lmdb_db.t
+
+let create_lmdb storage =
+  match storage with
+  | Some storage -> (Datascript_storage_lmdb.lmdb storage, Some storage)
+  | None ->
+      let lmdb = Datascript_lmdb_db.create_temp () in
+      (lmdb, Some (Datascript_storage_lmdb.wrap lmdb))
+
+let lmdb_of lmdb = lmdb
+let db_of t = Datascript_lmdb_index.db_of (project t)
+
+let empty index lmdb = Datascript_lmdb_index.empty index lmdb |> inject
+let of_sorted_list index datoms lmdb = Datascript_lmdb_index.of_sorted_list index datoms lmdb |> inject
+
+let add datom t = Datascript_lmdb_index.add datom (project t) |> inject
+let remove datom t = Datascript_lmdb_index.remove datom (project t) |> inject
+let to_list t = Datascript_lmdb_index.to_list (project t)
+let fold f init t = Datascript_lmdb_index.fold f init (project t)
+let slice ?from_ ?to_ ?cmp t = Datascript_lmdb_index.slice ?from_ ?to_ ?cmp (project t)
+let slice_seq ?from_ ?to_ ?cmp t = Datascript_lmdb_index.slice_seq ?from_ ?to_ ?cmp (project t)
+let rslice_seq ?from_ ?to_ ?cmp t = Datascript_lmdb_index.rslice_seq ?from_ ?to_ ?cmp (project t)
+let seq t = Datascript_lmdb_index.seq (project t)
+let seq_to_list = Datascript_lmdb_index.seq_to_list
+let fold_seq = Datascript_lmdb_index.fold_seq
+let to_seq = Datascript_lmdb_index.to_seq
+let seek = Datascript_lmdb_index.seek
diff --git a/impl/platform/jsoo/platform.ml b/impl/platform/jsoo/platform.ml
index 0642250..d9843b5 100644
--- a/impl/platform/jsoo/platform.ml
+++ b/impl/platform/jsoo/platform.ml
@@ -6,9 +6,6 @@ let now_seconds () =
   let now_ms = Js.Unsafe.meth_call Js.date "now" [||] in
   Js.to_float now_ms /. 1000.0
 
-let file_storage _dir =
-  invalid_arg "file_storage is not supported on js_of_ocaml"
-
 let compile_regex = Regexp.regexp
 
 let replace_regex ~first_only regex value replacement =
diff --git a/impl/platform/jsoo/storage.ml b/impl/platform/jsoo/storage.ml
new file mode 100644
index 0000000..78a6ec0
--- /dev/null
+++ b/impl/platform/jsoo/storage.ml
@@ -0,0 +1,105 @@
+open Datascript_types
+
+module Index = Index
+
+type restore_context = { next_db_uid : unit -> int }
+
+let memory_storage = Datascript_storage_lmdb.memory_storage
+
+let index_lmdb storage =
+  let lmdb, _ = Index.create_lmdb (Some storage) in
+  lmdb
+
+let store ?storage db =
+  match storage, db.storage_ref with
+  | Some storage, _ | None, Some storage ->
+      let lmdb = Datascript_storage_lmdb.lmdb storage in
+      Datascript_storage_lmdb.store_meta lmdb db
+  | None, None -> invalid_arg "db has no attached storage"
+
+let restore_root_snapshot storage =
+  let schema, max_eid, max_tx, duplicate_datoms =
+    Datascript_storage_lmdb.restore_meta (Datascript_storage_lmdb.lmdb storage)
+  in
+  let index_lmdb = index_lmdb storage in
+  Some
+    { serializable_schema = schema
+    ; serializable_datoms = Index.to_list (Index.empty Eavt index_lmdb) @ duplicate_datoms
+    ; serializable_max_eid = max_eid
+    ; serializable_max_tx = max_tx
+    }
+
+let restore context storage =
+  let schema, max_eid, max_tx, duplicate_datoms =
+    Datascript_storage_lmdb.restore_meta (Datascript_storage_lmdb.lmdb storage)
+  in
+  let schema = Schema.validate_schema schema in
+  let index_lmdb = index_lmdb storage in
+  let duplicate_eavt_by_entity =
+    let table = Hashtbl.create 1024 in
+    List.iter
+      (fun datom ->
+        let existing = Option.value (Hashtbl.find_opt table datom.e) ~default:[] in
+        Hashtbl.replace table datom.e (datom :: existing))
+      duplicate_datoms;
+    Hashtbl.iter (fun entity_id datoms -> Hashtbl.replace table entity_id (List.rev datoms)) table;
+    table
+  in
+  let duplicate_datoms_by_attr duplicate_datoms =
+    let table = Hashtbl.create 1024 in
+    List.iter
+      (fun datom ->
+        let existing = Option.value (Hashtbl.find_opt table datom.a) ~default:[] in
+        Hashtbl.replace table datom.a (datom :: existing))
+      duplicate_datoms;
+    Hashtbl.iter (fun attr datoms -> Hashtbl.replace table attr (List.rev datoms)) table;
+    table
+  in
+  let duplicate_aevt_datoms = List.sort (Util.compare_datom Aevt) duplicate_datoms in
+  let duplicate_avet_datoms =
+    duplicate_datoms
+    |> List.filter (fun datom -> Schema.schema_attr_is_avet_accessible schema datom.a)
+    |> List.sort (Util.compare_datom Avet)
+  in
+  Some
+    { db_uid = context.next_db_uid ()
+    ; schema
+    ; eavt_index = Index.empty Eavt index_lmdb
+    ; aevt_index = Index.empty Aevt index_lmdb
+    ; avet_index = Index.empty Avet index_lmdb
+    ; aevt_by_attr = Hashtbl.create 0
+    ; avet_by_attr = Hashtbl.create 0
+    ; duplicate_datoms
+    ; duplicate_aevt_datoms
+    ; duplicate_avet_datoms
+    ; duplicate_eavt_by_entity
+    ; duplicate_aevt_by_attr = duplicate_datoms_by_attr duplicate_aevt_datoms
+    ; duplicate_avet_by_attr = duplicate_datoms_by_attr duplicate_avet_datoms
+    ; max_eid
+    ; max_datom_e = max_eid
+    ; max_tx
+    ; filter_pred = None
+    ; storage_ref = Some storage
+    ; tx_fns = []
+    }
+
+let storage_addresses storage = storage.storage_list_addresses ()
+let storage (db : db) = db.storage_ref
+
+let storage_root_addresses storage = storage.storage_list_addresses ()
+
+let addresses dbs =
+  dbs
+  |> List.concat_map (fun db ->
+    match db.storage_ref with
+    | None -> []
+    | Some storage -> storage_root_addresses storage)
+  |> List.sort_uniq compare
+
+let settings (_db : db) =
+  [ "branching-factor", Int 32
+  ; "ref-type", Keyword "weak"
+  ; "storage", Bool (Option.is_some _db.storage_ref)
+  ]
+
+let collect_garbage _storage = ()
diff --git a/impl/platform/melange/dune b/impl/platform/melange/dune
index 9a91be2..917d3df 100644
--- a/impl/platform/melange/dune
+++ b/impl/platform/melange/dune
@@ -3,6 +3,10 @@
  (public_name datascript-ocaml-melange)
  (implements datascript)
  (modes melange)
- (libraries melange.js persistent_sorted_set_ocaml.melange)
+ (libraries
+  melange.js
+  lmdb_db_melange
+  lmdb_index_melange
+  storage_lmdb_melange)
  (preprocess
   (pps melange.ppx)))
diff --git a/impl/platform/melange/index.ml b/impl/platform/melange/index.ml
new file mode 100644
index 0000000..1749a76
--- /dev/null
+++ b/impl/platform/melange/index.ml
@@ -0,0 +1,36 @@
+open Datascript_types
+
+(* Native LMDB indexes use identity coercions because [index_set] stays abstract in
+   [Datascript_types] while this module owns the concrete LMDB representation. *)
+external inject : Datascript_lmdb_index.t -> index_set = "%identity"
+external project : index_set -> Datascript_lmdb_index.t = "%identity"
+
+type t = index_set
+type 'a seq = 'a Datascript_lmdb_index.seq
+type lmdb = Datascript_lmdb_db.t
+
+let create_lmdb storage =
+  match storage with
+  | Some storage -> (Datascript_storage_lmdb.lmdb storage, Some storage)
+  | None ->
+      let lmdb = Datascript_lmdb_db.create_temp () in
+      (lmdb, Some (Datascript_storage_lmdb.wrap lmdb))
+
+let lmdb_of lmdb = lmdb
+let db_of t = Datascript_lmdb_index.db_of (project t)
+
+let empty index lmdb = Datascript_lmdb_index.empty index lmdb |> inject
+let of_sorted_list index datoms lmdb = Datascript_lmdb_index.of_sorted_list index datoms lmdb |> inject
+
+let add datom t = Datascript_lmdb_index.add datom (project t) |> inject
+let remove datom t = Datascript_lmdb_index.remove datom (project t) |> inject
+let to_list t = Datascript_lmdb_index.to_list (project t)
+let fold f init t = Datascript_lmdb_index.fold f init (project t)
+let slice ?from_ ?to_ ?cmp t = Datascript_lmdb_index.slice ?from_ ?to_ ?cmp (project t)
+let slice_seq ?from_ ?to_ ?cmp t = Datascript_lmdb_index.slice_seq ?from_ ?to_ ?cmp (project t)
+let rslice_seq ?from_ ?to_ ?cmp t = Datascript_lmdb_index.rslice_seq ?from_ ?to_ ?cmp (project t)
+let seq t = Datascript_lmdb_index.seq (project t)
+let seq_to_list = Datascript_lmdb_index.seq_to_list
+let fold_seq = Datascript_lmdb_index.fold_seq
+let to_seq = Datascript_lmdb_index.to_seq
+let seek = Datascript_lmdb_index.seek
diff --git a/impl/platform/melange/platform.ml b/impl/platform/melange/platform.ml
index 5d9e848..78aee1c 100644
--- a/impl/platform/melange/platform.ml
+++ b/impl/platform/melange/platform.ml
@@ -6,9 +6,6 @@ external set_last_index : Js.Re.t -> int -> unit = "lastIndex" [@@mel.set]
 
 let now_seconds () = date_now () /. 1000.0
 
-let file_storage _dir =
-  invalid_arg "file_storage is not supported on Melange"
-
 let compile_regex pattern = pattern
 
 let regexp ?(global = false) pattern =
diff --git a/impl/platform/melange/storage.ml b/impl/platform/melange/storage.ml
new file mode 100644
index 0000000..78a6ec0
--- /dev/null
+++ b/impl/platform/melange/storage.ml
@@ -0,0 +1,105 @@
+open Datascript_types
+
+module Index = Index
+
+type restore_context = { next_db_uid : unit -> int }
+
+let memory_storage = Datascript_storage_lmdb.memory_storage
+
+let index_lmdb storage =
+  let lmdb, _ = Index.create_lmdb (Some storage) in
+  lmdb
+
+let store ?storage db =
+  match storage, db.storage_ref with
+  | Some storage, _ | None, Some storage ->
+      let lmdb = Datascript_storage_lmdb.lmdb storage in
+      Datascript_storage_lmdb.store_meta lmdb db
+  | None, None -> invalid_arg "db has no attached storage"
+
+let restore_root_snapshot storage =
+  let schema, max_eid, max_tx, duplicate_datoms =
+    Datascript_storage_lmdb.restore_meta (Datascript_storage_lmdb.lmdb storage)
+  in
+  let index_lmdb = index_lmdb storage in
+  Some
+    { serializable_schema = schema
+    ; serializable_datoms = Index.to_list (Index.empty Eavt index_lmdb) @ duplicate_datoms
+    ; serializable_max_eid = max_eid
+    ; serializable_max_tx = max_tx
+    }
+
+let restore context storage =
+  let schema, max_eid, max_tx, duplicate_datoms =
+    Datascript_storage_lmdb.restore_meta (Datascript_storage_lmdb.lmdb storage)
+  in
+  let schema = Schema.validate_schema schema in
+  let index_lmdb = index_lmdb storage in
+  let duplicate_eavt_by_entity =
+    let table = Hashtbl.create 1024 in
+    List.iter
+      (fun datom ->
+        let existing = Option.value (Hashtbl.find_opt table datom.e) ~default:[] in
+        Hashtbl.replace table datom.e (datom :: existing))
+      duplicate_datoms;
+    Hashtbl.iter (fun entity_id datoms -> Hashtbl.replace table entity_id (List.rev datoms)) table;
+    table
+  in
+  let duplicate_datoms_by_attr duplicate_datoms =
+    let table = Hashtbl.create 1024 in
+    List.iter
+      (fun datom ->
+        let existing = Option.value (Hashtbl.find_opt table datom.a) ~default:[] in
+        Hashtbl.replace table datom.a (datom :: existing))
+      duplicate_datoms;
+    Hashtbl.iter (fun attr datoms -> Hashtbl.replace table attr (List.rev datoms)) table;
+    table
+  in
+  let duplicate_aevt_datoms = List.sort (Util.compare_datom Aevt) duplicate_datoms in
+  let duplicate_avet_datoms =
+    duplicate_datoms
+    |> List.filter (fun datom -> Schema.schema_attr_is_avet_accessible schema datom.a)
+    |> List.sort (Util.compare_datom Avet)
+  in
+  Some
+    { db_uid = context.next_db_uid ()
+    ; schema
+    ; eavt_index = Index.empty Eavt index_lmdb
+    ; aevt_index = Index.empty Aevt index_lmdb
+    ; avet_index = Index.empty Avet index_lmdb
+    ; aevt_by_attr = Hashtbl.create 0
+    ; avet_by_attr = Hashtbl.create 0
+    ; duplicate_datoms
+    ; duplicate_aevt_datoms
+    ; duplicate_avet_datoms
+    ; duplicate_eavt_by_entity
+    ; duplicate_aevt_by_attr = duplicate_datoms_by_attr duplicate_aevt_datoms
+    ; duplicate_avet_by_attr = duplicate_datoms_by_attr duplicate_avet_datoms
+    ; max_eid
+    ; max_datom_e = max_eid
+    ; max_tx
+    ; filter_pred = None
+    ; storage_ref = Some storage
+    ; tx_fns = []
+    }
+
+let storage_addresses storage = storage.storage_list_addresses ()
+let storage (db : db) = db.storage_ref
+
+let storage_root_addresses storage = storage.storage_list_addresses ()
+
+let addresses dbs =
+  dbs
+  |> List.concat_map (fun db ->
+    match db.storage_ref with
+    | None -> []
+    | Some storage -> storage_root_addresses storage)
+  |> List.sort_uniq compare
+
+let settings (_db : db) =
+  [ "branching-factor", Int 32
+  ; "ref-type", Keyword "weak"
+  ; "storage", Bool (Option.is_some _db.storage_ref)
+  ]
+
+let collect_garbage _storage = ()
diff --git a/impl/platform/native/dune b/impl/platform/native/dune
index e6f1550..ea3584c 100644
--- a/impl/platform/native/dune
+++ b/impl/platform/native/dune
@@ -3,4 +3,9 @@
  (public_name datascript-ocaml-native)
  (implements datascript)
  (modes native byte)
- (libraries str unix persistent_sorted_set_ocaml.native))
+ (libraries
+  str
+  unix
+  lmdb_db_native
+  lmdb_index_native
+  storage_lmdb_native))
diff --git a/impl/platform/native/index.ml b/impl/platform/native/index.ml
new file mode 100644
index 0000000..1749a76
--- /dev/null
+++ b/impl/platform/native/index.ml
@@ -0,0 +1,36 @@
+open Datascript_types
+
+(* Native LMDB indexes use identity coercions because [index_set] stays abstract in
+   [Datascript_types] while this module owns the concrete LMDB representation. *)
+external inject : Datascript_lmdb_index.t -> index_set = "%identity"
+external project : index_set -> Datascript_lmdb_index.t = "%identity"
+
+type t = index_set
+type 'a seq = 'a Datascript_lmdb_index.seq
+type lmdb = Datascript_lmdb_db.t
+
+let create_lmdb storage =
+  match storage with
+  | Some storage -> (Datascript_storage_lmdb.lmdb storage, Some storage)
+  | None ->
+      let lmdb = Datascript_lmdb_db.create_temp () in
+      (lmdb, Some (Datascript_storage_lmdb.wrap lmdb))
+
+let lmdb_of lmdb = lmdb
+let db_of t = Datascript_lmdb_index.db_of (project t)
+
+let empty index lmdb = Datascript_lmdb_index.empty index lmdb |> inject
+let of_sorted_list index datoms lmdb = Datascript_lmdb_index.of_sorted_list index datoms lmdb |> inject
+
+let add datom t = Datascript_lmdb_index.add datom (project t) |> inject
+let remove datom t = Datascript_lmdb_index.remove datom (project t) |> inject
+let to_list t = Datascript_lmdb_index.to_list (project t)
+let fold f init t = Datascript_lmdb_index.fold f init (project t)
+let slice ?from_ ?to_ ?cmp t = Datascript_lmdb_index.slice ?from_ ?to_ ?cmp (project t)
+let slice_seq ?from_ ?to_ ?cmp t = Datascript_lmdb_index.slice_seq ?from_ ?to_ ?cmp (project t)
+let rslice_seq ?from_ ?to_ ?cmp t = Datascript_lmdb_index.rslice_seq ?from_ ?to_ ?cmp (project t)
+let seq t = Datascript_lmdb_index.seq (project t)
+let seq_to_list = Datascript_lmdb_index.seq_to_list
+let fold_seq = Datascript_lmdb_index.fold_seq
+let to_seq = Datascript_lmdb_index.to_seq
+let seek = Datascript_lmdb_index.seek
diff --git a/impl/platform/native/platform.ml b/impl/platform/native/platform.ml
index b620154..1e126c5 100644
--- a/impl/platform/native/platform.ml
+++ b/impl/platform/native/platform.ml
@@ -1,88 +1,7 @@
 type regex = Str.regexp
 
-open Datascript_types
-
 let now_seconds = Unix.gettimeofday
 
-let ensure_storage_dir dir =
-  if Sys.file_exists dir then begin
-    if not (Sys.is_directory dir) then
-      invalid_arg ("storage path is not a directory: " ^ dir)
-  end
-  else Sys.mkdir dir 0o755
-
-let hex_digit value =
-  Char.chr (if value < 10 then Char.code '0' + value else Char.code 'a' + value - 10)
-
-let hex_value = function
-  | '0' .. '9' as ch -> Char.code ch - Char.code '0'
-  | 'a' .. 'f' as ch -> Char.code ch - Char.code 'a' + 10
-  | 'A' .. 'F' as ch -> Char.code ch - Char.code 'A' + 10
-  | ch -> invalid_arg ("invalid storage address hex digit: " ^ String.make 1 ch)
-
-let encode_storage_address address =
-  String.init
-    (String.length address * 2)
-    (fun index ->
-      let code = Char.code address.[index / 2] in
-      if index mod 2 = 0 then hex_digit (code lsr 4) else hex_digit (code land 0x0f))
-
-let decode_storage_address encoded =
-  if String.length encoded mod 2 <> 0 then
-    invalid_arg ("invalid storage address filename: " ^ encoded);
-  String.init
-    (String.length encoded / 2)
-    (fun index ->
-      let high = hex_value encoded.[index * 2] in
-      let low = hex_value encoded.[index * 2 + 1] in
-      Char.chr ((high lsl 4) lor low))
-
-let storage_payload_path dir address =
-  Filename.concat dir (encode_storage_address address ^ ".bin")
-
-let file_storage dir =
-  ensure_storage_dir dir;
-  let write_payload address payload =
-    let channel = open_out_bin (storage_payload_path dir address) in
-    Fun.protect
-      ~finally:(fun () -> close_out_noerr channel)
-      (fun () -> Marshal.to_channel channel payload [])
-  in
-  let read_payload address =
-    let path = storage_payload_path dir address in
-    if not (Sys.file_exists path) then None
-    else
-      let channel = open_in_bin path in
-      Fun.protect
-        ~finally:(fun () -> close_in_noerr channel)
-        (fun () -> Some (Marshal.from_channel channel : storage_payload))
-  in
-  let list_addresses () =
-    Sys.readdir dir
-    |> Array.to_list
-    |> List.filter_map (fun filename ->
-      if Filename.extension filename = ".bin" then
-        let base = Filename.remove_extension filename in
-        Some (decode_storage_address base)
-      else
-        None)
-    |> List.sort_uniq compare
-  in
-  let delete addresses =
-    List.iter
-      (fun address ->
-        let path = storage_payload_path dir address in
-        if Sys.file_exists path then Sys.remove path)
-      addresses
-  in
-  { storage_store =
-      (fun entries ->
-        List.iter (fun (address, payload) -> write_payload address payload) entries)
-  ; storage_restore = read_payload
-  ; storage_list_addresses = list_addresses
-  ; storage_delete = delete
-  }
-
 let str_pattern_of_pattern pattern =
   let buffer = Buffer.create (String.length pattern) in
   let add_escaped = function
diff --git a/impl/platform/native/storage.ml b/impl/platform/native/storage.ml
new file mode 100644
index 0000000..78a6ec0
--- /dev/null
+++ b/impl/platform/native/storage.ml
@@ -0,0 +1,105 @@
+open Datascript_types
+
+module Index = Index
+
+type restore_context = { next_db_uid : unit -> int }
+
+let memory_storage = Datascript_storage_lmdb.memory_storage
+
+let index_lmdb storage =
+  let lmdb, _ = Index.create_lmdb (Some storage) in
+  lmdb
+
+let store ?storage db =
+  match storage, db.storage_ref with
+  | Some storage, _ | None, Some storage ->
+      let lmdb = Datascript_storage_lmdb.lmdb storage in
+      Datascript_storage_lmdb.store_meta lmdb db
+  | None, None -> invalid_arg "db has no attached storage"
+
+let restore_root_snapshot storage =
+  let schema, max_eid, max_tx, duplicate_datoms =
+    Datascript_storage_lmdb.restore_meta (Datascript_storage_lmdb.lmdb storage)
+  in
+  let index_lmdb = index_lmdb storage in
+  Some
+    { serializable_schema = schema
+    ; serializable_datoms = Index.to_list (Index.empty Eavt index_lmdb) @ duplicate_datoms
+    ; serializable_max_eid = max_eid
+    ; serializable_max_tx = max_tx
+    }
+
+let restore context storage =
+  let schema, max_eid, max_tx, duplicate_datoms =
+    Datascript_storage_lmdb.restore_meta (Datascript_storage_lmdb.lmdb storage)
+  in
+  let schema = Schema.validate_schema schema in
+  let index_lmdb = index_lmdb storage in
+  let duplicate_eavt_by_entity =
+    let table = Hashtbl.create 1024 in
+    List.iter
+      (fun datom ->
+        let existing = Option.value (Hashtbl.find_opt table datom.e) ~default:[] in
+        Hashtbl.replace table datom.e (datom :: existing))
+      duplicate_datoms;
+    Hashtbl.iter (fun entity_id datoms -> Hashtbl.replace table entity_id (List.rev datoms)) table;
+    table
+  in
+  let duplicate_datoms_by_attr duplicate_datoms =
+    let table = Hashtbl.create 1024 in
+    List.iter
+      (fun datom ->
+        let existing = Option.value (Hashtbl.find_opt table datom.a) ~default:[] in
+        Hashtbl.replace table datom.a (datom :: existing))
+      duplicate_datoms;
+    Hashtbl.iter (fun attr datoms -> Hashtbl.replace table attr (List.rev datoms)) table;
+    table
+  in
+  let duplicate_aevt_datoms = List.sort (Util.compare_datom Aevt) duplicate_datoms in
+  let duplicate_avet_datoms =
+    duplicate_datoms
+    |> List.filter (fun datom -> Schema.schema_attr_is_avet_accessible schema datom.a)
+    |> List.sort (Util.compare_datom Avet)
+  in
+  Some
+    { db_uid = context.next_db_uid ()
+    ; schema
+    ; eavt_index = Index.empty Eavt index_lmdb
+    ; aevt_index = Index.empty Aevt index_lmdb
+    ; avet_index = Index.empty Avet index_lmdb
+    ; aevt_by_attr = Hashtbl.create 0
+    ; avet_by_attr = Hashtbl.create 0
+    ; duplicate_datoms
+    ; duplicate_aevt_datoms
+    ; duplicate_avet_datoms
+    ; duplicate_eavt_by_entity
+    ; duplicate_aevt_by_attr = duplicate_datoms_by_attr duplicate_aevt_datoms
+    ; duplicate_avet_by_attr = duplicate_datoms_by_attr duplicate_avet_datoms
+    ; max_eid
+    ; max_datom_e = max_eid
+    ; max_tx
+    ; filter_pred = None
+    ; storage_ref = Some storage
+    ; tx_fns = []
+    }
+
+let storage_addresses storage = storage.storage_list_addresses ()
+let storage (db : db) = db.storage_ref
+
+let storage_root_addresses storage = storage.storage_list_addresses ()
+
+let addresses dbs =
+  dbs
+  |> List.concat_map (fun db ->
+    match db.storage_ref with
+    | None -> []
+    | Some storage -> storage_root_addresses storage)
+  |> List.sort_uniq compare
+
+let settings (_db : db) =
+  [ "branching-factor", Int 32
+  ; "ref-type", Keyword "weak"
+  ; "storage", Bool (Option.is_some _db.storage_ref)
+  ]
+
+let collect_garbage _storage = ()
diff --git a/impl/serialize.ml b/impl/serialize.ml
index 06dd4fc..d811c6a 100644
--- a/impl/serialize.ml
+++ b/impl/serialize.ml
@@ -1,6 +1,7 @@
 open Datascript_types
 
-module PSet = Persistent_sorted_set
+module Index = Index
+module Schema = Schema
 
 type context =
   { next_db_uid : unit -> int
@@ -12,39 +13,30 @@ type context =
 let serializable db =
   { serializable_schema = db.schema
   ; serializable_datoms =
-      PSet.to_list db.eavt_index @ db.duplicate_datoms |> List.sort (Util.compare_datom Eavt)
+      Index.to_list db.eavt_index @ db.duplicate_datoms |> List.sort (Datascript_types.Compare.compare_datom Eavt)
   ; serializable_max_eid = db.max_eid
   ; serializable_max_tx = db.max_tx
   }
 
-let empty_index index =
-  PSet.empty_by ~cmp:(Util.compare_datom index) ()
-
-let index_from_datoms index datoms =
-  let cmp = Util.compare_datom index in
-  let items = Array.of_list datoms in
-  Array.sort cmp items;
-  PSet.of_sorted_array_by ~cmp items
-
 let duplicate_datoms datoms =
-  let datoms = List.sort (Util.compare_datom Eavt) datoms in
+  let datoms = List.sort (Datascript_types.Compare.compare_datom Eavt) datoms in
   let rec loop previous duplicates = function
     | [] -> List.rev duplicates
     | datom :: rest ->
       (match previous with
-       | Some previous when Util.compare_datom Eavt previous datom = 0 ->
+       | Some previous when Datascript_types.Compare.compare_datom Eavt previous datom = 0 ->
          loop (Some datom) (datom :: duplicates) rest
        | _ -> loop (Some datom) duplicates rest)
   in
   loop None [] datoms
 
 let duplicate_aevt_datoms duplicate_datoms =
-  List.sort (Util.compare_datom Aevt) duplicate_datoms
+  List.sort (Datascript_types.Compare.compare_datom Aevt) duplicate_datoms
 
 let duplicate_avet_datoms schema duplicate_datoms =
   duplicate_datoms
   |> List.filter (fun datom -> Schema.schema_attr_is_avet_accessible schema datom.a)
-  |> List.sort (Util.compare_datom Avet)
+  |> List.sort (Datascript_types.Compare.compare_datom Avet)
 
 let duplicate_eavt_by_entity duplicate_datoms =
   let table = Hashtbl.create 1024 in
@@ -72,11 +64,12 @@ let from_serializable context snapshot =
   let duplicate_datoms = duplicate_datoms datoms in
   let duplicate_aevt_datoms = duplicate_aevt_datoms duplicate_datoms in
   let duplicate_avet_datoms = duplicate_avet_datoms schema duplicate_datoms in
+  let lmdb, storage_ref = Index.create_lmdb None in
   { db_uid = context.next_db_uid ()
   ; schema
-  ; eavt_index = index_from_datoms Eavt datoms
-  ; aevt_index = empty_index Aevt
-  ; avet_index = empty_index Avet
+  ; eavt_index = Index.empty Eavt lmdb
+  ; aevt_index = Index.empty Aevt lmdb
+  ; avet_index = Index.empty Avet lmdb
   ; aevt_by_attr = Hashtbl.create 0
   ; avet_by_attr = Hashtbl.create 0
   ; duplicate_datoms
@@ -89,7 +82,7 @@ let from_serializable context snapshot =
   ; max_datom_e = 0
   ; max_tx = snapshot.serializable_max_tx
   ; filter_pred = None
-  ; storage_ref = None
+  ; storage_ref
   ; tx_fns = []
   }
   |> context.refresh_db_indexes
diff --git a/impl/storage.mli b/impl/storage.mli
index f9020f7..0bc203b 100644
--- a/impl/storage.mli
+++ b/impl/storage.mli
@@ -1,25 +1,10 @@
 open Datascript_types
 
-type tail_context =
-  { apply_group : db -> datom list -> db
-  }
+type restore_context = { next_db_uid : unit -> int }
 
-type restore_context =
-  { next_db_uid : unit -> int
-  ; db_with_tail : db -> datom list list -> db
-  }
-
-val root_address : storage_address
-val tail_address : storage_address
 val memory_storage : unit -> storage
-val file_storage : string -> storage
 val store : ?storage:storage -> db -> unit
-val store_tail : storage -> datom list list -> unit
-val tail_compaction_threshold : int
-val tail_datom_count : datom list list -> int
 val restore_root_snapshot : storage -> serializable_db option
-val restore_tail_groups : storage -> datom list list
-val db_with_tail : tail_context -> db -> datom list list -> db
 val restore : restore_context -> storage -> db option
 val storage_addresses : storage -> storage_address list
 val storage : db -> storage option
diff --git a/impl/storage_lmdb_impl.ml b/impl/storage_lmdb_impl.ml
new file mode 100644
index 0000000..208aee3
--- /dev/null
+++ b/impl/storage_lmdb_impl.ml
@@ -0,0 +1,122 @@
+open Datascript_types
+
+module Index = Index
+
+type tail_context =
+  { apply_group : db -> datom list -> db
+  }
+
+type restore_context = { next_db_uid : unit -> int }
+
+let memory_storage = Datascript_storage_lmdb.memory_storage
+let file_storage = Datascript_storage_lmdb.file_storage
+
+let store ?storage db =
+  match storage, db.storage_ref with
+  | Some storage, _ | None, Some storage ->
+      let lmdb = Datascript_storage_lmdb.lmdb storage in
+      Datascript_storage_lmdb.store_meta lmdb db
+  | None, None -> invalid_arg "db has no attached storage"
+
+let store_tail _storage _tail = ()
+
+let tail_compaction_threshold = 0
+let tail_datom_count _tail = 0
+
+let restore_tail_groups _storage = []
+
+let restore_root_snapshot storage =
+  let lmdb = Datascript_storage_lmdb.lmdb storage in
+  let schema, max_eid, max_tx, duplicate_datoms =
+    Datascript_storage_lmdb.restore_meta lmdb
+  in
+  Some
+    { serializable_schema = schema
+    ; serializable_datoms = Index.to_list (Index.empty Eavt lmdb) @ duplicate_datoms
+    ; serializable_max_eid = max_eid
+    ; serializable_max_tx = max_tx
+    }
+
+let db_with_tail _context db tail =
+  List.fold_left
+    (fun db group ->
+      match group with
+      | [] -> db
+      | _ -> Db.refresh_indexes_with_tx_data db group)
+    db
+    tail
+
+let restore context storage =
+  let lmdb = Datascript_storage_lmdb.lmdb storage in
+  let schema, max_eid, max_tx, duplicate_datoms =
+    Datascript_storage_lmdb.restore_meta lmdb
+  in
+  let schema = Schema.validate_schema schema in
+  let duplicate_eavt_by_entity =
+    let table = Hashtbl.create 1024 in
+    List.iter
+      (fun datom ->
+        let existing = Option.value (Hashtbl.find_opt table datom.e) ~default:[] in
+        Hashtbl.replace table datom.e (datom :: existing))
+      duplicate_datoms;
+    Hashtbl.iter (fun entity_id datoms -> Hashtbl.replace table entity_id (List.rev datoms)) table;
+    table
+  in
+  let duplicate_datoms_by_attr duplicate_datoms =
+    let table = Hashtbl.create 1024 in
+    List.iter
+      (fun datom ->
+        let existing = Option.value (Hashtbl.find_opt table datom.a) ~default:[] in
+        Hashtbl.replace table datom.a (datom :: existing))
+      duplicate_datoms;
+    Hashtbl.iter (fun attr datoms -> Hashtbl.replace table attr (List.rev datoms)) table;
+    table
+  in
+  let duplicate_aevt_datoms = List.sort (Util.compare_datom Aevt) duplicate_datoms in
+  let duplicate_avet_datoms =
+    duplicate_datoms
+    |> List.filter (fun datom -> Schema.schema_attr_is_avet_accessible schema datom.a)
+    |> List.sort (Util.compare_datom Avet)
+  in
+  Some
+    { db_uid = context.next_db_uid ()
+    ; schema
+    ; eavt_index = Index.empty Eavt lmdb
+    ; aevt_index = Index.empty Aevt lmdb
+    ; avet_index = Index.empty Avet lmdb
+    ; aevt_by_attr = Hashtbl.create 0
+    ; avet_by_attr = Hashtbl.create 0
+    ; duplicate_datoms
+    ; duplicate_aevt_datoms
+    ; duplicate_avet_datoms
+    ; duplicate_eavt_by_entity
+    ; duplicate_aevt_by_attr = duplicate_datoms_by_attr duplicate_aevt_datoms
+    ; duplicate_avet_by_attr = duplicate_datoms_by_attr duplicate_avet_datoms
+    ; max_eid
+    ; max_datom_e = max_eid
+    ; max_tx
+    ; filter_pred = None
+    ; storage_ref = Some storage
+    ; tx_fns = []
+    }
+
+let storage_addresses storage = storage.storage_list_addresses ()
+let storage (db : db) = db.storage_ref
+
+let storage_root_addresses storage = storage.storage_list_addresses ()
+
+let addresses dbs =
+  dbs
+  |> List.concat_map (fun db ->
+    match db.storage_ref with
+    | None -> []
+    | Some storage -> storage_root_addresses storage)
+  |> List.sort_uniq compare
+
+let settings (db : db) =
+  [ "branching-factor", Int 32
+  ; "ref-type", Keyword "weak"
+  ; "storage", Bool (Option.is_some db.storage_ref)
+  ]
+
+let collect_garbage _storage = ()
diff --git a/impl/storage.ml b/impl/storage_pss.ml
similarity index 100%
rename from impl/storage.ml
rename to impl/storage_pss.ml
diff --git a/impl/util.ml b/impl/util.ml
index b0497da..3776793 100644
--- a/impl/util.ml
+++ b/impl/util.ml
@@ -53,306 +53,18 @@ and value_equal left right =
   | Ref_to left, Ref_to right -> entity_ref_equal left right
   | _ -> false
 
-let split_keyword keyword =
-  match String.index_opt keyword '/' with
-  | None -> "", keyword
-  | Some index ->
-    let namespace = String.sub keyword 0 index in
-    let name = String.sub keyword (index + 1) (String.length keyword - index - 1) in
-    namespace, name
-
-let rec compare_list_items_with compare_item left right =
-  match left, right with
-  | [], [] -> 0
-  | left :: left_rest, right :: right_rest ->
-    let comparison = compare_item left right in
-    if comparison <> 0 then comparison else compare_list_items_with compare_item left_rest right_rest
-  | [], _ | _, [] -> 0
-
-let compare_list_with compare_item left right =
-  let length_comparison = compare (List.length left) (List.length right) in
-  if length_comparison <> 0 then length_comparison
-  else compare_list_items_with compare_item left right
-
-let compare_option_with compare_item left right =
-  match left, right with
-  | None, None -> 0
-  | None, Some _ -> -1
-  | Some _, None -> 1
-  | Some left, Some right -> compare_item left right
-
-let i32 value = Int32.of_int value
-let i32_to_int value = Int32.to_int value
-let i32_add left right = Int32.add left right
-let i32_mul left right = Int32.mul left right
-let i32_xor left right = Int32.logxor left right
-let i32_shift_left value bits = Int32.shift_left value bits
-let i32_shift_right value bits = Int32.shift_right value bits
-let i32_shift_right_logical value bits = Int32.shift_right_logical value bits
-
-let i32_rotate_left value bits =
-  Int32.logor (Int32.shift_left value bits) (Int32.shift_right_logical value (32 - bits))
-
-let murmur3_mix_k1 value =
-  value
-  |> fun value -> i32_mul value (i32 (-862048943))
-  |> fun value -> i32_rotate_left value 15
-  |> fun value -> i32_mul value (i32 461845907)
-
-let murmur3_mix_h1 hash value =
-  i32_xor hash value
-  |> fun hash -> i32_rotate_left hash 13
-  |> fun hash -> i32_add (i32_mul hash (i32 5)) (i32 (-430675100))
-
-let murmur3_fmix hash length =
-  i32_xor hash (i32 length)
-  |> fun hash -> i32_xor hash (i32_shift_right_logical hash 16)
-  |> fun hash -> i32_mul hash (i32 (-2048144789))
-  |> fun hash -> i32_xor hash (i32_shift_right_logical hash 13)
-  |> fun hash -> i32_mul hash (i32 (-1028477387))
-  |> fun hash -> i32_xor hash (i32_shift_right_logical hash 16)
-
-let murmur3_hash_int value =
-  if value = 0 then 0
-  else
-    value
-    |> i32
-    |> murmur3_mix_k1
-    |> murmur3_mix_h1 Int32.zero
-    |> fun hash -> murmur3_fmix hash 4
-    |> i32_to_int
-
-let murmur3_hash_long value =
-  if value = Int64.zero then 0
-  else
-    let low = Int64.to_int value |> i32 in
-    let high = Int64.shift_right_logical value 32 |> Int64.to_int |> i32 in
-    Int32.zero
-    |> fun hash -> murmur3_mix_h1 hash (murmur3_mix_k1 low)
-    |> fun hash -> murmur3_mix_h1 hash (murmur3_mix_k1 high)
-    |> fun hash -> murmur3_fmix hash 8
-    |> i32_to_int
-
-let murmur3_hash_unencoded_chars text =
-  let hash = ref Int32.zero in
-  let index = ref 1 in
-  let length = String.length text in
-  while !index < length do
-    let code =
-      Char.code text.[!index - 1] lor (Char.code text.[!index] lsl 16)
-    in
-    hash := murmur3_mix_h1 !hash (murmur3_mix_k1 (i32 code));
-    index := !index + 2
-  done;
-  if length land 1 = 1 then
-    hash := i32_xor !hash (murmur3_mix_k1 (i32 (Char.code text.[length - 1])));
-  murmur3_fmix !hash (2 * length) |> i32_to_int
-
-let java_string_hash text =
-  let hash = ref Int32.zero in
-  String.iter
-    (fun ch -> hash := i32_add (i32_mul !hash (i32 31)) (i32 (Char.code ch)))
-    text;
-  i32_to_int !hash
-
-let hex_value = function
-  | '0' .. '9' as ch -> Char.code ch - Char.code '0'
-  | 'a' .. 'f' as ch -> 10 + Char.code ch - Char.code 'a'
-  | 'A' .. 'F' as ch -> 10 + Char.code ch - Char.code 'A'
-  | _ -> invalid_arg "invalid UUID hex digit"
-
-let uuid_halves uuid =
-  let digits =
-    uuid
-    |> String.to_seq
-    |> Seq.filter (( <> ) '-')
-    |> List.of_seq
-  in
-  if List.length digits <> 32 then invalid_arg ("invalid UUID: " ^ uuid);
-  let take_hex count digits =
-    let rec loop acc remaining rest =
-      if remaining = 0 then acc, rest
-      else
-        match rest with
-        | [] -> invalid_arg ("invalid UUID: " ^ uuid)
-        | ch :: rest ->
-          loop
-            (Int64.logor (Int64.shift_left acc 4) (Int64.of_int (hex_value ch)))
-            (remaining - 1)
-            rest
-    in
-    loop Int64.zero count digits
-  in
-  let most, rest = take_hex 16 digits in
-  let least, _ = take_hex 16 rest in
-  most, least
-
-let int64_low_i32 value =
-  Int64.logand value 0xffffffffL |> Int64.to_int |> i32
-
-let int64_high_i32 value =
-  Int64.shift_right_logical value 32 |> int64_low_i32
-
-let java_uuid_hash uuid =
-  let most, least = uuid_halves uuid in
-  i32_xor
-    (i32_xor (int64_high_i32 most) (int64_low_i32 most))
-    (i32_xor (int64_high_i32 least) (int64_low_i32 least))
-  |> i32_to_int
-
-let clojure_hash_combine seed hash =
-  i32_xor
-    (i32 seed)
-    (i32_add
-       (i32_add (i32 hash) (i32 (-1640531527)))
-       (i32_add (i32_shift_left (i32 seed) 6) (i32_shift_right (i32 seed) 2)))
-  |> i32_to_int
-
-let clojure_symbol_hash symbol =
-  let namespace, name = split_keyword symbol in
-  let namespace_hash = if namespace = "" then 0 else java_string_hash namespace in
-  clojure_hash_combine (murmur3_hash_unencoded_chars name) namespace_hash
-
-let clojure_keyword_hash name =
-  i32_add (i32 (clojure_symbol_hash name)) (i32 (-1640531527)) |> i32_to_int
-
-let murmur3_mix_coll_hash hash count =
-  hash
-  |> i32
-  |> murmur3_mix_k1
-  |> murmur3_mix_h1 Int32.zero
-  |> fun hash -> murmur3_fmix hash count
-  |> i32_to_int
-
-let murmur3_hash_ordered hashes =
-  let count, hash =
-    List.fold_left
-      (fun (count, hash) value_hash ->
-        count + 1, i32_add (i32_mul (i32 31) hash) (i32 value_hash))
-      (0, i32 1)
-      hashes
-  in
-  murmur3_mix_coll_hash (i32_to_int hash) count
-
-let murmur3_hash_unordered hashes =
-  let count, hash =
-    List.fold_left
-      (fun (count, hash) value_hash -> count + 1, i32_add hash (i32 value_hash))
-      (0, Int32.zero)
-      hashes
-  in
-  murmur3_mix_coll_hash (i32_to_int hash) count
-
-let rec clojure_hasheq = function
-  | Nil -> 0
-  | Bool true -> 1231
-  | Bool false -> 1237
-  | Int value -> murmur3_hash_long (Int64.of_int value)
-  | Float value -> Hashtbl.hash value
-  | String value -> murmur3_hash_int (java_string_hash value)
-  | Symbol value -> clojure_symbol_hash value
-  | Keyword value -> clojure_keyword_hash value
-  | List values | Vector values -> murmur3_hash_ordered (List.map clojure_hasheq values)
-  | Set values -> murmur3_hash_unordered (List.map clojure_hasheq values)
-  | Map entries ->
-    entries
-    |> List.map (fun (key, value) -> murmur3_hash_ordered [ clojure_hasheq key; clojure_hasheq value ])
-    |> murmur3_hash_unordered
-  | Tuple values ->
-    values
-    |> List.map (function None -> 0 | Some value -> clojure_hasheq value)
-    |> murmur3_hash_ordered
-  | Ref value -> murmur3_hash_long (Int64.of_int value)
-  | Uuid value -> java_uuid_hash value
-  | Instant value -> murmur3_hash_long (Int64.of_int value)
-  | Regex value -> Hashtbl.hash value
-  | TxRef -> Hashtbl.hash TxRef
-  | Ref_to value -> Hashtbl.hash (Ref_to value)
-
-let value_type_rank = function
-  | Nil -> 0
-  | Keyword _ -> 1
-  | Symbol _ -> 2
-  | Map _ -> 3
-  | Set _ -> 4
-  | List _ -> 5
-  | Vector _ -> 6
-  | Tuple _ -> 7
-  | Bool _ -> 8
-  | Int _ | Float _ | Ref _ -> 9
-  | String _ -> 10
-  | Regex _ -> 11
-  | Instant _ -> 12
-  | Uuid _ -> 13
-  | TxRef -> 14
-  | Ref_to _ -> 15
-
-let rec compare_value left right =
-  match left, right with
-  | Int left, Int right -> compare left right
-  | Float left, Float right -> compare left right
-  | Int left, Float right -> compare (float_of_int left) right
-  | Float left, Int right -> compare left (float_of_int right)
-  | Ref left, Ref right -> compare left right
-  | Int left, Ref right -> compare left right
-  | Ref left, Int right -> compare left right
-  | Float left, Ref right -> compare left (float_of_int right)
-  | Ref left, Float right -> compare (float_of_int left) right
-  | String left, String right -> compare left right
-  | Symbol left, Symbol right -> compare (split_keyword left) (split_keyword right)
-  | Bool left, Bool right -> compare left right
-  | Uuid left, Uuid right -> compare left right
-  | Instant left, Instant right -> compare left right
-  | Regex left, Regex right -> compare left right
-  | Nil, Nil -> 0
-  | Keyword left, Keyword right -> compare (split_keyword left) (split_keyword right)
-  | List left, List right -> compare_list_with compare_value left right
-  | Vector left, Vector right -> compare_list_with compare_value left right
-  | List left, Tuple right ->
-    compare_list_with (compare_option_with compare_value) (List.map (fun value -> Some value) left) right
-  | Set _, Set _ -> compare (clojure_hasheq left) (clojure_hasheq right)
-  | Map _, Map _ -> compare (clojure_hasheq left) (clojure_hasheq right)
-  | Tuple left, Tuple right -> compare_list_with (compare_option_with compare_value) left right
-  | Tuple left, List right ->
-    compare_list_with (compare_option_with compare_value) left (List.map (fun value -> Some value) right)
-  | _ ->
-    let rank_comparison = compare (value_type_rank left) (value_type_rank right) in
-    if rank_comparison <> 0 then rank_comparison else compare left right
-
-and compare_map_entry (left_key, left_value) (right_key, right_value) =
-  let comparison = compare_value left_key right_key in
-  if comparison <> 0 then comparison else compare_value left_value right_value
+let compare_list_with = Datascript_types.Compare.compare_list_with
+let compare_option_with = Datascript_types.Compare.compare_option_with
+let split_keyword = Datascript_types.Compare.split_keyword
+let compare_value = Datascript_types.Compare.compare_value
+let compare_datom = Datascript_types.Compare.compare_datom
+let compare_map_entry = Datascript_types.Compare.compare_map_entry
 
 let first_nonzero comparisons =
   List.find_opt (( <> ) 0) comparisons
   |> Option.value ~default:0
 
-let first_nonzero4 first second third fourth =
-  if first <> 0 then first
-  else if second <> 0 then second
-  else if third <> 0 then third
-  else fourth
-
-let compare_datom index left right =
-  match index with
-  | Eavt ->
-    first_nonzero4
-      (compare left.e right.e)
-      (compare left.a right.a)
-      (compare_value left.v right.v)
-      (compare left.tx right.tx)
-  | Aevt ->
-    first_nonzero4
-      (compare left.a right.a)
-      (compare left.e right.e)
-      (compare_value left.v right.v)
-      (compare left.tx right.tx)
-  | Avet ->
-    first_nonzero4
-      (compare left.a right.a)
-      (compare_value left.v right.v)
-      (compare left.e right.e)
-      (compare left.tx right.tx)
+let first_nonzero4 = Datascript_types.Compare.first_nonzero4
 
 let rec normalize_value = function
   | List values -> List (List.map normalize_value values)
diff --git a/impl/util.mli b/impl/util.mli
index b09ff8b..04bc8a4 100644
--- a/impl/util.mli
+++ b/impl/util.mli
@@ -8,6 +8,7 @@ val compare_list_with : ('a -> 'a -> int) -> 'a list -> 'a list -> int
 val compare_option_with : ('a -> 'a -> int) -> 'a option -> 'a option -> int
 val compare_value : value -> value -> int
 val first_nonzero : int list -> int
+val first_nonzero4 : int -> int -> int -> int -> int
 val compare_datom : index -> datom -> datom -> int
 val normalize_value : value -> value
 val normalize_datom_value : datom -> datom
diff --git a/lmdb/datascript_lmdb.ml b/lmdb/datascript_lmdb.ml
index 0d8586e..300cfe3 100644
--- a/lmdb/datascript_lmdb.ml
+++ b/lmdb/datascript_lmdb.ml
@@ -42,9 +42,9 @@ let close session =
     Env.close session.env;
     session.closed <- true)
 
-let encode_payload payload = Datascript_sqlite_codec.encode payload
+let encode_payload payload = Datascript_sqlite_codec.encode_storage_payload payload
 
-let decode_payload content = Datascript_sqlite_codec.decode content
+let decode_payload content = Datascript_sqlite_codec.decode_storage_payload content
 
 let storage session : Ds.storage =
   { storage_store =
diff --git a/lmdb/datascript_lmdb_codec.ml b/lmdb/datascript_lmdb_codec.ml
new file mode 100644
index 0000000..d527f2c
--- /dev/null
+++ b/lmdb/datascript_lmdb_codec.ml
@@ -0,0 +1,289 @@
+open Datascript_types
+
+let int32_be value =
+  let value = Int32.of_int value in
+  String.init 4 (fun index ->
+    let shift = (3 - index) * 8 in
+    Char.chr (Int32.to_int (Int32.shift_right_logical value shift) land 0xff))
+
+let int32_of_be bytes =
+  if String.length bytes <> 4 then invalid_arg "invalid int32 key segment";
+  let byte index = Char.code bytes.[index] in
+  Int32.of_int
+    ((byte 0 lsl 24) lor (byte 1 lsl 16) lor (byte 2 lsl 8) lor byte 3)
+  |> Int32.to_int
+
+let int64_be_int64 value =
+  String.init 8 (fun index ->
+    let shift = (7 - index) * 8 in
+    Char.chr (Int64.to_int (Int64.shift_right_logical value shift) land 0xff))
+
+let int64_of_be bytes =
+  if String.length bytes <> 8 then invalid_arg "invalid int64 key segment";
+  let byte index = Char.code bytes.[index] in
+  List.fold_left
+    (fun acc index -> Int64.logor (Int64.shift_left acc 8) (Int64.of_int (byte index)))
+    0L
+    [ 0; 1; 2; 3; 4; 5; 6; 7 ]
+
+let append_bytes buffer chunk = Buffer.add_string buffer chunk
+
+let append_int32 buffer value = append_bytes buffer (int32_be value)
+
+let append_int64 buffer value = append_bytes buffer (int64_be_int64 value)
+
+let float_sort_bits value =
+  let bits = Int64.bits_of_float value in
+  if Int64.compare bits 0L < 0 then Int64.logxor bits 0x7fffffffffffffffL else bits
+
+let append_string buffer text =
+  append_int32 buffer (String.length text);
+  Buffer.add_string buffer text
+
+let append_byte buffer value = Buffer.add_char buffer (Char.chr value)
+
+let read_int32 key offset =
+  if offset + 4 > String.length key then invalid_arg "truncated int32";
+  int32_of_be (String.sub key offset 4), offset + 4
+
+let read_string key offset =
+  let length, offset = read_int32 key offset in
+  if length < 0 || offset + length > String.length key then invalid_arg "truncated string";
+  String.sub key offset length, offset + length
+
+let read_byte key offset =
+  if offset >= String.length key then invalid_arg "truncated byte";
+  Char.code key.[offset], offset + 1
+
+let encode_keyword_like tag text =
+  let namespace, name = Datascript_types.Compare.split_keyword text in
+  let buffer = Buffer.create (String.length text + 16) in
+  append_byte buffer tag;
+  append_string buffer namespace;
+  append_string buffer name;
+  Buffer.contents buffer
+
+let encode_tagged_hash tag value =
+  let buffer = Buffer.create 8 in
+  append_byte buffer tag;
+  append_int32 buffer (Datascript_types.Compare.clojure_hasheq value);
+  Buffer.contents buffer
+
+let rec encode_value_key = function
+  | Nil -> "\000"
+  | Keyword value -> encode_keyword_like 1 value
+  | Symbol value -> encode_keyword_like 2 value
+  | Map _ as value -> encode_tagged_hash 3 value
+  | Set _ as value -> encode_tagged_hash 4 value
+  | List values ->
+      let buffer = Buffer.create 64 in
+      append_byte buffer 5;
+      append_int32 buffer (List.length values);
+      List.iter (fun value -> append_bytes buffer (encode_value_key value)) values;
+      Buffer.contents buffer
+  | Vector values ->
+      let buffer = Buffer.create 64 in
+      append_byte buffer 6;
+      append_int32 buffer (List.length values);
+      List.iter (fun value -> append_bytes buffer (encode_value_key value)) values;
+      Buffer.contents buffer
+  | Tuple values ->
+      let buffer = Buffer.create 64 in
+      append_byte buffer 7;
+      append_int32 buffer (List.length values);
+      List.iter
+        (function
+          | None -> append_byte buffer 0
+          | Some value ->
+              append_byte buffer 1;
+              append_bytes buffer (encode_value_key value))
+        values;
+      Buffer.contents buffer
+  | Bool false -> "\008\000"
+  | Bool true -> "\008\001"
+  | Int value ->
+      let buffer = Buffer.create 16 in
+      append_byte buffer 9;
+      append_byte buffer 0;
+      append_int64 buffer (float_sort_bits (float_of_int value));
+      Buffer.contents buffer
+  | Float value ->
+      let buffer = Buffer.create 16 in
+      append_byte buffer 9;
+      append_byte buffer 1;
+      append_int64 buffer (float_sort_bits value);
+      Buffer.contents buffer
+  | Ref value ->
+      let buffer = Buffer.create 16 in
+      append_byte buffer 9;
+      append_byte buffer 2;
+      append_int64 buffer (float_sort_bits (float_of_int value));
+      Buffer.contents buffer
+  | String value ->
+      let buffer = Buffer.create (String.length value + 8) in
+      append_byte buffer 10;
+      append_string buffer value;
+      Buffer.contents buffer
+  | Regex value ->
+      let buffer = Buffer.create (String.length value + 8) in
+      append_byte buffer 11;
+      append_string buffer value;
+      Buffer.contents buffer
+  | Instant value ->
+      let buffer = Buffer.create 16 in
+      append_byte buffer 12;
+      append_int32 buffer value;
+      Buffer.contents buffer
+  | Uuid value ->
+      let buffer = Buffer.create (String.length value + 8) in
+      append_byte buffer 13;
+      append_string buffer value;
+      Buffer.contents buffer
+  | TxRef -> "\014"
+  | Ref_to value ->
+      let buffer = Buffer.create 32 in
+      append_byte buffer 15;
+      append_int32 buffer (Hashtbl.hash value);
+      Buffer.contents buffer
+
+let rec decode_value_key bytes offset =
+  let tag, offset = read_byte bytes offset in
+  match tag with
+  | 0 -> Nil, offset
+  | 1 ->
+      let namespace, offset = read_string bytes offset in
+      let name, offset = read_string bytes offset in
+      (if namespace = "" then Keyword name else Keyword (namespace ^ "/" ^ name)), offset
+  | 2 ->
+      let namespace, offset = read_string bytes offset in
+      let name, offset = read_string bytes offset in
+      (if namespace = "" then Symbol name else Symbol (namespace ^ "/" ^ name)), offset
+  | 3 | 4 as tag ->
+      let _, offset = read_int32 bytes offset in
+      (if tag = 3 then Map [] else Set []), offset
+  | 5 | 6 as tag ->
+      let count, offset = read_int32 bytes offset in
+      if count < 0 then invalid_arg "invalid list length";
+      let rec loop remaining offset acc =
+        if remaining = 0 then
+          (if tag = 5 then List (List.rev acc) else Vector (List.rev acc)), offset
+        else
+          let value, offset = decode_value_key bytes offset in
+          loop (remaining - 1) offset (value :: acc)
+      in
+      loop count offset []
+  | 7 ->
+      let count, offset = read_int32 bytes offset in
+      if count < 0 then invalid_arg "invalid tuple length";
+      let rec loop remaining offset acc =
+        if remaining = 0 then Tuple (List.rev acc), offset
+        else
+          let marker, offset = read_byte bytes offset in
+          let value, offset =
+            match marker with
+            | 0 -> None, offset
+            | 1 ->
+                let value, offset = decode_value_key bytes offset in
+                Some value, offset
+            | _ -> invalid_arg "invalid tuple slot marker"
+          in
+          loop (remaining - 1) offset (value :: acc)
+      in
+      loop count offset []
+  | 8 ->
+      let value, offset = read_byte bytes offset in
+      (match value with 0 -> Bool false | 1 -> Bool true | _ -> invalid_arg "invalid bool key"), offset
+  | 9 ->
+      let kind, offset = read_byte bytes offset in
+      let bits, offset =
+        if offset + 8 > String.length bytes then invalid_arg "truncated numeric key"
+        else int64_of_be (String.sub bytes offset 8), offset + 8
+      in
+      let float_value =
+        let raw = if Int64.compare bits 0L < 0 then Int64.logxor bits 0x7fffffffffffffffL else bits in
+        Int64.float_of_bits raw
+      in
+      (match kind with
+       | 0 -> Int (int_of_float float_value)
+       | 1 -> Float float_value
+       | 2 -> Ref (int_of_float float_value)
+       | _ -> invalid_arg "invalid numeric kind"), offset
+  | 10 ->
+      let value, offset = read_string bytes offset in
+      String value, offset
+  | 11 ->
+      let value, offset = read_string bytes offset in
+      Regex value, offset
+  | 12 ->
+      let value, offset = read_int32 bytes offset in
+      Instant value, offset
+  | 13 ->
+      let value, offset = read_string bytes offset in
+      Uuid value, offset
+  | 14 -> TxRef, offset
+  | 15 -> Ref_to (Entity_id 0), offset + 4
+  | _ -> invalid_arg "invalid value key tag"
+
+let encode_datom_key index datom =
+  let buffer = Buffer.create 64 in
+  (match index with
+   | Eavt ->
+       append_int32 buffer datom.e;
+       append_string buffer datom.a;
+       append_bytes buffer (encode_value_key datom.v);
+       append_int32 buffer datom.tx
+   | Aevt ->
+       append_string buffer datom.a;
+       append_int32 buffer datom.e;
+       append_bytes buffer (encode_value_key datom.v);
+       append_int32 buffer datom.tx
+   | Avet ->
+       append_string buffer datom.a;
+       append_bytes buffer (encode_value_key datom.v);
+       append_int32 buffer datom.e;
+       append_int32 buffer datom.tx);
+  Buffer.contents buffer
+
+let decode_datom_key index bytes =
+  let e, a, v, tx =
+    match index with
+    | Eavt ->
+        let e, offset = read_int32 bytes 0 in
+        let a, offset = read_string bytes offset in
+        let v, offset = decode_value_key bytes offset in
+        let tx, offset = read_int32 bytes offset in
+        if offset <> String.length bytes then invalid_arg "trailing eavt key bytes";
+        e, a, v, tx
+    | Aevt ->
+        let a, offset = read_string bytes 0 in
+        let e, offset = read_int32 bytes offset in
+        let v, offset = decode_value_key bytes offset in
+        let tx, offset = read_int32 bytes offset in
+        if offset <> String.length bytes then invalid_arg "trailing aevt key bytes";
+        e, a, v, tx
+    | Avet ->
+        let a, offset = read_string bytes 0 in
+        let v, offset = decode_value_key bytes offset in
+        let e, offset = read_int32 bytes offset in
+        let tx, offset = read_int32 bytes offset in
+        if offset <> String.length bytes then invalid_arg "trailing avet key bytes";
+        e, a, v, tx
+  in
+  { e; a; v; tx; added = true }
+
+let encode_datom_value datom =
+  Marshal.to_string (datom.added, datom.v) []
+
+let decode_datom_value bytes =
+  let added, v = Marshal.from_string bytes 0 in
+  { e = 0; a = ""; v; tx = 0; added }
+
+let compare_encoded_keys index left right =
+  Datascript_types.Compare.compare_datom index
+    (decode_datom_key index left)
+    (decode_datom_key index right)
+
+let encode_schema schema = Marshal.to_string schema []
+let decode_schema bytes = Marshal.from_string bytes 0
+let encode_datoms datoms = Marshal.to_string datoms []
+let decode_datoms bytes = Marshal.from_string bytes 0
diff --git a/lmdb/datascript_lmdb_codec.mli b/lmdb/datascript_lmdb_codec.mli
new file mode 100644
index 0000000..79ce5f3
--- /dev/null
+++ b/lmdb/datascript_lmdb_codec.mli
@@ -0,0 +1,13 @@
+open Datascript_types
+
+val encode_datom_key : index -> datom -> string
+val decode_datom_key : index -> string -> datom
+val encode_datom_value : datom -> string
+val decode_datom_value : string -> datom
+
+val compare_encoded_keys : index -> string -> string -> int
+
+val encode_schema : schema -> string
+val decode_schema : string -> schema
+val encode_datoms : datom list -> string
+val decode_datoms : string -> datom list
diff --git a/lmdb/datascript_lmdb_db.mli b/lmdb/datascript_lmdb_db.mli
new file mode 100644
index 0000000..d53a813
--- /dev/null
+++ b/lmdb/datascript_lmdb_db.mli
@@ -0,0 +1,16 @@
+open Datascript_types
+
+type t
+
+val create_temp : unit -> t
+val open_path : string -> t
+val close : t -> unit
+val sync : t -> unit
+val remove_path : string -> unit
+
+val meta_get : t -> string -> string option
+val meta_set : t -> string -> string -> unit
+
+val fold_index : index -> t -> (string -> string -> unit) -> unit
+val put_index : index -> t -> string -> string -> unit
+val remove_index : index -> t -> string -> unit
diff --git a/lmdb/datascript_lmdb_db_melange.ml b/lmdb/datascript_lmdb_db_melange.ml
new file mode 100644
index 0000000..459980c
--- /dev/null
+++ b/lmdb/datascript_lmdb_db_melange.ml
@@ -0,0 +1,88 @@
+open Datascript_types
+
+type js = Js.t
+
+external open_root : string -> js = "open"
+  [@@mel.module "./datascript_lmdb_node.js"]
+
+external open_subdb : js -> string -> js = "openDB"
+  [@@mel.module "./datascript_lmdb_node.js"]
+
+external js_get : js -> string -> string Js.nullable = "get"
+  [@@mel.module "./datascript_lmdb_node.js"]
+
+external js_put : js -> string -> string -> unit = "put"
+  [@@mel.module "./datascript_lmdb_node.js"]
+
+external js_remove : js -> string -> unit = "remove"
+  [@@mel.module "./datascript_lmdb_node.js"]
+
+external js_sync : js -> unit = "sync"
+  [@@mel.module "./datascript_lmdb_node.js"]
+
+external js_close : js -> unit = "close"
+  [@@mel.module "./datascript_lmdb_node.js"]
+
+external js_range : js -> (string * string) array = "range"
+  [@@mel.module "./datascript_lmdb_node.js"]
+
+external temp_path : unit -> string = "tempPath"
+  [@@mel.module "./datascript_lmdb_node.js"]
+
+let remove_path _path = ()
+
+let open_db path =
+  let root = open_root path in
+  { Datascript_lmdb_db.path; env = root; eavt = open_subdb root "ds/eavt"
+  ; aevt = open_subdb root "ds/aevt"; avet = open_subdb root "ds/avet"
+  ; meta = open_subdb root "ds/meta"; closed = false
+  }
+
+let create_temp () = open_db (temp_path ())
+
+let open_path path = open_db path
+
+let ensure_open db =
+  if db.closed then invalid_arg "LMDB database is closed"
+
+let close db =
+  if not db.closed then (
+    js_close db.env;
+    db.closed <- true)
+
+let sync db =
+  ensure_open db;
+  js_sync db.env
+
+let map_for_index index db =
+  match index with
+  | Eavt -> db.eavt
+  | Aevt -> db.aevt
+  | Avet -> db.avet
+
+let meta_get db key =
+  ensure_open db;
+  match Js.Nullable.toOption (js_get db.meta key) with
+  | None -> None
+  | Some value -> Some value
+
+let meta_set db key value =
+  ensure_open db;
+  js_put db.meta key value
+
+let with_write db f =
+  ensure_open db;
+  f ()
+
+let fold_index index db f =
+  ensure_open db;
+  let map = map_for_index index db in
+  Array.iter (fun (key, value) -> f key value) (js_range map)
+
+let put_index index db key value =
+  ensure_open db;
+  js_put (map_for_index index db) key value
+
+let remove_index index db key =
+  ensure_open db;
+  js_remove (map_for_index index db) key
diff --git a/lmdb/datascript_lmdb_index.ml b/lmdb/datascript_lmdb_index.ml
new file mode 100644
index 0000000..2cfbc4b
--- /dev/null
+++ b/lmdb/datascript_lmdb_index.ml
@@ -0,0 +1,94 @@
+open Datascript_types
+
+type t = { db : Datascript_lmdb_db.t; which : index }
+
+type 'a seq = { cmp : datom -> datom -> int; datoms : datom list; offset : int }
+
+let db_of t = t.db
+let make index db = { db; which = index }
+let cmp_for index = Datascript_types.Compare.compare_datom index
+
+let decode_entry index key value =
+  let datom = Datascript_lmdb_codec.decode_datom_key index key in
+  let payload = Datascript_lmdb_codec.decode_datom_value value in
+  { datom with added = payload.added; v = payload.v }
+
+let put_datom t datom =
+  let key = Datascript_lmdb_codec.encode_datom_key t.which datom in
+  let value = Datascript_lmdb_codec.encode_datom_value datom in
+  Datascript_lmdb_db.put_index t.which t.db key value
+
+let remove_datom t datom =
+  let key = Datascript_lmdb_codec.encode_datom_key t.which datom in
+  Datascript_lmdb_db.remove_index t.which t.db key
+
+let empty index db = make index db
+
+let of_sorted_list index datoms db =
+  let t = empty index db in
+  List.iter (put_datom t) datoms;
+  t
+
+let add datom t =
+  put_datom t datom;
+  t
+
+let remove datom t =
+  remove_datom t datom;
+  t
+
+let collect_datoms t =
+  let datoms = ref [] in
+  Datascript_lmdb_db.fold_index t.which t.db (fun key value ->
+    datoms := decode_entry t.which key value :: !datoms);
+  List.rev !datoms
+
+let to_list t = collect_datoms t
+let fold f init t = List.fold_left f init (to_list t)
+
+let in_range cmp lower upper datom =
+  let above_lower =
+    match lower with
+    | None -> true
+    | Some lower -> cmp datom lower >= 0
+  in
+  let below_upper =
+    match upper with
+    | None -> true
+    | Some upper -> cmp datom upper <= 0
+  in
+  above_lower && below_upper
+
+let make_seq ?(cmp = cmp_for Eavt) ?from_ ?to_ datoms =
+  let datoms = List.filter (in_range cmp from_ to_) datoms in
+  { cmp; datoms; offset = 0 }
+
+let to_seq ({ datoms; offset } as seq) =
+  let rec loop index () =
+    if index >= List.length datoms then Seq.Nil
+    else Seq.Cons (List.nth datoms index, loop (index + 1))
+  in
+  loop seq.offset
+
+let seq t = make_seq ~cmp:(cmp_for t.which) (to_list t)
+
+let slice_seq ?from_ ?to_ ?cmp t =
+  let cmp = Option.value ~default:(cmp_for t.which) cmp in
+  make_seq ~cmp ?from_ ?to_ (to_list t)
+
+let rslice_seq ?from_ ?to_ ?cmp t =
+  let cmp = Option.value ~default:(cmp_for t.which) cmp in
+  make_seq ~cmp ?from_ ?to_ (List.rev (to_list t))
+
+let slice ?from_ ?to_ ?cmp t = slice_seq ?from_ ?to_ ?cmp t |> seq_to_list
+
+let seq_to_list seq = to_seq seq |> List.of_seq
+let fold_seq f init seq = List.fold_left f init (seq_to_list seq)
+
+let seek bound seq =
+  let rec count index =
+    if index >= List.length seq.datoms then index
+    else if seq.cmp (List.nth seq.datoms index) bound >= 0 then index
+    else count (index + 1)
+  in
+  { seq with offset = count 0 }
diff --git a/lmdb/datascript_lmdb_index.mli b/lmdb/datascript_lmdb_index.mli
new file mode 100644
index 0000000..9fe6431
--- /dev/null
+++ b/lmdb/datascript_lmdb_index.mli
@@ -0,0 +1,30 @@
+open Datascript_types
+
+type t
+type 'a seq
+
+val db_of : t -> Datascript_lmdb_db.t
+
+val empty : index -> Datascript_lmdb_db.t -> t
+val of_sorted_list : index -> datom list -> Datascript_lmdb_db.t -> t
+
+val add : datom -> t -> t
+val remove : datom -> t -> t
+
+val to_list : t -> datom list
+val fold : ('acc -> datom -> 'acc) -> 'acc -> t -> 'acc
+
+val slice :
+  ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom list
+
+val slice_seq :
+  ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom seq
+
+val rslice_seq :
+  ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom seq
+
+val seq : t -> datom seq
+val seq_to_list : datom seq -> datom list
+val fold_seq : ('acc -> datom -> 'acc) -> 'acc -> datom seq -> 'acc
+val to_seq : datom seq -> datom Seq.t
+val seek : datom -> datom seq -> datom seq
diff --git a/lmdb/datascript_lmdb_node.js b/lmdb/datascript_lmdb_node.js
new file mode 100644
index 0000000..f144c4f
--- /dev/null
+++ b/lmdb/datascript_lmdb_node.js
@@ -0,0 +1,51 @@
+// Node LMDB bindings for Melange. Requires the `lmdb` npm package.
+const { open } = require("lmdb");
+const fs = require("fs");
+const os = require("os");
+const path = require("path");
+
+function removePath(dbPath) {
+  if (fs.existsSync(dbPath)) fs.rmSync(dbPath, { recursive: true, force: true });
+}
+
+exports.open = function (dbPath) {
+  removePath(dbPath);
+  return open({ path: dbPath, compression: false });
+};
+
+exports.openDB = function (root, name) {
+  return root.openDB(name, {});
+};
+
+exports.get = function (db, key) {
+  return db.get(key);
+};
+
+exports.put = function (db, key, value) {
+  db.put(key, value);
+};
+
+exports.remove = function (db, key) {
+  db.remove(key);
+};
+
+exports.sync = function (_root) {};
+
+exports.close = function (root) {
+  root.close();
+};
+
+exports.range = function (db) {
+  const entries = [];
+  for (const { key, value } of db.getRange()) {
+    entries.push([key, value]);
+  }
+  return entries;
+};
+
+exports.tempPath = function () {
+  return path.join(
+    os.tmpdir(),
+    "datascript_lmdb_" + Date.now() + "_" + Math.random().toString(16).slice(2)
+  );
+};
diff --git a/lmdb/datascript_storage_lmdb.ml b/lmdb/datascript_storage_lmdb.ml
new file mode 100644
index 0000000..405958e
--- /dev/null
+++ b/lmdb/datascript_storage_lmdb.ml
@@ -0,0 +1,80 @@
+open Datascript_types
+
+type t = Datascript_lmdb_db.t
+
+let registry : (storage, t) Hashtbl.t = Hashtbl.create 16
+
+let lmdb storage =
+  match Hashtbl.find_opt registry storage with
+  | Some lmdb -> lmdb
+  | None -> invalid_arg "storage is not LMDB-backed"
+
+let register storage lmdb = Hashtbl.replace registry storage lmdb
+
+let create_temp () = Datascript_lmdb_db.create_temp ()
+let open_path path = Datascript_lmdb_db.open_path path
+let close = Datascript_lmdb_db.close
+let sync = Datascript_lmdb_db.sync
+
+let meta_get = Datascript_lmdb_db.meta_get
+let meta_set = Datascript_lmdb_db.meta_set
+
+let meta_schema_key = "schema"
+let meta_max_eid_key = "max_eid"
+let meta_max_tx_key = "max_tx"
+let meta_duplicates_key = "duplicate_datoms"
+
+let wrap lmdb =
+  let storage =
+    { storage_store =
+        (fun _entries -> sync lmdb)
+    ; storage_restore =
+        (fun address ->
+          if String.equal address "lmdb" then Some Storage_session else None)
+    ; storage_list_addresses = (fun () -> [ "lmdb" ])
+    ; storage_delete = (fun _addresses -> ())
+    }
+  in
+  register storage lmdb;
+  storage
+
+let memory_storage () = wrap (create_temp ())
+
+let encode_int value =
+  Datascript_lmdb_codec.encode_datoms
+    [ { e = value; a = ""; v = Nil; tx = 0; added = true } ]
+
+let decode_int bytes =
+  match Datascript_lmdb_codec.decode_datoms bytes with
+  | { e; _ } :: _ -> e
+  | [] -> 0
+
+let store_meta lmdb db =
+  meta_set lmdb meta_schema_key (Datascript_lmdb_codec.encode_schema db.schema);
+  meta_set lmdb meta_max_eid_key (encode_int db.max_eid);
+  meta_set lmdb meta_max_tx_key (encode_int db.max_tx);
+  meta_set lmdb meta_duplicates_key (Datascript_lmdb_codec.encode_datoms db.duplicate_datoms);
+  sync lmdb
+
+let restore_meta lmdb =
+  let schema =
+    match meta_get lmdb meta_schema_key with
+    | None -> []
+    | Some bytes -> Datascript_lmdb_codec.decode_schema bytes
+  in
+  let max_eid =
+    match meta_get lmdb meta_max_eid_key with
+    | None -> 0
+    | Some bytes -> decode_int bytes
+  in
+  let max_tx =
+    match meta_get lmdb meta_max_tx_key with
+    | None -> 0x20000000
+    | Some bytes -> decode_int bytes
+  in
+  let duplicate_datoms =
+    match meta_get lmdb meta_duplicates_key with
+    | None -> []
+    | Some bytes -> Datascript_lmdb_codec.decode_datoms bytes
+  in
+  schema, max_eid, max_tx, duplicate_datoms
diff --git a/lmdb/dune b/lmdb/dune
index 8e6ef09..38bda56 100644
--- a/lmdb/dune
+++ b/lmdb/dune
@@ -1,5 +1,23 @@
+(library
+ (name datascript_lmdb_codec)
+ (public_name datascript-ocaml-native.lmdb-codec)
+ (wrapped false)
+ (modes native melange)
+ (modules datascript_lmdb_codec)
+ (libraries datascript_types))
+
 (library
  (name datascript_lmdb)
  (public_name datascript-ocaml-native.lmdb)
  (wrapped false)
- (libraries datascript-ocaml-native datascript_sqlite lmdb))
+ (modes native)
+ (modules datascript_lmdb)
+ (libraries
+  datascript-ocaml-native
+  datascript_sqlite
+  lmdb_db_native
+  storage_lmdb_native
+  lmdb))
+
+(subdir native)
+(subdir melange)
diff --git a/lmdb/melange/datascript_lmdb_db.ml b/lmdb/melange/datascript_lmdb_db.ml
new file mode 100644
index 0000000..4117385
--- /dev/null
+++ b/lmdb/melange/datascript_lmdb_db.ml
@@ -0,0 +1,108 @@
+open Datascript_types
+
+type js = Js.t
+
+external open_root : string -> js = "open"
+  [@@mel.module "./datascript_lmdb_node.js"]
+
+external open_subdb : js -> string -> js = "openDB"
+  [@@mel.module "./datascript_lmdb_node.js"]
+
+external js_get : js -> string -> string Js.nullable = "get"
+  [@@mel.module "./datascript_lmdb_node.js"]
+
+external js_put : js -> string -> string -> unit = "put"
+  [@@mel.module "./datascript_lmdb_node.js"]
+
+external js_remove : js -> string -> unit = "remove"
+  [@@mel.module "./datascript_lmdb_node.js"]
+
+external js_sync : js -> unit = "sync"
+  [@@mel.module "./datascript_lmdb_node.js"]
+
+external js_close : js -> unit = "close"
+  [@@mel.module "./datascript_lmdb_node.js"]
+
+external js_range : js -> (string * string) array = "range"
+  [@@mel.module "./datascript_lmdb_node.js"]
+
+external temp_path : unit -> string = "tempPath"
+  [@@mel.module "./datascript_lmdb_node.js"]
+
+type t =
+  { path : string
+  ; env : js
+  ; eavt : js
+  ; aevt : js
+  ; avet : js
+  ; meta : js
+  ; mutable closed : bool
+  }
+
+let remove_path _path = ()
+
+let open_db path =
+  let root = open_root path in
+  { path; env = root; eavt = open_subdb root "ds/eavt"; aevt = open_subdb root "ds/aevt"
+  ; avet = open_subdb root "ds/avet"; meta = open_subdb root "ds/meta"; closed = false
+  }
+
+let create_temp () = open_db (temp_path ())
+
+let open_path path = open_db path
+
+let ensure_open db =
+  if db.closed then invalid_arg ("LMDB database is closed: " ^ db.path)
+
+let close db =
+  if not db.closed then (
+    js_close db.env;
+    db.closed <- true)
+
+let sync db =
+  ensure_open db;
+  js_sync db.env
+
+let meta_get db key =
+  ensure_open db;
+  match Js.Nullable.toOption (js_get db.meta key) with
+  | None -> None
+  | Some value -> Some value
+
+let meta_set db key value =
+  ensure_open db;
+  js_put db.meta key value
+
+let with_write db f =
+  ensure_open db;
+  f ()
+
+let fold_index index db f =
+  ensure_open db;
+  let map =
+    match index with
+    | Eavt -> db.eavt
+    | Aevt -> db.aevt
+    | Avet -> db.avet
+  in
+  Array.iter (fun (key, value) -> f key value) (js_range map)
+
+let put_index index db key value =
+  ensure_open db;
+  let map =
+    match index with
+    | Eavt -> db.eavt
+    | Aevt -> db.aevt
+    | Avet -> db.avet
+  in
+  js_put map key value
+
+let remove_index index db key =
+  ensure_open db;
+  let map =
+    match index with
+    | Eavt -> db.eavt
+    | Aevt -> db.aevt
+    | Avet -> db.avet
+  in
+  js_remove map key
diff --git a/lmdb/melange/datascript_lmdb_db.mli b/lmdb/melange/datascript_lmdb_db.mli
new file mode 100644
index 0000000..d53a813
--- /dev/null
+++ b/lmdb/melange/datascript_lmdb_db.mli
@@ -0,0 +1,16 @@
+open Datascript_types
+
+type t
+
+val create_temp : unit -> t
+val open_path : string -> t
+val close : t -> unit
+val sync : t -> unit
+val remove_path : string -> unit
+
+val meta_get : t -> string -> string option
+val meta_set : t -> string -> string -> unit
+
+val fold_index : index -> t -> (string -> string -> unit) -> unit
+val put_index : index -> t -> string -> string -> unit
+val remove_index : index -> t -> string -> unit
diff --git a/lmdb/melange/datascript_lmdb_index.ml b/lmdb/melange/datascript_lmdb_index.ml
new file mode 100644
index 0000000..d0b23c8
--- /dev/null
+++ b/lmdb/melange/datascript_lmdb_index.ml
@@ -0,0 +1,94 @@
+open Datascript_types
+
+type t = { db : Datascript_lmdb_db.t; which : index }
+
+type 'a seq = { cmp : datom -> datom -> int; datoms : datom list; offset : int }
+
+let db_of t = t.db
+let make index db = { db; which = index }
+let cmp_for index = Datascript_types.Compare.compare_datom index
+
+let decode_entry index key value =
+  let datom = Datascript_lmdb_codec.decode_datom_key index key in
+  let payload = Datascript_lmdb_codec.decode_datom_value value in
+  { datom with added = payload.added; v = payload.v }
+
+let put_datom t datom =
+  let key = Datascript_lmdb_codec.encode_datom_key t.which datom in
+  let value = Datascript_lmdb_codec.encode_datom_value datom in
+  Datascript_lmdb_db.put_index t.which t.db key value
+
+let remove_datom t datom =
+  let key = Datascript_lmdb_codec.encode_datom_key t.which datom in
+  Datascript_lmdb_db.remove_index t.which t.db key
+
+let empty index db = make index db
+
+let of_sorted_list index datoms db =
+  let t = empty index db in
+  List.iter (put_datom t) datoms;
+  t
+
+let add datom t =
+  put_datom t datom;
+  t
+
+let remove datom t =
+  remove_datom t datom;
+  t
+
+let collect_datoms t =
+  let datoms = ref [] in
+  Datascript_lmdb_db.fold_index t.which t.db (fun key value ->
+    datoms := decode_entry t.which key value :: !datoms);
+  List.rev !datoms
+
+let to_list t = collect_datoms t
+let fold f init t = List.fold_left f init (to_list t)
+
+let in_range cmp lower upper datom =
+  let above_lower =
+    match lower with
+    | None -> true
+    | Some lower -> cmp datom lower >= 0
+  in
+  let below_upper =
+    match upper with
+    | None -> true
+    | Some upper -> cmp datom upper <= 0
+  in
+  above_lower && below_upper
+
+let make_seq ?(cmp = cmp_for Eavt) ?from_ ?to_ datoms =
+  let datoms = List.filter (in_range cmp from_ to_) datoms in
+  { cmp; datoms; offset = 0 }
+
+let to_seq ({ cmp = _; datoms; offset = start }) =
+  let rec loop index () =
+    if index >= List.length datoms then Seq.Nil
+    else Seq.Cons (List.nth datoms index, loop (index + 1))
+  in
+  loop start
+
+let seq t = make_seq ~cmp:(cmp_for t.which) (to_list t)
+
+let slice_seq ?from_ ?to_ ?cmp t =
+  let cmp = Option.value ~default:(cmp_for t.which) cmp in
+  make_seq ~cmp ?from_ ?to_ (to_list t)
+
+let rslice_seq ?from_ ?to_ ?cmp t =
+  let cmp = Option.value ~default:(cmp_for t.which) cmp in
+  make_seq ~cmp ?from_ ?to_ (List.rev (to_list t))
+
+let seq_to_list seq = to_seq seq |> List.of_seq
+
+let slice ?from_ ?to_ ?cmp t = slice_seq ?from_ ?to_ ?cmp t |> seq_to_list
+let fold_seq f init seq = List.fold_left f init (seq_to_list seq)
+
+let seek bound seq =
+  let rec count index =
+    if index >= List.length seq.datoms then index
+    else if seq.cmp (List.nth seq.datoms index) bound >= 0 then index
+    else count (index + 1)
+  in
+  { seq with offset = count 0 }
diff --git a/lmdb/melange/datascript_lmdb_index.mli b/lmdb/melange/datascript_lmdb_index.mli
new file mode 100644
index 0000000..9fe6431
--- /dev/null
+++ b/lmdb/melange/datascript_lmdb_index.mli
@@ -0,0 +1,30 @@
+open Datascript_types
+
+type t
+type 'a seq
+
+val db_of : t -> Datascript_lmdb_db.t
+
+val empty : index -> Datascript_lmdb_db.t -> t
+val of_sorted_list : index -> datom list -> Datascript_lmdb_db.t -> t
+
+val add : datom -> t -> t
+val remove : datom -> t -> t
+
+val to_list : t -> datom list
+val fold : ('acc -> datom -> 'acc) -> 'acc -> t -> 'acc
+
+val slice :
+  ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom list
+
+val slice_seq :
+  ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom seq
+
+val rslice_seq :
+  ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom seq
+
+val seq : t -> datom seq
+val seq_to_list : datom seq -> datom list
+val fold_seq : ('acc -> datom -> 'acc) -> 'acc -> datom seq -> 'acc
+val to_seq : datom seq -> datom Seq.t
+val seek : datom -> datom seq -> datom seq
diff --git a/lmdb/melange/datascript_lmdb_node.js b/lmdb/melange/datascript_lmdb_node.js
new file mode 100644
index 0000000..f144c4f
--- /dev/null
+++ b/lmdb/melange/datascript_lmdb_node.js
@@ -0,0 +1,51 @@
+// Node LMDB bindings for Melange. Requires the `lmdb` npm package.
+const { open } = require("lmdb");
+const fs = require("fs");
+const os = require("os");
+const path = require("path");
+
+function removePath(dbPath) {
+  if (fs.existsSync(dbPath)) fs.rmSync(dbPath, { recursive: true, force: true });
+}
+
+exports.open = function (dbPath) {
+  removePath(dbPath);
+  return open({ path: dbPath, compression: false });
+};
+
+exports.openDB = function (root, name) {
+  return root.openDB(name, {});
+};
+
+exports.get = function (db, key) {
+  return db.get(key);
+};
+
+exports.put = function (db, key, value) {
+  db.put(key, value);
+};
+
+exports.remove = function (db, key) {
+  db.remove(key);
+};
+
+exports.sync = function (_root) {};
+
+exports.close = function (root) {
+  root.close();
+};
+
+exports.range = function (db) {
+  const entries = [];
+  for (const { key, value } of db.getRange()) {
+    entries.push([key, value]);
+  }
+  return entries;
+};
+
+exports.tempPath = function () {
+  return path.join(
+    os.tmpdir(),
+    "datascript_lmdb_" + Date.now() + "_" + Math.random().toString(16).slice(2)
+  );
+};
diff --git a/lmdb/melange/datascript_storage_lmdb.ml b/lmdb/melange/datascript_storage_lmdb.ml
new file mode 100644
index 0000000..405958e
--- /dev/null
+++ b/lmdb/melange/datascript_storage_lmdb.ml
@@ -0,0 +1,80 @@
+open Datascript_types
+
+type t = Datascript_lmdb_db.t
+
+let registry : (storage, t) Hashtbl.t = Hashtbl.create 16
+
+let lmdb storage =
+  match Hashtbl.find_opt registry storage with
+  | Some lmdb -> lmdb
+  | None -> invalid_arg "storage is not LMDB-backed"
+
+let register storage lmdb = Hashtbl.replace registry storage lmdb
+
+let create_temp () = Datascript_lmdb_db.create_temp ()
+let open_path path = Datascript_lmdb_db.open_path path
+let close = Datascript_lmdb_db.close
+let sync = Datascript_lmdb_db.sync
+
+let meta_get = Datascript_lmdb_db.meta_get
+let meta_set = Datascript_lmdb_db.meta_set
+
+let meta_schema_key = "schema"
+let meta_max_eid_key = "max_eid"
+let meta_max_tx_key = "max_tx"
+let meta_duplicates_key = "duplicate_datoms"
+
+let wrap lmdb =
+  let storage =
+    { storage_store =
+        (fun _entries -> sync lmdb)
+    ; storage_restore =
+        (fun address ->
+          if String.equal address "lmdb" then Some Storage_session else None)
+    ; storage_list_addresses = (fun () -> [ "lmdb" ])
+    ; storage_delete = (fun _addresses -> ())
+    }
+  in
+  register storage lmdb;
+  storage
+
+let memory_storage () = wrap (create_temp ())
+
+let encode_int value =
+  Datascript_lmdb_codec.encode_datoms
+    [ { e = value; a = ""; v = Nil; tx = 0; added = true } ]
+
+let decode_int bytes =
+  match Datascript_lmdb_codec.decode_datoms bytes with
+  | { e; _ } :: _ -> e
+  | [] -> 0
+
+let store_meta lmdb db =
+  meta_set lmdb meta_schema_key (Datascript_lmdb_codec.encode_schema db.schema);
+  meta_set lmdb meta_max_eid_key (encode_int db.max_eid);
+  meta_set lmdb meta_max_tx_key (encode_int db.max_tx);
+  meta_set lmdb meta_duplicates_key (Datascript_lmdb_codec.encode_datoms db.duplicate_datoms);
+  sync lmdb
+
+let restore_meta lmdb =
+  let schema =
+    match meta_get lmdb meta_schema_key with
+    | None -> []
+    | Some bytes -> Datascript_lmdb_codec.decode_schema bytes
+  in
+  let max_eid =
+    match meta_get lmdb meta_max_eid_key with
+    | None -> 0
+    | Some bytes -> decode_int bytes
+  in
+  let max_tx =
+    match meta_get lmdb meta_max_tx_key with
+    | None -> 0x20000000
+    | Some bytes -> decode_int bytes
+  in
+  let duplicate_datoms =
+    match meta_get lmdb meta_duplicates_key with
+    | None -> []
+    | Some bytes -> Datascript_lmdb_codec.decode_datoms bytes
+  in
+  schema, max_eid, max_tx, duplicate_datoms
diff --git a/lmdb/melange/dune b/lmdb/melange/dune
new file mode 100644
index 0000000..dadefd3
--- /dev/null
+++ b/lmdb/melange/dune
@@ -0,0 +1,25 @@
+(include_subdirs no)
+
+(library
+ (name lmdb_db_melange)
+ (public_name datascript-ocaml-melange.lmdb-db)
+ (wrapped false)
+ (modes melange)
+ (modules datascript_lmdb_db)
+ (libraries datascript_lmdb_codec melange.js))
+
+(library
+ (name lmdb_index_melange)
+ (public_name datascript-ocaml-melange.lmdb-index)
+ (wrapped false)
+ (modes melange)
+ (modules datascript_lmdb_index)
+ (libraries datascript_lmdb_codec lmdb_db_melange))
+
+(library
+ (name storage_lmdb_melange)
+ (public_name datascript-ocaml-melange.storage-lmdb)
+ (wrapped false)
+ (modes melange)
+ (modules datascript_storage_lmdb)
+ (libraries datascript_lmdb_codec lmdb_db_melange datascript_types))
diff --git a/lmdb/native/datascript_lmdb_db.ml b/lmdb/native/datascript_lmdb_db.ml
new file mode 100644
index 0000000..515353d
--- /dev/null
+++ b/lmdb/native/datascript_lmdb_db.ml
@@ -0,0 +1,104 @@
+open Datascript_types
+open Lmdb
+
+type t =
+  { path : string
+  ; env : Env.t
+  ; eavt : (string, string, [ `Uni ]) Map.t
+  ; aevt : (string, string, [ `Uni ]) Map.t
+  ; avet : (string, string, [ `Uni ]) Map.t
+  ; meta : (string, string, [ `Uni ]) Map.t
+  ; mutable closed : bool
+  }
+
+let default_map_size = 1024 * 1024 * 1024
+let lock_path path = path ^ "-lock"
+
+let remove_path path =
+  if Sys.file_exists path then Sys.remove path;
+  let lock = lock_path path in
+  if Sys.file_exists lock then Sys.remove lock
+
+let open_env db_path =
+  Env.(create Rw ~flags:Flags.no_subdir ~map_size:default_map_size ~max_maps:8 db_path)
+
+let open_named_map env name =
+  try Map.open_existing Nodup ~key:Conv.string ~value:Conv.string ~name env
+  with Not_found -> Map.create Nodup ~key:Conv.string ~value:Conv.string ~name env
+
+let open_db path =
+  remove_path path;
+  let env = open_env path in
+  { path; env; eavt = open_named_map env "ds/eavt"; aevt = open_named_map env "ds/aevt"
+  ; avet = open_named_map env "ds/avet"; meta = open_named_map env "ds/meta"; closed = false
+  }
+
+let create_temp () =
+  open_db
+    (Filename.temp_file
+       ~temp_dir:(Filename.get_temp_dir_name ())
+       "datascript_lmdb"
+       ".mdb")
+
+let open_path path = open_db path
+
+let ensure_open db =
+  if db.closed then invalid_arg ("LMDB database is closed: " ^ db.path)
+
+let close db =
+  if not db.closed then (
+    Map.close db.eavt;
+    Map.close db.aevt;
+    Map.close db.avet;
+    Map.close db.meta;
+    Env.sync db.env;
+    Env.close db.env;
+    db.closed <- true)
+
+let sync db =
+  ensure_open db;
+  Env.sync db.env
+
+let map_for_index index db =
+  match index with
+  | Eavt -> db.eavt
+  | Aevt -> db.aevt
+  | Avet -> db.avet
+
+let meta_get db key =
+  ensure_open db;
+  try Some (Map.get db.meta key) with Not_found -> None
+
+let meta_set db key value =
+  ensure_open db;
+  ignore
+    (Txn.go Rw db.env (fun txn ->
+       Map.set ~txn db.meta key value;
+       ()))
+
+let fold_index index db f =
+  ensure_open db;
+  let map = map_for_index index db in
+  let next = Map.to_dispenser map in
+  let rec loop () =
+    match next () with
+    | None -> ()
+    | Some (key, value) ->
+        f key value;
+        loop ()
+  in
+  loop ()
+
+let put_index index db key value =
+  ensure_open db;
+  ignore
+    (Txn.go Rw db.env (fun txn ->
+       Map.set ~txn (map_for_index index db) key value;
+       ()))
+
+let remove_index index db key =
+  ensure_open db;
+  ignore
+    (Txn.go Rw db.env (fun txn ->
+       (try Map.remove ~txn (map_for_index index db) key with Not_found -> ());
+       ()))
diff --git a/lmdb/native/datascript_lmdb_db.mli b/lmdb/native/datascript_lmdb_db.mli
new file mode 100644
index 0000000..d53a813
--- /dev/null
+++ b/lmdb/native/datascript_lmdb_db.mli
@@ -0,0 +1,16 @@
+open Datascript_types
+
+type t
+
+val create_temp : unit -> t
+val open_path : string -> t
+val close : t -> unit
+val sync : t -> unit
+val remove_path : string -> unit
+
+val meta_get : t -> string -> string option
+val meta_set : t -> string -> string -> unit
+
+val fold_index : index -> t -> (string -> string -> unit) -> unit
+val put_index : index -> t -> string -> string -> unit
+val remove_index : index -> t -> string -> unit
diff --git a/lmdb/native/datascript_lmdb_index.ml b/lmdb/native/datascript_lmdb_index.ml
new file mode 100644
index 0000000..d0b23c8
--- /dev/null
+++ b/lmdb/native/datascript_lmdb_index.ml
@@ -0,0 +1,94 @@
+open Datascript_types
+
+type t = { db : Datascript_lmdb_db.t; which : index }
+
+type 'a seq = { cmp : datom -> datom -> int; datoms : datom list; offset : int }
+
+let db_of t = t.db
+let make index db = { db; which = index }
+let cmp_for index = Datascript_types.Compare.compare_datom index
+
+let decode_entry index key value =
+  let datom = Datascript_lmdb_codec.decode_datom_key index key in
+  let payload = Datascript_lmdb_codec.decode_datom_value value in
+  { datom with added = payload.added; v = payload.v }
+
+let put_datom t datom =
+  let key = Datascript_lmdb_codec.encode_datom_key t.which datom in
+  let value = Datascript_lmdb_codec.encode_datom_value datom in
+  Datascript_lmdb_db.put_index t.which t.db key value
+
+let remove_datom t datom =
+  let key = Datascript_lmdb_codec.encode_datom_key t.which datom in
+  Datascript_lmdb_db.remove_index t.which t.db key
+
+let empty index db = make index db
+
+let of_sorted_list index datoms db =
+  let t = empty index db in
+  List.iter (put_datom t) datoms;
+  t
+
+let add datom t =
+  put_datom t datom;
+  t
+
+let remove datom t =
+  remove_datom t datom;
+  t
+
+let collect_datoms t =
+  let datoms = ref [] in
+  Datascript_lmdb_db.fold_index t.which t.db (fun key value ->
+    datoms := decode_entry t.which key value :: !datoms);
+  List.rev !datoms
+
+let to_list t = collect_datoms t
+let fold f init t = List.fold_left f init (to_list t)
+
+let in_range cmp lower upper datom =
+  let above_lower =
+    match lower with
+    | None -> true
+    | Some lower -> cmp datom lower >= 0
+  in
+  let below_upper =
+    match upper with
+    | None -> true
+    | Some upper -> cmp datom upper <= 0
+  in
+  above_lower && below_upper
+
+let make_seq ?(cmp = cmp_for Eavt) ?from_ ?to_ datoms =
+  let datoms = List.filter (in_range cmp from_ to_) datoms in
+  { cmp; datoms; offset = 0 }
+
+let to_seq ({ cmp = _; datoms; offset = start }) =
+  let rec loop index () =
+    if index >= List.length datoms then Seq.Nil
+    else Seq.Cons (List.nth datoms index, loop (index + 1))
+  in
+  loop start
+
+let seq t = make_seq ~cmp:(cmp_for t.which) (to_list t)
+
+let slice_seq ?from_ ?to_ ?cmp t =
+  let cmp = Option.value ~default:(cmp_for t.which) cmp in
+  make_seq ~cmp ?from_ ?to_ (to_list t)
+
+let rslice_seq ?from_ ?to_ ?cmp t =
+  let cmp = Option.value ~default:(cmp_for t.which) cmp in
+  make_seq ~cmp ?from_ ?to_ (List.rev (to_list t))
+
+let seq_to_list seq = to_seq seq |> List.of_seq
+
+let slice ?from_ ?to_ ?cmp t = slice_seq ?from_ ?to_ ?cmp t |> seq_to_list
+let fold_seq f init seq = List.fold_left f init (seq_to_list seq)
+
+let seek bound seq =
+  let rec count index =
+    if index >= List.length seq.datoms then index
+    else if seq.cmp (List.nth seq.datoms index) bound >= 0 then index
+    else count (index + 1)
+  in
+  { seq with offset = count 0 }
diff --git a/lmdb/native/datascript_lmdb_index.mli b/lmdb/native/datascript_lmdb_index.mli
new file mode 100644
index 0000000..9fe6431
--- /dev/null
+++ b/lmdb/native/datascript_lmdb_index.mli
@@ -0,0 +1,30 @@
+open Datascript_types
+
+type t
+type 'a seq
+
+val db_of : t -> Datascript_lmdb_db.t
+
+val empty : index -> Datascript_lmdb_db.t -> t
+val of_sorted_list : index -> datom list -> Datascript_lmdb_db.t -> t
+
+val add : datom -> t -> t
+val remove : datom -> t -> t
+
+val to_list : t -> datom list
+val fold : ('acc -> datom -> 'acc) -> 'acc -> t -> 'acc
+
+val slice :
+  ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom list
+
+val slice_seq :
+  ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom seq
+
+val rslice_seq :
+  ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom seq
+
+val seq : t -> datom seq
+val seq_to_list : datom seq -> datom list
+val fold_seq : ('acc -> datom -> 'acc) -> 'acc -> datom seq -> 'acc
+val to_seq : datom seq -> datom Seq.t
+val seek : datom -> datom seq -> datom seq
diff --git a/lmdb/native/datascript_storage_lmdb.ml b/lmdb/native/datascript_storage_lmdb.ml
new file mode 100644
index 0000000..405958e
--- /dev/null
+++ b/lmdb/native/datascript_storage_lmdb.ml
@@ -0,0 +1,80 @@
+open Datascript_types
+
+type t = Datascript_lmdb_db.t
+
+let registry : (storage, t) Hashtbl.t = Hashtbl.create 16
+
+let lmdb storage =
+  match Hashtbl.find_opt registry storage with
+  | Some lmdb -> lmdb
+  | None -> invalid_arg "storage is not LMDB-backed"
+
+let register storage lmdb = Hashtbl.replace registry storage lmdb
+
+let create_temp () = Datascript_lmdb_db.create_temp ()
+let open_path path = Datascript_lmdb_db.open_path path
+let close = Datascript_lmdb_db.close
+let sync = Datascript_lmdb_db.sync
+
+let meta_get = Datascript_lmdb_db.meta_get
+let meta_set = Datascript_lmdb_db.meta_set
+
+let meta_schema_key = "schema"
+let meta_max_eid_key = "max_eid"
+let meta_max_tx_key = "max_tx"
+let meta_duplicates_key = "duplicate_datoms"
+
+let wrap lmdb =
+  let storage =
+    { storage_store =
+        (fun _entries -> sync lmdb)
+    ; storage_restore =
+        (fun address ->
+          if String.equal address "lmdb" then Some Storage_session else None)
+    ; storage_list_addresses = (fun () -> [ "lmdb" ])
+    ; storage_delete = (fun _addresses -> ())
+    }
+  in
+  register storage lmdb;
+  storage
+
+let memory_storage () = wrap (create_temp ())
+
+let encode_int value =
+  Datascript_lmdb_codec.encode_datoms
+    [ { e = value; a = ""; v = Nil; tx = 0; added = true } ]
+
+let decode_int bytes =
+  match Datascript_lmdb_codec.decode_datoms bytes with
+  | { e; _ } :: _ -> e
+  | [] -> 0
+
+let store_meta lmdb db =
+  meta_set lmdb meta_schema_key (Datascript_lmdb_codec.encode_schema db.schema);
+  meta_set lmdb meta_max_eid_key (encode_int db.max_eid);
+  meta_set lmdb meta_max_tx_key (encode_int db.max_tx);
+  meta_set lmdb meta_duplicates_key (Datascript_lmdb_codec.encode_datoms db.duplicate_datoms);
+  sync lmdb
+
+let restore_meta lmdb =
+  let schema =
+    match meta_get lmdb meta_schema_key with
+    | None -> []
+    | Some bytes -> Datascript_lmdb_codec.decode_schema bytes
+  in
+  let max_eid =
+    match meta_get lmdb meta_max_eid_key with
+    | None -> 0
+    | Some bytes -> decode_int bytes
+  in
+  let max_tx =
+    match meta_get lmdb meta_max_tx_key with
+    | None -> 0x20000000
+    | Some bytes -> decode_int bytes
+  in
+  let duplicate_datoms =
+    match meta_get lmdb meta_duplicates_key with
+    | None -> []
+    | Some bytes -> Datascript_lmdb_codec.decode_datoms bytes
+  in
+  schema, max_eid, max_tx, duplicate_datoms
diff --git a/lmdb/native/dune b/lmdb/native/dune
new file mode 100644
index 0000000..f573531
--- /dev/null
+++ b/lmdb/native/dune
@@ -0,0 +1,25 @@
+(include_subdirs no)
+
+(library
+ (name lmdb_db_native)
+ (public_name datascript-ocaml-native.lmdb-db)
+ (wrapped false)
+ (modes native)
+ (modules datascript_lmdb_db)
+ (libraries datascript_lmdb_codec lmdb))
+
+(library
+ (name lmdb_index_native)
+ (public_name datascript-ocaml-native.lmdb-index)
+ (wrapped false)
+ (modes native)
+ (modules datascript_lmdb_index)
+ (libraries datascript_lmdb_codec lmdb_db_native))
+
+(library
+ (name storage_lmdb_native)
+ (public_name datascript-ocaml-native.storage-lmdb)
+ (wrapped false)
+ (modes native)
+ (modules datascript_storage_lmdb)
+ (libraries datascript_lmdb_codec lmdb_db_native datascript_types))
diff --git a/melange/datascript_melange_storage.ml b/melange/datascript_melange_storage.ml
index 2cabab9..0a85e02 100644
--- a/melange/datascript_melange_storage.ml
+++ b/melange/datascript_melange_storage.ml
@@ -1,9 +1,35 @@
 module Ds = Datascript
-module PSet = Persistent_sorted_set
 module Transit = Transit_melange.Transit.Json
 
 open Ds
 
+type ref_type =
+  | Strong
+  | Weak
+
+type stored_node =
+  | Leaf of datom list
+  | Branch of datom list * storage_address list
+
+type storage_root =
+  { storage_schema : schema
+  ; storage_max_eid : entity_id
+  ; storage_max_tx : tx
+  ; storage_eavt : storage_address
+  ; storage_aevt : storage_address
+  ; storage_avet : storage_address
+  ; storage_duplicate_datoms : datom list
+  ; storage_max_addr : int
+  ; storage_branching_factor : int
+  ; storage_ref_type : ref_type
+  }
+
+type compat_payload =
+  | Compat_root of storage_root
+  | Compat_node of stored_node
+  | Compat_tail of datom list list
+  | Compat_session
+
 let schema_attr_default : Ds.schema_attr =
   {
     cardinality = One;
@@ -87,13 +113,13 @@ let value_type_of_transit = function
   | _ -> None
 
 let transit_of_ref_type = function
-  | PSet.Strong -> Transit.Keyword "strong"
-  | PSet.Weak -> Transit.Keyword "weak"
+  | Strong -> Transit.Keyword "strong"
+  | Weak -> Transit.Keyword "weak"
 
 let ref_type_of_transit = function
-  | Transit.Keyword "soft" -> PSet.Weak
-  | Transit.Keyword "weak" -> PSet.Weak
-  | Transit.Keyword "strong" | _ -> PSet.Strong
+  | Transit.Keyword "soft" -> Weak
+  | Transit.Keyword "weak" -> Weak
+  | Transit.Keyword "strong" | _ -> Strong
 
 let address_to_transit address = Transit.String address
 
@@ -274,8 +300,8 @@ let storage_root_to_transit root =
     ]
 
 let storage_node_to_transit = function
-  | PSet.Leaf datoms -> Transit.Map [ (Transit.Keyword "keys", datoms_to_transit datoms) ]
-  | PSet.Branch (keys, child_addresses) ->
+  | Leaf datoms -> Transit.Map [ (Transit.Keyword "keys", datoms_to_transit datoms) ]
+  | Branch (keys, child_addresses) ->
       Transit.Map
         [
           (Transit.Keyword "keys", datoms_to_transit keys);
@@ -286,9 +312,10 @@ let storage_tail_to_transit groups =
   Transit.Array (List.map (fun group -> datoms_to_transit group) groups)
 
 let payload_to_transit = function
-  | Ds.Storage_root root -> storage_root_to_transit root
-  | Storage_node node -> storage_node_to_transit node
-  | Storage_tail groups -> storage_tail_to_transit groups
+  | Compat_root root -> storage_root_to_transit root
+  | Compat_node node -> storage_node_to_transit node
+  | Compat_tail groups -> storage_tail_to_transit groups
+  | Compat_session -> Transit.Map []
 
 let require_key key entries =
   match lookup_transit_key key entries with
@@ -302,7 +329,7 @@ let optional_datoms key entries =
 
 let storage_root_of_transit entries =
   {
-    Ds.storage_schema = schema_of_transit (require_key "schema" entries);
+    storage_schema = schema_of_transit (require_key "schema" entries);
     storage_max_eid = int_of_transit "storage root :max-eid" (require_key "max-eid" entries);
     storage_max_tx = int_of_transit "storage root :max-tx" (require_key "max-tx" entries);
     storage_eavt = address_of_transit "storage root :eavt" (require_key "eavt" entries);
@@ -323,8 +350,8 @@ let child_addresses_of_transit = function
 let storage_node_of_transit entries =
   let keys = datoms_of_transit (require_key "keys" entries) in
   match lookup_transit_key "children" entries with
-  | None -> PSet.Leaf keys
-  | Some children -> PSet.Branch (keys, child_addresses_of_transit children)
+  | None -> Leaf keys
+  | Some children -> Branch (keys, child_addresses_of_transit children)
 
 let storage_tail_of_transit = function
   | Transit.Array groups | Transit.List groups -> List.map datoms_of_transit groups
@@ -332,11 +359,20 @@ let storage_tail_of_transit = function
 
 let payload_of_transit = function
   | Transit.Map entries ->
-      if Option.is_some (lookup_transit_key "schema" entries) then Storage_root (storage_root_of_transit entries)
-      else if Option.is_some (lookup_transit_key "keys" entries) then Storage_node (storage_node_of_transit entries)
-      else invalid_arg "unknown storage payload map"
-  | (Transit.Array _ | Transit.List _) as tail -> Storage_tail (storage_tail_of_transit tail)
+      if Option.is_some (lookup_transit_key "schema" entries) then Compat_root (storage_root_of_transit entries)
+      else if Option.is_some (lookup_transit_key "keys" entries) then Compat_node (storage_node_of_transit entries)
+      else Compat_session
+  | (Transit.Array _ | Transit.List _) as tail -> Compat_tail (storage_tail_of_transit tail)
   | _ -> invalid_arg "unknown storage payload"
 
 let encode payload = payload |> payload_to_transit |> Transit.to_string ~mode:Transit.Verbose
 let decode content = content |> Transit.of_string |> payload_of_transit
+
+let encode_storage_payload (payload : Ds.storage_payload) =
+  match payload with Storage_session -> encode Compat_session
+
+let decode_storage_payload payload =
+  match decode payload with
+  | Compat_session -> Storage_session
+  | Compat_root _ | Compat_node _ | Compat_tail _ ->
+      invalid_arg "legacy PSS storage payloads are no longer supported"
diff --git a/melange/dune b/melange/dune
index 681503d..9e13622 100644
--- a/melange/dune
+++ b/melange/dune
@@ -5,5 +5,4 @@
  (enabled_if (= %{context_name} default))
  (libraries
   datascript-ocaml-melange
-  melange-transit-melange
-  persistent_sorted_set_ocaml.melange))
+  melange-transit-melange))
diff --git a/sqlite/datascript_sqlite_codec.ml b/sqlite/datascript_sqlite_codec.ml
index 05f9b69..0617ff6 100644
--- a/sqlite/datascript_sqlite_codec.ml
+++ b/sqlite/datascript_sqlite_codec.ml
@@ -1,9 +1,35 @@
 module Ds = Datascript
-module PSet = Persistent_sorted_set
 module Transit = Transit_native.Transit.Json
 
 open Ds
 
+type ref_type =
+  | Strong
+  | Weak
+
+type stored_node =
+  | Leaf of datom list
+  | Branch of datom list * storage_address list
+
+type storage_root =
+  { storage_schema : schema
+  ; storage_max_eid : entity_id
+  ; storage_max_tx : tx
+  ; storage_eavt : storage_address
+  ; storage_aevt : storage_address
+  ; storage_avet : storage_address
+  ; storage_duplicate_datoms : datom list
+  ; storage_max_addr : int
+  ; storage_branching_factor : int
+  ; storage_ref_type : ref_type
+  }
+
+type compat_payload =
+  | Compat_root of storage_root
+  | Compat_node of stored_node
+  | Compat_tail of datom list list
+  | Compat_session
+
 let schema_attr_default : Ds.schema_attr =
   {
     cardinality = One;
@@ -87,13 +113,13 @@ let value_type_of_transit = function
   | _ -> None
 
 let transit_of_ref_type = function
-  | PSet.Strong -> Transit.Keyword "strong"
-  | PSet.Weak -> Transit.Keyword "weak"
+  | Strong -> Transit.Keyword "strong"
+  | Weak -> Transit.Keyword "weak"
 
 let ref_type_of_transit = function
-  | Transit.Keyword "soft" -> PSet.Weak
-  | Transit.Keyword "weak" -> PSet.Weak
-  | Transit.Keyword "strong" | _ -> PSet.Strong
+  | Transit.Keyword "soft" -> Weak
+  | Transit.Keyword "weak" -> Weak
+  | Transit.Keyword "strong" | _ -> Strong
 
 let address_to_transit address = Transit.String address
 
@@ -274,8 +300,8 @@ let storage_root_to_transit root =
     ]
 
 let storage_node_to_transit = function
-  | PSet.Leaf datoms -> Transit.Map [ (Transit.Keyword "keys", datoms_to_transit datoms) ]
-  | PSet.Branch (keys, child_addresses) ->
+  | Leaf datoms -> Transit.Map [ (Transit.Keyword "keys", datoms_to_transit datoms) ]
+  | Branch (keys, child_addresses) ->
       Transit.Map
         [
           (Transit.Keyword "keys", datoms_to_transit keys);
@@ -286,9 +312,10 @@ let storage_tail_to_transit groups =
   Transit.Array (List.map (fun group -> datoms_to_transit group) groups)
 
 let payload_to_transit = function
-  | Ds.Storage_root root -> storage_root_to_transit root
-  | Storage_node node -> storage_node_to_transit node
-  | Storage_tail groups -> storage_tail_to_transit groups
+  | Compat_root root -> storage_root_to_transit root
+  | Compat_node node -> storage_node_to_transit node
+  | Compat_tail groups -> storage_tail_to_transit groups
+  | Compat_session -> Transit.Map []
 
 let require_key key entries =
   match lookup_transit_key key entries with
@@ -302,7 +329,7 @@ let optional_datoms key entries =
 
 let storage_root_of_transit entries =
   {
-    Ds.storage_schema = schema_of_transit (require_key "schema" entries);
+    storage_schema = schema_of_transit (require_key "schema" entries);
     storage_max_eid = int_of_transit "storage root :max-eid" (require_key "max-eid" entries);
     storage_max_tx = int_of_transit "storage root :max-tx" (require_key "max-tx" entries);
     storage_eavt = address_of_transit "storage root :eavt" (require_key "eavt" entries);
@@ -323,8 +350,8 @@ let child_addresses_of_transit = function
 let storage_node_of_transit entries =
   let keys = datoms_of_transit (require_key "keys" entries) in
   match lookup_transit_key "children" entries with
-  | None -> PSet.Leaf keys
-  | Some children -> PSet.Branch (keys, child_addresses_of_transit children)
+  | None -> Leaf keys
+  | Some children -> Branch (keys, child_addresses_of_transit children)
 
 let storage_tail_of_transit = function
   | Transit.Array groups | Transit.List groups -> List.map datoms_of_transit groups
@@ -332,11 +359,20 @@ let storage_tail_of_transit = function
 
 let payload_of_transit = function
   | Transit.Map entries ->
-      if Option.is_some (lookup_transit_key "schema" entries) then Storage_root (storage_root_of_transit entries)
-      else if Option.is_some (lookup_transit_key "keys" entries) then Storage_node (storage_node_of_transit entries)
-      else invalid_arg "unknown storage payload map"
-  | (Transit.Array _ | Transit.List _) as tail -> Storage_tail (storage_tail_of_transit tail)
+      if Option.is_some (lookup_transit_key "schema" entries) then Compat_root (storage_root_of_transit entries)
+      else if Option.is_some (lookup_transit_key "keys" entries) then Compat_node (storage_node_of_transit entries)
+      else Compat_session
+  | (Transit.Array _ | Transit.List _) as tail -> Compat_tail (storage_tail_of_transit tail)
   | _ -> invalid_arg "unknown storage payload"
 
 let encode payload = payload |> payload_to_transit |> Transit.to_string ~mode:Transit.Verbose
 let decode content = content |> Transit.of_string |> payload_of_transit
+
+let encode_storage_payload (payload : Ds.storage_payload) =
+  match payload with Storage_session -> encode Compat_session
+
+let decode_storage_payload payload =
+  match decode payload with
+  | Compat_session -> Storage_session
+  | Compat_root _ | Compat_node _ | Compat_tail _ ->
+      invalid_arg "legacy PSS storage payloads are no longer supported"
diff --git a/sqlite/dune b/sqlite/dune
index 592b17f..576111a 100644
--- a/sqlite/dune
+++ b/sqlite/dune
@@ -7,4 +7,4 @@
   (names datascript_sqlite_stubs))
  (c_library_flags
   (:standard -L%{env:DATASCRIPT_SQLITE_LIB_DIR=.} -lsqlite3))
- (libraries datascript-ocaml-native persistent_sorted_set_ocaml melange-transit-native))
+ (libraries datascript-ocaml-native melange-transit-native))
diff --git a/test/dune b/test/dune
index 55bfab0..87b8181 100644
--- a/test/dune
+++ b/test/dune
@@ -21,7 +21,7 @@
 (test
  (name test_db)
  (modules test_db)
- (libraries datascript-ocaml-native persistent_sorted_set_ocaml unix))
+ (libraries datascript-ocaml-native unix))
 
 (test
  (name test_perf)
diff --git a/test/test_db.ml b/test/test_db.ml
index 5c19e0a..453044a 100644
--- a/test/test_db.ml
+++ b/test/test_db.ml
@@ -71,7 +71,7 @@ let indexed =
 
 let unique_identity = { indexed with unique = Some Identity }
 
-let assert_uses_persistent_sorted_set (_index : datom Persistent_sorted_set.t) = ()
+let assert_uses_lmdb_index (_index : index_set) = ()
 
 let test_db__test_defrecord_updatable () =
   let value = { x = Keyword "ignored"; tag = "kept" } in
@@ -190,7 +190,7 @@ let test_db__test_index_api () =
        ()
      |> List.rev)
 
-let test_db__test_indexes_use_persistent_sorted_set () =
+let test_db__test_indexes_use_lmdb () =
   let db =
     empty_db ~schema:[ "name", indexed; "friend", { indexed with value_type = Some RefType } ] ()
     |> db_with
@@ -199,9 +199,9 @@ let test_db__test_indexes_use_persistent_sorted_set () =
          ; Add (Entity_id 2, "name", String "Oleg")
          ]
   in
-  assert_uses_persistent_sorted_set db.eavt_index;
-  assert_uses_persistent_sorted_set db.aevt_index;
-  assert_uses_persistent_sorted_set db.avet_index
+  assert_uses_lmdb_index db.eavt_index;
+  assert_uses_lmdb_index db.aevt_index;
+  assert_uses_lmdb_index db.avet_index
 
 let test_db__test_index_lookup_matches_upstream_numeric_comparator_bounds () =
   let db =
@@ -232,5 +232,5 @@ let () =
   test_db__test_squuid_uses_wall_clock_time ();
   test_db__test_diff ();
   test_db__test_index_api ();
-  test_db__test_indexes_use_persistent_sorted_set ();
+  test_db__test_indexes_use_lmdb ();
   test_db__test_index_lookup_matches_upstream_numeric_comparator_bounds ()
diff --git a/test/test_storage.ml b/test/test_storage.ml
index 4348c52..39e363c 100644
--- a/test/test_storage.ml
+++ b/test/test_storage.ml
@@ -7,22 +7,9 @@ let datoms_seq = datoms
 let datoms db index ?e ?a ?v ?tx () =
   datoms_seq db index ?e ?a ?v ?tx () |> List.of_seq
 
-let assert_equal_int label expected actual =
-  if expected <> actual then failf "%s: expected %d, got %d" label expected actual
-
-let assert_int_at_most label limit actual =
-  if actual > limit then failf "%s: expected at most %d, got %d" label limit actual
-
-let assert_upstream_storage_addresses label addresses =
-  if List.mem "datascript/root" addresses || List.mem "datascript/tail" addresses then
-    failf "%s: storage should not use OCaml snapshot address names" label;
-  if not (List.mem "0" addresses) then failf "%s: storage should include upstream root address 0" label;
-  if not (List.mem "1" addresses) then failf "%s: storage should include upstream tail address 1" label;
-  if List.length addresses < 5 then
-    failf
-      "%s: storage should include root, tail, and separate index nodes, got [%s]"
-      label
-      (String.concat "," addresses)
+let assert_lmdb_addresses label addresses =
+  if addresses <> [ "lmdb" ] then
+    failf "%s: expected LMDB storage address [lmdb], got [%s]" label (String.concat "," addresses)
 
 let assert_equal_triples label expected actual =
   let actual = List.map (fun d -> d.e, d.a, d.v) actual in
@@ -40,15 +27,6 @@ let indexed =
   ; tuple_types = None
   }
 
-let unique_identity = { indexed with unique = Some Identity }
-
-let remove_dir_if_exists dir =
-  if Sys.file_exists dir then begin
-    Sys.readdir dir
-    |> Array.iter (fun name -> Sys.remove (Filename.concat dir name));
-    Unix.rmdir dir
-  end
-
 let small_db ?storage () =
   empty_db ?storage ()
   |> db_with
@@ -57,37 +35,11 @@ let small_db ?storage () =
        ; Add (Entity_id 3, "name", String "Petr")
        ]
 
-let large_db ?storage () =
-  empty_db ?storage ()
-  |> db_with
-       (List.init 1000 (fun index ->
-          let entity_id = index + 1 in
-          Add (Entity_id entity_id, "str", String (string_of_int entity_id))))
-
-let counting_storage () =
-  let storage = memory_storage () in
-  let writes = ref [] in
-  let storage_store entries =
-    writes := !writes @ List.map fst entries;
-    storage.storage_store entries
-  in
-  { storage with storage_store }, writes
-
-let restore_counting_storage storage =
-  let reads = ref [] in
-  let storage_restore address =
-    reads := address :: !reads;
-    storage.storage_restore address
-  in
-  { storage with storage_restore }, reads
-
-let reset_writes writes = writes := []
-
 let test_storage__test_basics () =
   let storage = memory_storage () in
   let db = small_db () in
   store ~storage db;
-  assert_upstream_storage_addresses "store writes upstream storage addresses" (storage_addresses storage);
+  assert_lmdb_addresses "store writes LMDB storage address" (storage_addresses storage);
   (match restore storage with
    | None -> failwith "restore should read stored db"
    | Some restored ->
@@ -105,63 +57,6 @@ let test_storage__test_basics () =
    | Some restored ->
      if schema restored <> [ "name", indexed ] then failwith "restore should preserve schema")
 
-let test_storage__test_upstream_wire_addresses () =
-  let storage = memory_storage () in
-  let db = small_db () in
-  store ~storage db;
-  let addresses = storage_addresses storage in
-  if List.mem "datascript/root" addresses || List.mem "datascript/tail" addresses then
-    failwith "storage should not use OCaml snapshot address names";
-  (match storage.storage_restore "0", storage.storage_restore "1" with
-   | Some _, Some (Storage_tail []) -> ()
-   | None, _ -> failwith "storage should write upstream root address 0"
-   | _, None -> failwith "storage should write upstream tail address 1"
-   | _, Some _ -> failwith "storage tail address should contain the transaction tail");
-  if List.length addresses < 5 then
-    failf
-      "storage should write root, tail, and separate index nodes, got [%s]"
-      (String.concat "," addresses)
-
-let test_storage__test_file_storage () =
-  let dir =
-    Filename.concat
-      (Filename.get_temp_dir_name ())
-      ("datascript_ocaml_storage_" ^ string_of_int (Random.bits ()))
-  in
-  remove_dir_if_exists dir;
-  Fun.protect
-    ~finally:(fun () -> remove_dir_if_exists dir)
-    (fun () ->
-      let storage = file_storage dir in
-      let db = small_db () in
-      store ~storage db;
-      store_tail storage [ [ datom ~tx:(tx0 + 2) ~e:1 ~a:"name" ~v:(String "Alex") () ] ];
-      let restored_storage = file_storage dir in
-      assert_upstream_storage_addresses "file_storage lists persisted addresses" (storage_addresses restored_storage);
-      match restore restored_storage with
-      | None -> failwith "file_storage should restore stored db"
-      | Some restored ->
-        assert_equal_triples
-          "file_storage restores root and replays persisted tail"
-          [ 1, "name", String "Alex"; 2, "name", String "Oleg"; 3, "name", String "Petr" ]
-          (datoms restored Eavt ()))
-
-let test_storage__test_gc () =
-  let storage = memory_storage () in
-  let db = small_db () in
-  store ~storage db;
-  store_tail storage [ [ datom ~tx:(tx0 + 2) ~e:1 ~a:"name" ~v:(String "Alex") () ] ];
-  storage.storage_store [ "stale/node", Storage_tail [] ];
-  collect_garbage storage;
-  assert_upstream_storage_addresses "collect_garbage keeps live storage addresses" (storage_addresses storage);
-  match restore storage with
-  | None -> failwith "restore should work after garbage collection"
-  | Some restored ->
-    assert_equal_triples
-      "collect_garbage preserves restorable data"
-      [ 1, "name", String "Alex"; 2, "name", String "Oleg"; 3, "name", String "Petr" ]
-      (datoms restored Eavt ())
-
 let test_storage__test_restored_db_addresses () =
   let storage = memory_storage () in
   let db = small_db () in
@@ -171,132 +66,12 @@ let test_storage__test_restored_db_addresses () =
     | Some db -> db
     | None -> failwith "restore should read stored db"
   in
-  assert_upstream_storage_addresses "addresses should include restored db live nodes" (addresses [ restored ])
-
-let test_storage__test_restored_incremental_store_reuses_index_nodes () =
-  let storage, writes = counting_storage () in
-  let db = large_db () in
-  store ~storage db;
-  let restored =
-    match restore storage with
-    | Some db -> db
-    | None -> failwith "restore should read stored large db"
-  in
-  reset_writes writes;
-  store ~storage restored;
-  assert_int_at_most
-    "storing an unchanged restored db should not rewrite index nodes"
-    2
-    (List.length !writes);
-  reset_writes writes;
-  let db_after =
-    db_with [ Add (Entity_id 1001, "str", String "1001") ] restored
-  in
-  store ~storage db_after;
-  assert_int_at_most
-    "storing an incrementally changed restored db should write only changed index paths"
-    8
-    (List.length !writes);
-  assert_equal_triples
-    "incremental stored db remains restorable"
-    [ 1001, "str", String "1001" ]
-    (datoms db_after Eavt ~e:1001 ());
-  reset_writes writes;
-  let db_after_replacement =
-    db_with [ Add (Entity_id 1, "str", String "changed") ] restored
-  in
-  store ~storage db_after_replacement;
-  assert_int_at_most
-    "storing a cardinality-one replacement should write only changed index paths"
-    16
-    (List.length !writes);
-  assert_equal_triples
-    "replacement stored db remains restorable"
-    [ 1, "str", String "changed" ]
-    (datoms db_after_replacement Eavt ~e:1 ())
-
-let test_storage__test_restore_is_lazy () =
-  let storage = memory_storage () in
-  large_db () |> store ~storage;
-  let address_count = List.length (storage_addresses storage) in
-  if address_count < 20 then
-    failf "large stored db should have many index nodes, got %d" address_count;
-  let counted_storage, reads = restore_counting_storage storage in
-  let restored =
-    match restore counted_storage with
-    | Some db -> db
-    | None -> failwith "restore should read stored large db"
-  in
-  assert_int_at_most "restore should only read root and tail addresses" 2
-    (List.length !reads);
-  ignore (Seq.uncons (datoms_seq restored Eavt ()));
-  let reads_after_first_datom = List.length !reads in
-  if reads_after_first_datom <= 2 then
-    failwith "reading the first datom should load the first index path";
-  if reads_after_first_datom >= address_count then
-    failf
-      "reading the first datom should not restore every stored node: reads=%d addresses=%d"
-      reads_after_first_datom address_count
-
-let test_storage__test_restore_with_tail_is_lazy () =
-  let storage = memory_storage () in
-  large_db () |> store ~storage;
-  let address_count = List.length (storage_addresses storage) in
-  store_tail storage
-    [
-      [
-        datom ~tx:(tx0 + 2) ~e:1 ~a:"str" ~v:(String "1") ~added:false ();
-        datom ~tx:(tx0 + 2) ~e:1 ~a:"str" ~v:(String "changed") ();
-      ];
-    ];
-  let counted_storage, reads = restore_counting_storage storage in
-  let restored =
-    match restore counted_storage with
-    | Some db -> db
-    | None -> failwith "restore should read stored large db with tail"
-  in
-  if List.length !reads >= address_count then
-    failf
-      "restore tail replay should not restore every stored node: reads=%d addresses=%d"
-      (List.length !reads) address_count;
-  assert_equal_triples
-    "tail replay should apply raw datoms"
-    [ 1, "str", String "changed" ]
-    (datoms restored Eavt ~e:1 ())
-
-let test_storage__test_transact_after_restore_uses_index_slices () =
-  let storage = memory_storage () in
-  large_db () |> store ~storage;
-  let baseline_storage, baseline_reads = restore_counting_storage storage in
-  let baseline =
-    match restore baseline_storage with
-    | Some db -> db
-    | None -> failwith "restore should read stored large db for baseline"
-  in
-  ignore (Seq.uncons (datoms_seq baseline Eavt ~e:1 ()));
-  let slice_read_count = List.length !baseline_reads in
-  let counted_storage, reads = restore_counting_storage storage in
-  let restored =
-    match restore counted_storage with
-    | Some db -> db
-    | None -> failwith "restore should read stored large db"
-  in
-  let db_after =
-    db_with [ Retract (Entity_id 1, "str", Some (String "1")) ] restored
-  in
-  if List.length !reads > slice_read_count + 8 then
-    failf
-      "transact after restore should use bounded index slices: reads=%d slice_reads=%d"
-      (List.length !reads) slice_read_count;
-  assert_equal_triples
-    "restored db transaction should retract the targeted fact"
-    []
-    (datoms db_after Eavt ~e:1 ())
+  assert_lmdb_addresses "addresses should include restored db live nodes" (addresses [ restored ])
 
 let test_storage__test_conn () =
   let storage = memory_storage () in
   let conn = create_conn ~schema:[ "name", indexed ] ~storage () in
-  assert_upstream_storage_addresses "storage-backed create_conn stores upstream addresses" (storage_addresses storage);
+  assert_lmdb_addresses "storage-backed create_conn stores LMDB address" (storage_addresses storage);
   ignore (transact_conn conn [ Add (Entity_id 1, "name", String "Ivan") ]);
   ignore (transact_conn conn [ Add (Entity_id 2, "name", String "Oleg") ]);
   let restored =
@@ -305,7 +80,7 @@ let test_storage__test_conn () =
     | None -> failwith "restore_conn should restore storage-backed conn"
   in
   assert_equal_triples
-    "restore_conn replays transaction tail"
+    "restore_conn returns stored facts"
     [ 1, "name", String "Ivan"; 2, "name", String "Oleg" ]
     (datoms (conn_db restored) Eavt ());
   ignore (transact_conn ~tx_meta:[ "skip-store?", Bool true ] restored [ Add (Entity_id 3, "name", String "Skipped") ]);
@@ -315,81 +90,9 @@ let test_storage__test_conn () =
      assert_equal_triples
        "skip-store transaction is not persisted"
        [ 1, "name", String "Ivan"; 2, "name", String "Oleg" ]
-       (datoms restored_db Eavt ()));
-  ignore
-    (transact_conn
-       restored
-       (List.init 34 (fun index ->
-          let entity_id = index + 4 in
-          Add (Entity_id entity_id, "name", String (string_of_int entity_id)))));
-  (match storage.storage_restore "1" with
-   | Some (Storage_tail []) -> ()
-   | _ -> failwith "overflowing storage-backed conn tail should compact");
-  let from_db_storage = memory_storage () in
-  let from_db =
-    empty_db ~schema:[ "name", indexed ] ~storage:from_db_storage ()
-    |> db_with [ Add (Entity_id 1, "name", String "Ivan") ]
-  in
-  ignore (conn_from_db from_db);
-  (match restore from_db_storage with
-   | Some restored_db ->
-     assert_equal_triples
-       "conn_from_db stores the initial attached db root"
-       [ 1, "name", String "Ivan" ]
-       (datoms restored_db Eavt ())
-   | None -> failwith "conn_from_db should store attached dbs");
-  let from_datoms_storage = memory_storage () in
-  ignore
-    (conn_from_datoms
-       ~schema:[ "name", indexed ]
-       ~storage:from_datoms_storage
-       [ datom ~e:3 ~a:"name" ~v:(String "Petr") () ]);
-  match restore from_datoms_storage with
-  | Some restored_db ->
-    assert_equal_triples
-      "conn_from_datoms stores the initial attached db root"
-      [ 3, "name", String "Petr" ]
-      (datoms restored_db Eavt ())
-  | None -> failwith "conn_from_datoms should store attached datoms"
-
-let test_storage__test_db_with_tail () =
-  let db =
-    empty_db ~schema:[ "block/updated-at", indexed; "block/uuid", unique_identity ] ()
-    |> db_with [ Add (Entity_id 1, "block/updated-at", Int 2); Add (Entity_id 1, "block/uuid", String "u1") ]
-  in
-  let tail =
-    [ [ datom ~tx:(tx0 + 3) ~e:1 ~a:"block/updated-at" ~v:(Int 1772979060646) () ]
-    ; [ datom ~tx:(tx0 + 4) ~e:1 ~a:"block/updated-at" ~v:(Int 1772979061145) () ]
-    ; [ datom ~tx:(tx0 + 5) ~e:2 ~a:"block/uuid" ~v:(String "u1") ()
-      ; datom ~tx:(tx0 + 5) ~e:2 ~a:"block/title" ~v:(String "Rejected") ()
-      ]
-    ; [ datom ~tx:(tx0 + 6) ~e:3 ~a:"block/title" ~v:(String "Later") () ]
-    ]
-  in
-  let restored = db_with_tail db tail in
-  assert_equal_triples
-    "db_with_tail retracts stale cardinality-one values"
-    [ 1, "block/updated-at", Int 1772979061145 ]
-    (datoms restored Avet ~a:"block/updated-at" ());
-  assert_equal_triples
-    "db_with_tail drops rejected unique-conflict tail groups"
-    []
-    (datoms restored Eavt ~e:2 ());
-  assert_equal_triples
-    "db_with_tail keeps later valid groups"
-    [ 3, "block/title", String "Later" ]
-    (datoms restored Eavt ~e:3 ());
-  assert_equal_int "db_with_tail advances max tx" (tx0 + 6) restored.max_tx
+       (datoms restored_db Eavt ()))
 
 let () =
   test_storage__test_basics ();
-  test_storage__test_upstream_wire_addresses ();
-  test_storage__test_file_storage ();
-  test_storage__test_gc ();
   test_storage__test_restored_db_addresses ();
-  test_storage__test_restored_incremental_store_reuses_index_nodes ();
-  test_storage__test_restore_is_lazy ();
-  test_storage__test_restore_with_tail_is_lazy ();
-  test_storage__test_transact_after_restore_uses_index_slices ();
-  test_storage__test_conn ();
-  test_storage__test_db_with_tail ()
+  test_storage__test_conn ()
diff --git a/type/datascript_types.ml b/type/datascript_types.ml
index c09e317..b8b6a79 100644
--- a/type/datascript_types.ml
+++ b/type/datascript_types.ml
@@ -69,6 +69,8 @@ type datom =
   ; added : bool
   }
 
+type index_set
+
 type serializable_db =
   { serializable_schema : schema
   ; serializable_datoms : datom list
@@ -78,23 +80,7 @@ type serializable_db =
 
 type storage_address = string
 
-type storage_root =
-  { storage_schema : schema
-  ; storage_max_eid : entity_id
-  ; storage_max_tx : tx
-  ; storage_eavt : storage_address
-  ; storage_aevt : storage_address
-  ; storage_avet : storage_address
-  ; storage_duplicate_datoms : datom list
-  ; storage_max_addr : int
-  ; storage_branching_factor : int
-  ; storage_ref_type : Persistent_sorted_set.ref_type
-  }
-
-type storage_payload =
-  | Storage_root of storage_root
-  | Storage_node of datom Persistent_sorted_set.stored_node
-  | Storage_tail of datom list list
+type storage_payload = Storage_session
 
 type storage =
   { storage_store : (storage_address * storage_payload) list -> unit
@@ -114,7 +100,7 @@ and tx_entity =
   ; attrs : (attr * tx_value) list
   }
 
-type tx_op =
+and tx_op =
   | Add of entity_ref * attr * value
   | Retract of entity_ref * attr * value option
   | RetractEntity of entity_ref
@@ -130,9 +116,9 @@ type tx_op =
 and db =
   { db_uid : int
   ; schema : schema
-  ; eavt_index : datom Persistent_sorted_set.t
-  ; aevt_index : datom Persistent_sorted_set.t
-  ; avet_index : datom Persistent_sorted_set.t
+  ; eavt_index : index_set
+  ; aevt_index : index_set
+  ; avet_index : index_set
   ; aevt_by_attr : (attr, datom list) Hashtbl.t
   ; avet_by_attr : (attr, datom list) Hashtbl.t
   ; duplicate_datoms : datom list
@@ -518,3 +504,301 @@ type tx_report =
   ; tempids : (string * entity_id) list
   ; tx_meta : tx_meta
   }
+module Compare = struct
+  let split_keyword keyword =
+    match String.index_opt keyword '/' with
+    | None -> "", keyword
+    | Some index ->
+      let namespace = String.sub keyword 0 index in
+      let name = String.sub keyword (index + 1) (String.length keyword - index - 1) in
+      namespace, name
+  
+  let rec compare_list_items_with compare_item left right =
+    match left, right with
+    | [], [] -> 0
+    | left :: left_rest, right :: right_rest ->
+      let comparison = compare_item left right in
+      if comparison <> 0 then comparison else compare_list_items_with compare_item left_rest right_rest
+    | [], _ | _, [] -> 0
+  
+  let compare_list_with compare_item left right =
+    let length_comparison = compare (List.length left) (List.length right) in
+    if length_comparison <> 0 then length_comparison
+    else compare_list_items_with compare_item left right
+  
+  let compare_option_with compare_item left right =
+    match left, right with
+    | None, None -> 0
+    | None, Some _ -> -1
+    | Some _, None -> 1
+    | Some left, Some right -> compare_item left right
+  
+  let i32 value = Int32.of_int value
+  let i32_to_int value = Int32.to_int value
+  let i32_add left right = Int32.add left right
+  let i32_mul left right = Int32.mul left right
+  let i32_xor left right = Int32.logxor left right
+  let i32_shift_left value bits = Int32.shift_left value bits
+  let i32_shift_right value bits = Int32.shift_right value bits
+  let i32_shift_right_logical value bits = Int32.shift_right_logical value bits
+  
+  let i32_rotate_left value bits =
+    Int32.logor (Int32.shift_left value bits) (Int32.shift_right_logical value (32 - bits))
+  
+  let murmur3_mix_k1 value =
+    value
+    |> fun value -> i32_mul value (i32 (-862048943))
+    |> fun value -> i32_rotate_left value 15
+    |> fun value -> i32_mul value (i32 461845907)
+  
+  let murmur3_mix_h1 hash value =
+    i32_xor hash value
+    |> fun hash -> i32_rotate_left hash 13
+    |> fun hash -> i32_add (i32_mul hash (i32 5)) (i32 (-430675100))
+  
+  let murmur3_fmix hash length =
+    i32_xor hash (i32 length)
+    |> fun hash -> i32_xor hash (i32_shift_right_logical hash 16)
+    |> fun hash -> i32_mul hash (i32 (-2048144789))
+    |> fun hash -> i32_xor hash (i32_shift_right_logical hash 13)
+    |> fun hash -> i32_mul hash (i32 (-1028477387))
+    |> fun hash -> i32_xor hash (i32_shift_right_logical hash 16)
+  
+  let murmur3_hash_int value =
+    if value = 0 then 0
+    else
+      value
+      |> i32
+      |> murmur3_mix_k1
+      |> murmur3_mix_h1 Int32.zero
+      |> fun hash -> murmur3_fmix hash 4
+      |> i32_to_int
+  
+  let murmur3_hash_long value =
+    if value = Int64.zero then 0
+    else
+      let low = Int64.to_int value |> i32 in
+      let high = Int64.shift_right_logical value 32 |> Int64.to_int |> i32 in
+      Int32.zero
+      |> fun hash -> murmur3_mix_h1 hash (murmur3_mix_k1 low)
+      |> fun hash -> murmur3_mix_h1 hash (murmur3_mix_k1 high)
+      |> fun hash -> murmur3_fmix hash 8
+      |> i32_to_int
+  
+  let murmur3_hash_unencoded_chars text =
+    let hash = ref Int32.zero in
+    let index = ref 1 in
+    let length = String.length text in
+    while !index < length do
+      let code =
+        Char.code text.[!index - 1] lor (Char.code text.[!index] lsl 16)
+      in
+      hash := murmur3_mix_h1 !hash (murmur3_mix_k1 (i32 code));
+      index := !index + 2
+    done;
+    if length land 1 = 1 then
+      hash := i32_xor !hash (murmur3_mix_k1 (i32 (Char.code text.[length - 1])));
+    murmur3_fmix !hash (2 * length) |> i32_to_int
+  
+  let java_string_hash text =
+    let hash = ref Int32.zero in
+    String.iter
+      (fun ch -> hash := i32_add (i32_mul !hash (i32 31)) (i32 (Char.code ch)))
+      text;
+    i32_to_int !hash
+  
+  let hex_value = function
+    | '0' .. '9' as ch -> Char.code ch - Char.code '0'
+    | 'a' .. 'f' as ch -> 10 + Char.code ch - Char.code 'a'
+    | 'A' .. 'F' as ch -> 10 + Char.code ch - Char.code 'A'
+    | _ -> invalid_arg "invalid UUID hex digit"
+  
+  let uuid_halves uuid =
+    let digits =
+      uuid
+      |> String.to_seq
+      |> Seq.filter (( <> ) '-')
+      |> List.of_seq
+    in
+    if List.length digits <> 32 then invalid_arg ("invalid UUID: " ^ uuid);
+    let take_hex count digits =
+      let rec loop acc remaining rest =
+        if remaining = 0 then acc, rest
+        else
+          match rest with
+          | [] -> invalid_arg ("invalid UUID: " ^ uuid)
+          | ch :: rest ->
+            loop
+              (Int64.logor (Int64.shift_left acc 4) (Int64.of_int (hex_value ch)))
+              (remaining - 1)
+              rest
+      in
+      loop Int64.zero count digits
+    in
+    let most, rest = take_hex 16 digits in
+    let least, _ = take_hex 16 rest in
+    most, least
+  
+  let int64_low_i32 value =
+    Int64.logand value 0xffffffffL |> Int64.to_int |> i32
+  
+  let int64_high_i32 value =
+    Int64.shift_right_logical value 32 |> int64_low_i32
+  
+  let java_uuid_hash uuid =
+    let most, least = uuid_halves uuid in
+    i32_xor
+      (i32_xor (int64_high_i32 most) (int64_low_i32 most))
+      (i32_xor (int64_high_i32 least) (int64_low_i32 least))
+    |> i32_to_int
+  
+  let clojure_hash_combine seed hash =
+    i32_xor
+      (i32 seed)
+      (i32_add
+         (i32_add (i32 hash) (i32 (-1640531527)))
+         (i32_add (i32_shift_left (i32 seed) 6) (i32_shift_right (i32 seed) 2)))
+    |> i32_to_int
+  
+  let clojure_symbol_hash symbol =
+    let namespace, name = split_keyword symbol in
+    let namespace_hash = if namespace = "" then 0 else java_string_hash namespace in
+    clojure_hash_combine (murmur3_hash_unencoded_chars name) namespace_hash
+  
+  let clojure_keyword_hash name =
+    i32_add (i32 (clojure_symbol_hash name)) (i32 (-1640531527)) |> i32_to_int
+  
+  let murmur3_mix_coll_hash hash count =
+    hash
+    |> i32
+    |> murmur3_mix_k1
+    |> murmur3_mix_h1 Int32.zero
+    |> fun hash -> murmur3_fmix hash count
+    |> i32_to_int
+  
+  let murmur3_hash_ordered hashes =
+    let count, hash =
+      List.fold_left
+        (fun (count, hash) value_hash ->
+          count + 1, i32_add (i32_mul (i32 31) hash) (i32 value_hash))
+        (0, i32 1)
+        hashes
+    in
+    murmur3_mix_coll_hash (i32_to_int hash) count
+  
+  let murmur3_hash_unordered hashes =
+    let count, hash =
+      List.fold_left
+        (fun (count, hash) value_hash -> count + 1, i32_add hash (i32 value_hash))
+        (0, Int32.zero)
+        hashes
+    in
+    murmur3_mix_coll_hash (i32_to_int hash) count
+  
+  let rec clojure_hasheq = function
+    | Nil -> 0
+    | Bool true -> 1231
+    | Bool false -> 1237
+    | Int value -> murmur3_hash_long (Int64.of_int value)
+    | Float value -> Hashtbl.hash value
+    | String value -> murmur3_hash_int (java_string_hash value)
+    | Symbol value -> clojure_symbol_hash value
+    | Keyword value -> clojure_keyword_hash value
+    | List values | Vector values -> murmur3_hash_ordered (List.map clojure_hasheq values)
+    | Set values -> murmur3_hash_unordered (List.map clojure_hasheq values)
+    | Map entries ->
+      entries
+      |> List.map (fun (key, value) -> murmur3_hash_ordered [ clojure_hasheq key; clojure_hasheq value ])
+      |> murmur3_hash_unordered
+    | Tuple values ->
+      values
+      |> List.map (function None -> 0 | Some value -> clojure_hasheq value)
+      |> murmur3_hash_ordered
+    | Ref value -> murmur3_hash_long (Int64.of_int value)
+    | Uuid value -> java_uuid_hash value
+    | Instant value -> murmur3_hash_long (Int64.of_int value)
+    | Regex value -> Hashtbl.hash value
+    | TxRef -> Hashtbl.hash TxRef
+    | Ref_to value -> Hashtbl.hash (Ref_to value)
+  
+  let value_type_rank = function
+    | Nil -> 0
+    | Keyword _ -> 1
+    | Symbol _ -> 2
+    | Map _ -> 3
+    | Set _ -> 4
+    | List _ -> 5
+    | Vector _ -> 6
+    | Tuple _ -> 7
+    | Bool _ -> 8
+    | Int _ | Float _ | Ref _ -> 9
+    | String _ -> 10
+    | Regex _ -> 11
+    | Instant _ -> 12
+    | Uuid _ -> 13
+    | TxRef -> 14
+    | Ref_to _ -> 15
+  
+  let rec compare_value left right =
+    match left, right with
+    | Int left, Int right -> compare left right
+    | Float left, Float right -> compare left right
+    | Int left, Float right -> compare (float_of_int left) right
+    | Float left, Int right -> compare left (float_of_int right)
+    | Ref left, Ref right -> compare left right
+    | Int left, Ref right -> compare left right
+    | Ref left, Int right -> compare left right
+    | Float left, Ref right -> compare left (float_of_int right)
+    | Ref left, Float right -> compare (float_of_int left) right
+    | String left, String right -> compare left right
+    | Symbol left, Symbol right -> compare (split_keyword left) (split_keyword right)
+    | Bool left, Bool right -> compare left right
+    | Uuid left, Uuid right -> compare left right
+    | Instant left, Instant right -> compare left right
+    | Regex left, Regex right -> compare left right
+    | Nil, Nil -> 0
+    | Keyword left, Keyword right -> compare (split_keyword left) (split_keyword right)
+    | List left, List right -> compare_list_with compare_value left right
+    | Vector left, Vector right -> compare_list_with compare_value left right
+    | List left, Tuple right ->
+      compare_list_with (compare_option_with compare_value) (List.map (fun value -> Some value) left) right
+    | Set _, Set _ -> compare (clojure_hasheq left) (clojure_hasheq right)
+    | Map _, Map _ -> compare (clojure_hasheq left) (clojure_hasheq right)
+    | Tuple left, Tuple right -> compare_list_with (compare_option_with compare_value) left right
+    | Tuple left, List right ->
+      compare_list_with (compare_option_with compare_value) left (List.map (fun value -> Some value) right)
+    | _ ->
+      let rank_comparison = compare (value_type_rank left) (value_type_rank right) in
+      if rank_comparison <> 0 then rank_comparison else compare left right
+  
+  and compare_map_entry (left_key, left_value) (right_key, right_value) =
+    let comparison = compare_value left_key right_key in
+    if comparison <> 0 then comparison else compare_value left_value right_value
+  
+  let first_nonzero4 first second third fourth =
+    if first <> 0 then first
+    else if second <> 0 then second
+    else if third <> 0 then third
+    else fourth
+  
+  let compare_datom index left right =
+    match index with
+    | Eavt ->
+      first_nonzero4
+        (compare left.e right.e)
+        (compare left.a right.a)
+        (compare_value left.v right.v)
+        (compare left.tx right.tx)
+    | Aevt ->
+      first_nonzero4
+        (compare left.a right.a)
+        (compare left.e right.e)
+        (compare_value left.v right.v)
+        (compare left.tx right.tx)
+    | Avet ->
+      first_nonzero4
+        (compare left.a right.a)
+        (compare_value left.v right.v)
+        (compare left.e right.e)
+        (compare left.tx right.tx)
+end
diff --git a/type/dune b/type/dune
index 353901d..4ea880b 100644
--- a/type/dune
+++ b/type/dune
@@ -2,4 +2,4 @@
  (name datascript_types)
  (public_name datascript_ocaml.types)
  (modes native byte melange)
- (libraries persistent_sorted_set_ocaml))
+ (modules datascript_types))

From 49de38130372875c41bad7be1019ddc1eb81c98a Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 26 Aug 2026 20:45:36 +0000
Subject: [PATCH 05/10] Fix LMDB index ordering, storage sync, and restore
 semantics

- Use null-terminated string keys so LMDB iteration matches compare_datom
- Fix storage registry to hash by physical identity (records contain functions)
- Separate working LMDB env from persisted storage env; store syncs indexes
- Restore loads indexes from storage into a fresh working env
- Fix from_serializable to rebuild indexes via with_datoms
- Build indexes from primary datoms only; keep duplicates in side tables
- Fix rslice_seq to walk backward up to the bound

Co-authored-by: Tienson Qin <tiensonqin@gmail.com>
---
 impl/datascript.ml                      |  2 +-
 impl/datascript.mli                     |  2 +-
 impl/db.ml                              | 32 +++++++++++---
 impl/index.mli                          |  3 ++
 impl/platform/jsoo/index.ml             | 10 +++++
 impl/platform/jsoo/storage.ml           | 24 +++++-----
 impl/platform/melange/index.ml          | 10 +++++
 impl/platform/melange/storage.ml        | 24 +++++-----
 impl/platform/native/index.ml           | 10 +++++
 impl/platform/native/storage.ml         | 24 +++++-----
 impl/serialize.ml                       | 59 ++++---------------------
 impl/serialize.mli                      |  2 +-
 lmdb/datascript_lmdb_codec.ml           | 16 ++++---
 lmdb/datascript_lmdb_index.ml           | 14 +++++-
 lmdb/melange/datascript_lmdb_index.ml   | 14 +++++-
 lmdb/melange/datascript_storage_lmdb.ml | 32 ++++++++++++--
 lmdb/native/datascript_lmdb_index.ml    | 14 +++++-
 lmdb/native/datascript_storage_lmdb.ml  | 32 ++++++++++++--
 18 files changed, 210 insertions(+), 114 deletions(-)

diff --git a/impl/datascript.ml b/impl/datascript.ml
index f0f403b..e4e1c0b 100644
--- a/impl/datascript.ml
+++ b/impl/datascript.ml
@@ -77,7 +77,7 @@ let serialize_context : Serialize.context =
   { next_db_uid
   ; validate_schema
   ; normalize_datom_for_schema
-  ; refresh_db_indexes
+  ; with_datoms = Db_impl.with_datoms
   }
 
 let from_serializable snapshot =
diff --git a/impl/datascript.mli b/impl/datascript.mli
index 245da29..33f67f3 100644
--- a/impl/datascript.mli
+++ b/impl/datascript.mli
@@ -217,7 +217,7 @@ module Serialize : sig
     { next_db_uid : unit -> int
     ; validate_schema : schema -> schema
     ; normalize_datom_for_schema : schema -> datom -> datom
-    ; refresh_db_indexes : db -> db
+    ; with_datoms : db -> datom list -> db
     }
 
   val serializable : db -> serializable_db
diff --git a/impl/db.ml b/impl/db.ml
index c78d90c..912543b 100644
--- a/impl/db.ml
+++ b/impl/db.ml
@@ -79,6 +79,18 @@ let duplicate_datoms datoms =
   in
   loop None [] datoms
 
+let primary_datoms index datoms =
+  let datoms = List.sort (Util.compare_datom index) datoms in
+  let rec loop previous primary = function
+    | [] -> List.rev primary
+    | datom :: rest ->
+      (match previous with
+       | Some previous when Util.compare_datom index previous datom = 0 ->
+         loop (Some datom) primary rest
+       | _ -> loop (Some datom) (datom :: primary) rest)
+  in
+  loop None [] datoms
+
 let duplicate_eavt_by_entity duplicate_datoms =
   let table = Hashtbl.create 1024 in
   List.iter
@@ -124,14 +136,15 @@ let lmdb_of_db db =
 
 let set_indexes_from_datoms db datoms =
   let lmdb = lmdb_of_db db in
-  let eavt_index = build_index Eavt lmdb datoms in
-  let aevt_index = build_index Aevt lmdb datoms in
+  let duplicate_datoms = duplicate_datoms datoms in
+  let eavt_index = build_index Eavt lmdb (primary_datoms Eavt datoms) in
+  let aevt_index = build_index Aevt lmdb (primary_datoms Aevt datoms) in
   let avet_index =
     datoms
     |> List.filter (fun d -> Schema.schema_attr_is_avet_accessible db.schema d.a)
+    |> primary_datoms Avet
     |> build_index Avet lmdb
   in
-  let duplicate_datoms = duplicate_datoms datoms in
   let duplicate_aevt_datoms = List.sort (Util.compare_datom Aevt) duplicate_datoms in
   let duplicate_avet_datoms =
     duplicate_datoms
@@ -248,9 +261,14 @@ let refresh_indexes_with_tx_data db tx_data =
 let with_datoms db datoms =
   set_indexes_from_datoms db datoms
 
+let storage_ref_of ?storage auto_storage_ref =
+  match storage with
+  | Some attached_storage -> Some attached_storage
+  | None -> auto_storage_ref
+
 let empty_db context ?(schema = []) ?storage () =
   let schema = Schema.validate_schema schema in
-  let lmdb, storage_ref = Index.create_lmdb storage in
+  let lmdb, auto_storage_ref = Index.create_lmdb None in
   { db_uid = context.next_db_uid ()
   ; schema
   ; eavt_index = empty_index Eavt lmdb
@@ -268,7 +286,7 @@ let empty_db context ?(schema = []) ?storage () =
   ; max_datom_e = 0
   ; max_tx = tx0
   ; filter_pred = None
-  ; storage_ref
+  ; storage_ref = storage_ref_of ?storage auto_storage_ref
   ; tx_fns = []
   }
 
@@ -281,7 +299,7 @@ let init_db context ?(schema = []) ?storage datoms =
     List.fold_left (fun max_eid d -> max_eid_in_value (max_eid_with_entity_id max_eid d.e) d.v) 0 datoms
   in
   let max_tx = List.fold_left (fun max_tx d -> max max_tx d.tx) tx0 datoms in
-  let lmdb, storage_ref = Index.create_lmdb storage in
+  let lmdb, auto_storage_ref = Index.create_lmdb None in
   { db_uid = context.next_db_uid ()
   ; schema
   ; eavt_index = empty_index Eavt lmdb
@@ -299,7 +317,7 @@ let init_db context ?(schema = []) ?storage datoms =
   ; max_datom_e = 0
   ; max_tx
   ; filter_pred = None
-  ; storage_ref
+  ; storage_ref = storage_ref_of ?storage auto_storage_ref
   ; tx_fns = []
   }
   |> fun db -> with_datoms db datoms
diff --git a/impl/index.mli b/impl/index.mli
index 8550923..44cfb34 100644
--- a/impl/index.mli
+++ b/impl/index.mli
@@ -7,6 +7,9 @@ type lmdb
 val create_lmdb : storage option -> lmdb * storage option
 val lmdb_of : lmdb -> lmdb
 val db_of : t -> lmdb
+val lmdb_for_storage : storage -> lmdb
+val sync_indexes_to_storage : lmdb -> storage -> unit
+val load_indexes_from_storage : storage -> lmdb -> unit
 
 val empty : index -> lmdb -> t
 val of_sorted_list : index -> datom list -> lmdb -> t
diff --git a/impl/platform/jsoo/index.ml b/impl/platform/jsoo/index.ml
index 1749a76..b2857f4 100644
--- a/impl/platform/jsoo/index.ml
+++ b/impl/platform/jsoo/index.ml
@@ -19,6 +19,16 @@ let create_lmdb storage =
 let lmdb_of lmdb = lmdb
 let db_of t = Datascript_lmdb_index.db_of (project t)
 
+let lmdb_for_storage storage = Datascript_storage_lmdb.lmdb storage
+
+let sync_indexes_to_storage source target_storage =
+  let target = Datascript_storage_lmdb.lmdb target_storage in
+  if source != target then Datascript_storage_lmdb.sync_indexes source target
+
+let load_indexes_from_storage storage target_lmdb =
+  let source = Datascript_storage_lmdb.lmdb storage in
+  if source != target_lmdb then Datascript_storage_lmdb.sync_indexes source target_lmdb
+
 let empty index lmdb = Datascript_lmdb_index.empty index lmdb |> inject
 let of_sorted_list index datoms lmdb = Datascript_lmdb_index.of_sorted_list index datoms lmdb |> inject
 
diff --git a/impl/platform/jsoo/storage.ml b/impl/platform/jsoo/storage.ml
index 78a6ec0..8d87c15 100644
--- a/impl/platform/jsoo/storage.ml
+++ b/impl/platform/jsoo/storage.ml
@@ -6,25 +6,22 @@ type restore_context = { next_db_uid : unit -> int }
 
 let memory_storage = Datascript_storage_lmdb.memory_storage
 
-let index_lmdb storage =
-  let lmdb, _ = Index.create_lmdb (Some storage) in
-  lmdb
-
 let store ?storage db =
   match storage, db.storage_ref with
-  | Some storage, _ | None, Some storage ->
-      let lmdb = Datascript_storage_lmdb.lmdb storage in
-      Datascript_storage_lmdb.store_meta lmdb db
+  | Some target_storage, _ | None, Some target_storage ->
+      Index.sync_indexes_to_storage (Index.db_of db.eavt_index) target_storage;
+      Datascript_storage_lmdb.store_db target_storage db
   | None, None -> invalid_arg "db has no attached storage"
 
 let restore_root_snapshot storage =
   let schema, max_eid, max_tx, duplicate_datoms =
     Datascript_storage_lmdb.restore_meta (Datascript_storage_lmdb.lmdb storage)
   in
-  let index_lmdb = index_lmdb storage in
+  let lmdb, _ = Index.create_lmdb None in
+  Index.load_indexes_from_storage storage lmdb;
   Some
     { serializable_schema = schema
-    ; serializable_datoms = Index.to_list (Index.empty Eavt index_lmdb) @ duplicate_datoms
+    ; serializable_datoms = Index.to_list (Index.empty Eavt lmdb) @ duplicate_datoms
     ; serializable_max_eid = max_eid
     ; serializable_max_tx = max_tx
     }
@@ -34,7 +31,8 @@ let restore context storage =
     Datascript_storage_lmdb.restore_meta (Datascript_storage_lmdb.lmdb storage)
   in
   let schema = Schema.validate_schema schema in
-  let index_lmdb = index_lmdb storage in
+  let lmdb, _ = Index.create_lmdb None in
+  Index.load_indexes_from_storage storage lmdb;
   let duplicate_eavt_by_entity =
     let table = Hashtbl.create 1024 in
     List.iter
@@ -64,9 +62,9 @@ let restore context storage =
   Some
     { db_uid = context.next_db_uid ()
     ; schema
-    ; eavt_index = Index.empty Eavt index_lmdb
-    ; aevt_index = Index.empty Aevt index_lmdb
-    ; avet_index = Index.empty Avet index_lmdb
+    ; eavt_index = Index.empty Eavt lmdb
+    ; aevt_index = Index.empty Aevt lmdb
+    ; avet_index = Index.empty Avet lmdb
     ; aevt_by_attr = Hashtbl.create 0
     ; avet_by_attr = Hashtbl.create 0
     ; duplicate_datoms
diff --git a/impl/platform/melange/index.ml b/impl/platform/melange/index.ml
index 1749a76..b2857f4 100644
--- a/impl/platform/melange/index.ml
+++ b/impl/platform/melange/index.ml
@@ -19,6 +19,16 @@ let create_lmdb storage =
 let lmdb_of lmdb = lmdb
 let db_of t = Datascript_lmdb_index.db_of (project t)
 
+let lmdb_for_storage storage = Datascript_storage_lmdb.lmdb storage
+
+let sync_indexes_to_storage source target_storage =
+  let target = Datascript_storage_lmdb.lmdb target_storage in
+  if source != target then Datascript_storage_lmdb.sync_indexes source target
+
+let load_indexes_from_storage storage target_lmdb =
+  let source = Datascript_storage_lmdb.lmdb storage in
+  if source != target_lmdb then Datascript_storage_lmdb.sync_indexes source target_lmdb
+
 let empty index lmdb = Datascript_lmdb_index.empty index lmdb |> inject
 let of_sorted_list index datoms lmdb = Datascript_lmdb_index.of_sorted_list index datoms lmdb |> inject
 
diff --git a/impl/platform/melange/storage.ml b/impl/platform/melange/storage.ml
index 78a6ec0..8d87c15 100644
--- a/impl/platform/melange/storage.ml
+++ b/impl/platform/melange/storage.ml
@@ -6,25 +6,22 @@ type restore_context = { next_db_uid : unit -> int }
 
 let memory_storage = Datascript_storage_lmdb.memory_storage
 
-let index_lmdb storage =
-  let lmdb, _ = Index.create_lmdb (Some storage) in
-  lmdb
-
 let store ?storage db =
   match storage, db.storage_ref with
-  | Some storage, _ | None, Some storage ->
-      let lmdb = Datascript_storage_lmdb.lmdb storage in
-      Datascript_storage_lmdb.store_meta lmdb db
+  | Some target_storage, _ | None, Some target_storage ->
+      Index.sync_indexes_to_storage (Index.db_of db.eavt_index) target_storage;
+      Datascript_storage_lmdb.store_db target_storage db
   | None, None -> invalid_arg "db has no attached storage"
 
 let restore_root_snapshot storage =
   let schema, max_eid, max_tx, duplicate_datoms =
     Datascript_storage_lmdb.restore_meta (Datascript_storage_lmdb.lmdb storage)
   in
-  let index_lmdb = index_lmdb storage in
+  let lmdb, _ = Index.create_lmdb None in
+  Index.load_indexes_from_storage storage lmdb;
   Some
     { serializable_schema = schema
-    ; serializable_datoms = Index.to_list (Index.empty Eavt index_lmdb) @ duplicate_datoms
+    ; serializable_datoms = Index.to_list (Index.empty Eavt lmdb) @ duplicate_datoms
     ; serializable_max_eid = max_eid
     ; serializable_max_tx = max_tx
     }
@@ -34,7 +31,8 @@ let restore context storage =
     Datascript_storage_lmdb.restore_meta (Datascript_storage_lmdb.lmdb storage)
   in
   let schema = Schema.validate_schema schema in
-  let index_lmdb = index_lmdb storage in
+  let lmdb, _ = Index.create_lmdb None in
+  Index.load_indexes_from_storage storage lmdb;
   let duplicate_eavt_by_entity =
     let table = Hashtbl.create 1024 in
     List.iter
@@ -64,9 +62,9 @@ let restore context storage =
   Some
     { db_uid = context.next_db_uid ()
     ; schema
-    ; eavt_index = Index.empty Eavt index_lmdb
-    ; aevt_index = Index.empty Aevt index_lmdb
-    ; avet_index = Index.empty Avet index_lmdb
+    ; eavt_index = Index.empty Eavt lmdb
+    ; aevt_index = Index.empty Aevt lmdb
+    ; avet_index = Index.empty Avet lmdb
     ; aevt_by_attr = Hashtbl.create 0
     ; avet_by_attr = Hashtbl.create 0
     ; duplicate_datoms
diff --git a/impl/platform/native/index.ml b/impl/platform/native/index.ml
index 1749a76..b2857f4 100644
--- a/impl/platform/native/index.ml
+++ b/impl/platform/native/index.ml
@@ -19,6 +19,16 @@ let create_lmdb storage =
 let lmdb_of lmdb = lmdb
 let db_of t = Datascript_lmdb_index.db_of (project t)
 
+let lmdb_for_storage storage = Datascript_storage_lmdb.lmdb storage
+
+let sync_indexes_to_storage source target_storage =
+  let target = Datascript_storage_lmdb.lmdb target_storage in
+  if source != target then Datascript_storage_lmdb.sync_indexes source target
+
+let load_indexes_from_storage storage target_lmdb =
+  let source = Datascript_storage_lmdb.lmdb storage in
+  if source != target_lmdb then Datascript_storage_lmdb.sync_indexes source target_lmdb
+
 let empty index lmdb = Datascript_lmdb_index.empty index lmdb |> inject
 let of_sorted_list index datoms lmdb = Datascript_lmdb_index.of_sorted_list index datoms lmdb |> inject
 
diff --git a/impl/platform/native/storage.ml b/impl/platform/native/storage.ml
index 78a6ec0..8d87c15 100644
--- a/impl/platform/native/storage.ml
+++ b/impl/platform/native/storage.ml
@@ -6,25 +6,22 @@ type restore_context = { next_db_uid : unit -> int }
 
 let memory_storage = Datascript_storage_lmdb.memory_storage
 
-let index_lmdb storage =
-  let lmdb, _ = Index.create_lmdb (Some storage) in
-  lmdb
-
 let store ?storage db =
   match storage, db.storage_ref with
-  | Some storage, _ | None, Some storage ->
-      let lmdb = Datascript_storage_lmdb.lmdb storage in
-      Datascript_storage_lmdb.store_meta lmdb db
+  | Some target_storage, _ | None, Some target_storage ->
+      Index.sync_indexes_to_storage (Index.db_of db.eavt_index) target_storage;
+      Datascript_storage_lmdb.store_db target_storage db
   | None, None -> invalid_arg "db has no attached storage"
 
 let restore_root_snapshot storage =
   let schema, max_eid, max_tx, duplicate_datoms =
     Datascript_storage_lmdb.restore_meta (Datascript_storage_lmdb.lmdb storage)
   in
-  let index_lmdb = index_lmdb storage in
+  let lmdb, _ = Index.create_lmdb None in
+  Index.load_indexes_from_storage storage lmdb;
   Some
     { serializable_schema = schema
-    ; serializable_datoms = Index.to_list (Index.empty Eavt index_lmdb) @ duplicate_datoms
+    ; serializable_datoms = Index.to_list (Index.empty Eavt lmdb) @ duplicate_datoms
     ; serializable_max_eid = max_eid
     ; serializable_max_tx = max_tx
     }
@@ -34,7 +31,8 @@ let restore context storage =
     Datascript_storage_lmdb.restore_meta (Datascript_storage_lmdb.lmdb storage)
   in
   let schema = Schema.validate_schema schema in
-  let index_lmdb = index_lmdb storage in
+  let lmdb, _ = Index.create_lmdb None in
+  Index.load_indexes_from_storage storage lmdb;
   let duplicate_eavt_by_entity =
     let table = Hashtbl.create 1024 in
     List.iter
@@ -64,9 +62,9 @@ let restore context storage =
   Some
     { db_uid = context.next_db_uid ()
     ; schema
-    ; eavt_index = Index.empty Eavt index_lmdb
-    ; aevt_index = Index.empty Aevt index_lmdb
-    ; avet_index = Index.empty Avet index_lmdb
+    ; eavt_index = Index.empty Eavt lmdb
+    ; aevt_index = Index.empty Aevt lmdb
+    ; avet_index = Index.empty Avet lmdb
     ; aevt_by_attr = Hashtbl.create 0
     ; avet_by_attr = Hashtbl.create 0
     ; duplicate_datoms
diff --git a/impl/serialize.ml b/impl/serialize.ml
index d811c6a..a0397d1 100644
--- a/impl/serialize.ml
+++ b/impl/serialize.ml
@@ -7,7 +7,7 @@ type context =
   { next_db_uid : unit -> int
   ; validate_schema : schema -> schema
   ; normalize_datom_for_schema : schema -> datom -> datom
-  ; refresh_db_indexes : db -> db
+  ; with_datoms : db -> datom list -> db
   }
 
 let serializable db =
@@ -18,52 +18,9 @@ let serializable db =
   ; serializable_max_tx = db.max_tx
   }
 
-let duplicate_datoms datoms =
-  let datoms = List.sort (Datascript_types.Compare.compare_datom Eavt) datoms in
-  let rec loop previous duplicates = function
-    | [] -> List.rev duplicates
-    | datom :: rest ->
-      (match previous with
-       | Some previous when Datascript_types.Compare.compare_datom Eavt previous datom = 0 ->
-         loop (Some datom) (datom :: duplicates) rest
-       | _ -> loop (Some datom) duplicates rest)
-  in
-  loop None [] datoms
-
-let duplicate_aevt_datoms duplicate_datoms =
-  List.sort (Datascript_types.Compare.compare_datom Aevt) duplicate_datoms
-
-let duplicate_avet_datoms schema duplicate_datoms =
-  duplicate_datoms
-  |> List.filter (fun datom -> Schema.schema_attr_is_avet_accessible schema datom.a)
-  |> List.sort (Datascript_types.Compare.compare_datom Avet)
-
-let duplicate_eavt_by_entity duplicate_datoms =
-  let table = Hashtbl.create 1024 in
-  List.iter
-    (fun datom ->
-      let existing = Option.value (Hashtbl.find_opt table datom.e) ~default:[] in
-      Hashtbl.replace table datom.e (datom :: existing))
-    duplicate_datoms;
-  Hashtbl.iter (fun entity_id datoms -> Hashtbl.replace table entity_id (List.rev datoms)) table;
-  table
-
-let duplicate_datoms_by_attr duplicate_datoms =
-  let table = Hashtbl.create 1024 in
-  List.iter
-    (fun datom ->
-      let existing = Option.value (Hashtbl.find_opt table datom.a) ~default:[] in
-      Hashtbl.replace table datom.a (datom :: existing))
-    duplicate_datoms;
-  Hashtbl.iter (fun attr datoms -> Hashtbl.replace table attr (List.rev datoms)) table;
-  table
-
 let from_serializable context snapshot =
   let schema = context.validate_schema snapshot.serializable_schema in
   let datoms = List.map (context.normalize_datom_for_schema schema) snapshot.serializable_datoms in
-  let duplicate_datoms = duplicate_datoms datoms in
-  let duplicate_aevt_datoms = duplicate_aevt_datoms duplicate_datoms in
-  let duplicate_avet_datoms = duplicate_avet_datoms schema duplicate_datoms in
   let lmdb, storage_ref = Index.create_lmdb None in
   { db_uid = context.next_db_uid ()
   ; schema
@@ -72,12 +29,12 @@ let from_serializable context snapshot =
   ; avet_index = Index.empty Avet lmdb
   ; aevt_by_attr = Hashtbl.create 0
   ; avet_by_attr = Hashtbl.create 0
-  ; duplicate_datoms
-  ; duplicate_aevt_datoms
-  ; duplicate_avet_datoms
-  ; duplicate_eavt_by_entity = duplicate_eavt_by_entity duplicate_datoms
-  ; duplicate_aevt_by_attr = duplicate_datoms_by_attr duplicate_aevt_datoms
-  ; duplicate_avet_by_attr = duplicate_datoms_by_attr duplicate_avet_datoms
+  ; duplicate_datoms = []
+  ; duplicate_aevt_datoms = []
+  ; duplicate_avet_datoms = []
+  ; duplicate_eavt_by_entity = Hashtbl.create 0
+  ; duplicate_aevt_by_attr = Hashtbl.create 0
+  ; duplicate_avet_by_attr = Hashtbl.create 0
   ; max_eid = snapshot.serializable_max_eid
   ; max_datom_e = 0
   ; max_tx = snapshot.serializable_max_tx
@@ -85,4 +42,4 @@ let from_serializable context snapshot =
   ; storage_ref
   ; tx_fns = []
   }
-  |> context.refresh_db_indexes
+  |> fun db -> context.with_datoms db datoms
diff --git a/impl/serialize.mli b/impl/serialize.mli
index c071cbb..3dac898 100644
--- a/impl/serialize.mli
+++ b/impl/serialize.mli
@@ -4,7 +4,7 @@ type context =
   { next_db_uid : unit -> int
   ; validate_schema : schema -> schema
   ; normalize_datom_for_schema : schema -> datom -> datom
-  ; refresh_db_indexes : db -> db
+  ; with_datoms : db -> datom list -> db
   }
 
 val serializable : db -> serializable_db
diff --git a/lmdb/datascript_lmdb_codec.ml b/lmdb/datascript_lmdb_codec.ml
index d527f2c..2ce2417 100644
--- a/lmdb/datascript_lmdb_codec.ml
+++ b/lmdb/datascript_lmdb_codec.ml
@@ -37,8 +37,8 @@ let float_sort_bits value =
   if Int64.compare bits 0L < 0 then Int64.logxor bits 0x7fffffffffffffffL else bits
 
 let append_string buffer text =
-  append_int32 buffer (String.length text);
-  Buffer.add_string buffer text
+  Buffer.add_string buffer text;
+  Buffer.add_char buffer '\000'
 
 let append_byte buffer value = Buffer.add_char buffer (Char.chr value)
 
@@ -47,9 +47,15 @@ let read_int32 key offset =
   int32_of_be (String.sub key offset 4), offset + 4
 
 let read_string key offset =
-  let length, offset = read_int32 key offset in
-  if length < 0 || offset + length > String.length key then invalid_arg "truncated string";
-  String.sub key offset length, offset + length
+  let len = String.length key in
+  if offset >= len then invalid_arg "truncated string";
+  let rec find_end index =
+    if index >= len then invalid_arg "unterminated string"
+    else if key.[index] = '\000' then index
+    else find_end (index + 1)
+  in
+  let end_offset = find_end offset in
+  String.sub key offset (end_offset - offset), end_offset + 1
 
 let read_byte key offset =
   if offset >= String.length key then invalid_arg "truncated byte";
diff --git a/lmdb/datascript_lmdb_index.ml b/lmdb/datascript_lmdb_index.ml
index 2cfbc4b..35cd033 100644
--- a/lmdb/datascript_lmdb_index.ml
+++ b/lmdb/datascript_lmdb_index.ml
@@ -78,7 +78,19 @@ let slice_seq ?from_ ?to_ ?cmp t =
 
 let rslice_seq ?from_ ?to_ ?cmp t =
   let cmp = Option.value ~default:(cmp_for t.which) cmp in
-  make_seq ~cmp ?from_ ?to_ (List.rev (to_list t))
+  let datoms =
+    to_list t
+    |> List.filter (fun datom ->
+      match from_ with
+      | None -> true
+      | Some bound -> cmp datom bound <= 0)
+    |> List.filter (fun datom ->
+      match to_ with
+      | None -> true
+      | Some bound -> cmp datom bound >= 0)
+    |> List.rev
+  in
+  make_seq ~cmp datoms
 
 let slice ?from_ ?to_ ?cmp t = slice_seq ?from_ ?to_ ?cmp t |> seq_to_list
 
diff --git a/lmdb/melange/datascript_lmdb_index.ml b/lmdb/melange/datascript_lmdb_index.ml
index d0b23c8..94d180f 100644
--- a/lmdb/melange/datascript_lmdb_index.ml
+++ b/lmdb/melange/datascript_lmdb_index.ml
@@ -78,7 +78,19 @@ let slice_seq ?from_ ?to_ ?cmp t =
 
 let rslice_seq ?from_ ?to_ ?cmp t =
   let cmp = Option.value ~default:(cmp_for t.which) cmp in
-  make_seq ~cmp ?from_ ?to_ (List.rev (to_list t))
+  let datoms =
+    to_list t
+    |> List.filter (fun datom ->
+      match from_ with
+      | None -> true
+      | Some bound -> cmp datom bound <= 0)
+    |> List.filter (fun datom ->
+      match to_ with
+      | None -> true
+      | Some bound -> cmp datom bound >= 0)
+    |> List.rev
+  in
+  make_seq ~cmp datoms
 
 let seq_to_list seq = to_seq seq |> List.of_seq
 
diff --git a/lmdb/melange/datascript_storage_lmdb.ml b/lmdb/melange/datascript_storage_lmdb.ml
index 405958e..a66a3ea 100644
--- a/lmdb/melange/datascript_storage_lmdb.ml
+++ b/lmdb/melange/datascript_storage_lmdb.ml
@@ -2,14 +2,24 @@ open Datascript_types
 
 type t = Datascript_lmdb_db.t
 
-let registry : (storage, t) Hashtbl.t = Hashtbl.create 16
+module Storage_registry = struct
+  type t = storage
+
+  let equal left right = left == right
+
+  let hash storage = Hashtbl.hash (Obj.repr storage)
+end
+
+module Registry = Hashtbl.Make (Storage_registry)
+
+let registry = Registry.create 16
 
 let lmdb storage =
-  match Hashtbl.find_opt registry storage with
+  match Registry.find_opt registry storage with
   | Some lmdb -> lmdb
   | None -> invalid_arg "storage is not LMDB-backed"
 
-let register storage lmdb = Hashtbl.replace registry storage lmdb
+let register storage lmdb = Registry.replace registry storage lmdb
 
 let create_temp () = Datascript_lmdb_db.create_temp ()
 let open_path path = Datascript_lmdb_db.open_path path
@@ -78,3 +88,19 @@ let restore_meta lmdb =
     | Some bytes -> Datascript_lmdb_codec.decode_datoms bytes
   in
   schema, max_eid, max_tx, duplicate_datoms
+
+let sync_indexes from_lmdb to_lmdb =
+  let clear_index index db =
+    let keys = ref [] in
+    Datascript_lmdb_db.fold_index index db (fun key _ -> keys := key :: !keys);
+    List.iter (fun key -> Datascript_lmdb_db.remove_index index db key) !keys
+  in
+  List.iter (fun index -> clear_index index to_lmdb) [ Eavt; Aevt; Avet ];
+  List.iter
+    (fun index ->
+      Datascript_lmdb_db.fold_index index from_lmdb (fun key value ->
+        Datascript_lmdb_db.put_index index to_lmdb key value))
+    [ Eavt; Aevt; Avet ]
+
+let store_db storage db =
+  store_meta (lmdb storage) db
diff --git a/lmdb/native/datascript_lmdb_index.ml b/lmdb/native/datascript_lmdb_index.ml
index d0b23c8..94d180f 100644
--- a/lmdb/native/datascript_lmdb_index.ml
+++ b/lmdb/native/datascript_lmdb_index.ml
@@ -78,7 +78,19 @@ let slice_seq ?from_ ?to_ ?cmp t =
 
 let rslice_seq ?from_ ?to_ ?cmp t =
   let cmp = Option.value ~default:(cmp_for t.which) cmp in
-  make_seq ~cmp ?from_ ?to_ (List.rev (to_list t))
+  let datoms =
+    to_list t
+    |> List.filter (fun datom ->
+      match from_ with
+      | None -> true
+      | Some bound -> cmp datom bound <= 0)
+    |> List.filter (fun datom ->
+      match to_ with
+      | None -> true
+      | Some bound -> cmp datom bound >= 0)
+    |> List.rev
+  in
+  make_seq ~cmp datoms
 
 let seq_to_list seq = to_seq seq |> List.of_seq
 
diff --git a/lmdb/native/datascript_storage_lmdb.ml b/lmdb/native/datascript_storage_lmdb.ml
index 405958e..a66a3ea 100644
--- a/lmdb/native/datascript_storage_lmdb.ml
+++ b/lmdb/native/datascript_storage_lmdb.ml
@@ -2,14 +2,24 @@ open Datascript_types
 
 type t = Datascript_lmdb_db.t
 
-let registry : (storage, t) Hashtbl.t = Hashtbl.create 16
+module Storage_registry = struct
+  type t = storage
+
+  let equal left right = left == right
+
+  let hash storage = Hashtbl.hash (Obj.repr storage)
+end
+
+module Registry = Hashtbl.Make (Storage_registry)
+
+let registry = Registry.create 16
 
 let lmdb storage =
-  match Hashtbl.find_opt registry storage with
+  match Registry.find_opt registry storage with
   | Some lmdb -> lmdb
   | None -> invalid_arg "storage is not LMDB-backed"
 
-let register storage lmdb = Hashtbl.replace registry storage lmdb
+let register storage lmdb = Registry.replace registry storage lmdb
 
 let create_temp () = Datascript_lmdb_db.create_temp ()
 let open_path path = Datascript_lmdb_db.open_path path
@@ -78,3 +88,19 @@ let restore_meta lmdb =
     | Some bytes -> Datascript_lmdb_codec.decode_datoms bytes
   in
   schema, max_eid, max_tx, duplicate_datoms
+
+let sync_indexes from_lmdb to_lmdb =
+  let clear_index index db =
+    let keys = ref [] in
+    Datascript_lmdb_db.fold_index index db (fun key _ -> keys := key :: !keys);
+    List.iter (fun key -> Datascript_lmdb_db.remove_index index db key) !keys
+  in
+  List.iter (fun index -> clear_index index to_lmdb) [ Eavt; Aevt; Avet ];
+  List.iter
+    (fun index ->
+      Datascript_lmdb_db.fold_index index from_lmdb (fun key value ->
+        Datascript_lmdb_db.put_index index to_lmdb key value))
+    [ Eavt; Aevt; Avet ]
+
+let store_db storage db =
+  store_meta (lmdb storage) db

From 1eeda4a30866010b9b2b99c19ce81bd6a4ade06c Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 26 Aug 2026 21:21:33 +0000
Subject: [PATCH 06/10] Fix LMDB overlay immutability and storage sync

Use functional overlay indexes so transact returns new db handles without
mutating the input db. Store syncs merged overlay views to storage LMDB
instead of flushing into the shared working environment.

Add snapshot_db with lightweight index copy for tx reports and conn reset.
Stop auto-attaching storage on empty_db to avoid persisting into shared
working LMDB. Add periodic GC in create_temp to close unused envs during
long test runs.

Co-authored-by: Tienson Qin <tiensonqin@gmail.com>
---
 impl/conn.ml                           |  4 +-
 impl/conn.mli                          |  1 +
 impl/datascript.ml                     |  9 +++-
 impl/datascript.mli                    |  1 +
 impl/db.ml                             |  7 +++
 impl/db.mli                            |  1 +
 impl/index.mli                         |  4 +-
 impl/platform/jsoo/index.ml            | 12 +++--
 impl/platform/jsoo/storage.ml          |  2 +-
 impl/platform/melange/index.ml         | 12 +++--
 impl/platform/melange/storage.ml       |  2 +-
 impl/platform/native/index.ml          | 12 +++--
 impl/platform/native/storage.ml        |  2 +-
 lmdb/datascript_lmdb_codec.ml          | 15 ++----
 lmdb/melange/datascript_lmdb_index.ml  | 70 ++++++++++++++++++++++----
 lmdb/melange/datascript_lmdb_index.mli | 19 +++----
 lmdb/native/datascript_lmdb_db.ml      | 25 ++++++---
 lmdb/native/datascript_lmdb_index.ml   | 70 ++++++++++++++++++++++----
 lmdb/native/datascript_lmdb_index.mli  | 19 +++----
 19 files changed, 202 insertions(+), 85 deletions(-)

diff --git a/impl/conn.ml b/impl/conn.ml
index 9a68d7b..66a36aa 100644
--- a/impl/conn.ml
+++ b/impl/conn.ml
@@ -28,6 +28,7 @@ type transact_context =
 type reset_context =
   { store : ?storage:storage -> db -> unit
   ; datoms : db -> datom list
+  ; snapshot_db : db -> db
   }
 
 type context =
@@ -131,6 +132,7 @@ let transact (context : transact_context) ?(tx_meta = []) conn tx_data =
   report
 
 let reset (context : reset_context) ?(tx_meta = []) conn db =
+  let db_before = context.snapshot_db conn.db in
   let db =
     match conn.storage with
     | None -> db
@@ -140,7 +142,7 @@ let reset (context : reset_context) ?(tx_meta = []) conn db =
     List.map (fun datom -> { datom with added = false }) (context.datoms conn.db)
     @ context.datoms db
   in
-  let report = { db_before = conn.db; db_after = db; tx_data; tempids = []; tx_meta } in
+  let report = { db_before; db_after = db; tx_data; tempids = []; tx_meta } in
   conn.db <- db;
   (match conn.storage with
    | None -> ()
diff --git a/impl/conn.mli b/impl/conn.mli
index c8b37d7..04a6201 100644
--- a/impl/conn.mli
+++ b/impl/conn.mli
@@ -23,6 +23,7 @@ type transact_context =
 type reset_context =
   { store : ?storage:storage -> db -> unit
   ; datoms : db -> datom list
+  ; snapshot_db : db -> db
   }
 
 type context =
diff --git a/impl/datascript.ml b/impl/datascript.ml
index e4e1c0b..d30e65c 100644
--- a/impl/datascript.ml
+++ b/impl/datascript.ml
@@ -53,6 +53,7 @@ let normalize_datom_for_schema = Db_impl.normalize_datom_for_schema
 let refresh_db_indexes = Db_impl.refresh_indexes
 let refresh_db_indexes_with_added_datoms = Db_impl.refresh_indexes_with_added_datoms
 let refresh_db_indexes_with_tx_data = Db_impl.refresh_indexes_with_tx_data
+let snapshot_db = Db_impl.snapshot_db
 
 let empty_db ?(schema = []) ?storage () =
   Db_impl.empty_db db_core_context ~schema ?storage ()
@@ -754,8 +755,9 @@ let persist_transact ~tx_meta db =
     | Some storage -> store ~storage db
 
 let transact_report ?(tx_meta = []) db tx_ops =
+  let db_before = snapshot_db db in
   let db_after, tempids, tx_data = apply_tx tx_ops db in
-  { db_before = db; db_after; tx_data; tempids; tx_meta }
+  { db_before; db_after; tx_data; tempids; tx_meta }
 
 let transact ?(tx_meta = []) db tx_ops =
   let report = transact_report ~tx_meta db tx_ops in
@@ -814,7 +816,10 @@ let squuid_time_millis = Db_impl.squuid_time_millis
 
 let reset_conn ?(tx_meta = []) conn db =
   let context : Conn.reset_context =
-    { store; datoms = (fun db -> datoms_list db Eavt ()) }
+    { store
+    ; datoms = (fun db -> datoms_list db Eavt ())
+    ; snapshot_db
+    }
   in
   Conn.reset context ~tx_meta conn db
 
diff --git a/impl/datascript.mli b/impl/datascript.mli
index 33f67f3..7fa533b 100644
--- a/impl/datascript.mli
+++ b/impl/datascript.mli
@@ -98,6 +98,7 @@ module Conn : sig
   type reset_context =
     { store : ?storage:storage -> db -> unit
     ; datoms : db -> datom list
+    ; snapshot_db : db -> db
     }
 
   val create : creation_context -> ?schema:schema -> ?storage:storage -> unit -> t
diff --git a/impl/db.ml b/impl/db.ml
index 912543b..4c3ecbe 100644
--- a/impl/db.ml
+++ b/impl/db.ml
@@ -258,6 +258,13 @@ let refresh_indexes_with_tx_data db tx_data =
   in
   invalidate_attr_tables db
 
+let snapshot_db db =
+  { db with
+    eavt_index = Index.copy db.eavt_index
+  ; aevt_index = Index.copy db.aevt_index
+  ; avet_index = Index.copy db.avet_index
+  }
+
 let with_datoms db datoms =
   set_indexes_from_datoms db datoms
 
diff --git a/impl/db.mli b/impl/db.mli
index fd8d812..a157562 100644
--- a/impl/db.mli
+++ b/impl/db.mli
@@ -18,6 +18,7 @@ val normalize_datom_for_schema : schema -> datom -> datom
 val refresh_indexes : db -> db
 val refresh_indexes_with_added_datoms : db -> datom list -> db
 val refresh_indexes_with_tx_data : db -> datom list -> db
+val snapshot_db : db -> db
 val with_datoms : db -> datom list -> db
 val empty_db : core_context -> ?schema:schema -> ?storage:storage -> unit -> db
 val empty : core_context -> db -> db
diff --git a/impl/index.mli b/impl/index.mli
index 44cfb34..c3041eb 100644
--- a/impl/index.mli
+++ b/impl/index.mli
@@ -8,7 +8,7 @@ val create_lmdb : storage option -> lmdb * storage option
 val lmdb_of : lmdb -> lmdb
 val db_of : t -> lmdb
 val lmdb_for_storage : storage -> lmdb
-val sync_indexes_to_storage : lmdb -> storage -> unit
+val sync_indexes_to_storage : t -> t -> t -> storage -> unit
 val load_indexes_from_storage : storage -> lmdb -> unit
 
 val empty : index -> lmdb -> t
@@ -25,3 +25,5 @@ val seq_to_list : datom seq -> datom list
 val fold_seq : ('acc -> datom -> 'acc) -> 'acc -> datom seq -> 'acc
 val to_seq : datom seq -> datom Seq.t
 val seek : datom -> datom seq -> datom seq
+val flush : t -> t
+val copy : t -> t
diff --git a/impl/platform/jsoo/index.ml b/impl/platform/jsoo/index.ml
index b2857f4..da378fd 100644
--- a/impl/platform/jsoo/index.ml
+++ b/impl/platform/jsoo/index.ml
@@ -12,18 +12,18 @@ type lmdb = Datascript_lmdb_db.t
 let create_lmdb storage =
   match storage with
   | Some storage -> (Datascript_storage_lmdb.lmdb storage, Some storage)
-  | None ->
-      let lmdb = Datascript_lmdb_db.create_temp () in
-      (lmdb, Some (Datascript_storage_lmdb.wrap lmdb))
+  | None -> (Datascript_lmdb_db.create_temp (), None)
 
 let lmdb_of lmdb = lmdb
 let db_of t = Datascript_lmdb_index.db_of (project t)
 
 let lmdb_for_storage storage = Datascript_storage_lmdb.lmdb storage
 
-let sync_indexes_to_storage source target_storage =
+let sync_indexes_to_storage eavt aevt avet target_storage =
   let target = Datascript_storage_lmdb.lmdb target_storage in
-  if source != target then Datascript_storage_lmdb.sync_indexes source target
+  Datascript_lmdb_index.sync_merged_to_lmdb (project eavt) target;
+  Datascript_lmdb_index.sync_merged_to_lmdb (project aevt) target;
+  Datascript_lmdb_index.sync_merged_to_lmdb (project avet) target
 
 let load_indexes_from_storage storage target_lmdb =
   let source = Datascript_storage_lmdb.lmdb storage in
@@ -44,3 +44,5 @@ let seq_to_list = Datascript_lmdb_index.seq_to_list
 let fold_seq = Datascript_lmdb_index.fold_seq
 let to_seq = Datascript_lmdb_index.to_seq
 let seek = Datascript_lmdb_index.seek
+let flush t = Datascript_lmdb_index.flush (project t) |> inject
+let copy t = Datascript_lmdb_index.copy (project t) |> inject
diff --git a/impl/platform/jsoo/storage.ml b/impl/platform/jsoo/storage.ml
index 8d87c15..68dff44 100644
--- a/impl/platform/jsoo/storage.ml
+++ b/impl/platform/jsoo/storage.ml
@@ -9,7 +9,7 @@ let memory_storage = Datascript_storage_lmdb.memory_storage
 let store ?storage db =
   match storage, db.storage_ref with
   | Some target_storage, _ | None, Some target_storage ->
-      Index.sync_indexes_to_storage (Index.db_of db.eavt_index) target_storage;
+      Index.sync_indexes_to_storage db.eavt_index db.aevt_index db.avet_index target_storage;
       Datascript_storage_lmdb.store_db target_storage db
   | None, None -> invalid_arg "db has no attached storage"
 
diff --git a/impl/platform/melange/index.ml b/impl/platform/melange/index.ml
index b2857f4..da378fd 100644
--- a/impl/platform/melange/index.ml
+++ b/impl/platform/melange/index.ml
@@ -12,18 +12,18 @@ type lmdb = Datascript_lmdb_db.t
 let create_lmdb storage =
   match storage with
   | Some storage -> (Datascript_storage_lmdb.lmdb storage, Some storage)
-  | None ->
-      let lmdb = Datascript_lmdb_db.create_temp () in
-      (lmdb, Some (Datascript_storage_lmdb.wrap lmdb))
+  | None -> (Datascript_lmdb_db.create_temp (), None)
 
 let lmdb_of lmdb = lmdb
 let db_of t = Datascript_lmdb_index.db_of (project t)
 
 let lmdb_for_storage storage = Datascript_storage_lmdb.lmdb storage
 
-let sync_indexes_to_storage source target_storage =
+let sync_indexes_to_storage eavt aevt avet target_storage =
   let target = Datascript_storage_lmdb.lmdb target_storage in
-  if source != target then Datascript_storage_lmdb.sync_indexes source target
+  Datascript_lmdb_index.sync_merged_to_lmdb (project eavt) target;
+  Datascript_lmdb_index.sync_merged_to_lmdb (project aevt) target;
+  Datascript_lmdb_index.sync_merged_to_lmdb (project avet) target
 
 let load_indexes_from_storage storage target_lmdb =
   let source = Datascript_storage_lmdb.lmdb storage in
@@ -44,3 +44,5 @@ let seq_to_list = Datascript_lmdb_index.seq_to_list
 let fold_seq = Datascript_lmdb_index.fold_seq
 let to_seq = Datascript_lmdb_index.to_seq
 let seek = Datascript_lmdb_index.seek
+let flush t = Datascript_lmdb_index.flush (project t) |> inject
+let copy t = Datascript_lmdb_index.copy (project t) |> inject
diff --git a/impl/platform/melange/storage.ml b/impl/platform/melange/storage.ml
index 8d87c15..68dff44 100644
--- a/impl/platform/melange/storage.ml
+++ b/impl/platform/melange/storage.ml
@@ -9,7 +9,7 @@ let memory_storage = Datascript_storage_lmdb.memory_storage
 let store ?storage db =
   match storage, db.storage_ref with
   | Some target_storage, _ | None, Some target_storage ->
-      Index.sync_indexes_to_storage (Index.db_of db.eavt_index) target_storage;
+      Index.sync_indexes_to_storage db.eavt_index db.aevt_index db.avet_index target_storage;
       Datascript_storage_lmdb.store_db target_storage db
   | None, None -> invalid_arg "db has no attached storage"
 
diff --git a/impl/platform/native/index.ml b/impl/platform/native/index.ml
index b2857f4..da378fd 100644
--- a/impl/platform/native/index.ml
+++ b/impl/platform/native/index.ml
@@ -12,18 +12,18 @@ type lmdb = Datascript_lmdb_db.t
 let create_lmdb storage =
   match storage with
   | Some storage -> (Datascript_storage_lmdb.lmdb storage, Some storage)
-  | None ->
-      let lmdb = Datascript_lmdb_db.create_temp () in
-      (lmdb, Some (Datascript_storage_lmdb.wrap lmdb))
+  | None -> (Datascript_lmdb_db.create_temp (), None)
 
 let lmdb_of lmdb = lmdb
 let db_of t = Datascript_lmdb_index.db_of (project t)
 
 let lmdb_for_storage storage = Datascript_storage_lmdb.lmdb storage
 
-let sync_indexes_to_storage source target_storage =
+let sync_indexes_to_storage eavt aevt avet target_storage =
   let target = Datascript_storage_lmdb.lmdb target_storage in
-  if source != target then Datascript_storage_lmdb.sync_indexes source target
+  Datascript_lmdb_index.sync_merged_to_lmdb (project eavt) target;
+  Datascript_lmdb_index.sync_merged_to_lmdb (project aevt) target;
+  Datascript_lmdb_index.sync_merged_to_lmdb (project avet) target
 
 let load_indexes_from_storage storage target_lmdb =
   let source = Datascript_storage_lmdb.lmdb storage in
@@ -44,3 +44,5 @@ let seq_to_list = Datascript_lmdb_index.seq_to_list
 let fold_seq = Datascript_lmdb_index.fold_seq
 let to_seq = Datascript_lmdb_index.to_seq
 let seek = Datascript_lmdb_index.seek
+let flush t = Datascript_lmdb_index.flush (project t) |> inject
+let copy t = Datascript_lmdb_index.copy (project t) |> inject
diff --git a/impl/platform/native/storage.ml b/impl/platform/native/storage.ml
index 8d87c15..68dff44 100644
--- a/impl/platform/native/storage.ml
+++ b/impl/platform/native/storage.ml
@@ -9,7 +9,7 @@ let memory_storage = Datascript_storage_lmdb.memory_storage
 let store ?storage db =
   match storage, db.storage_ref with
   | Some target_storage, _ | None, Some target_storage ->
-      Index.sync_indexes_to_storage (Index.db_of db.eavt_index) target_storage;
+      Index.sync_indexes_to_storage db.eavt_index db.aevt_index db.avet_index target_storage;
       Datascript_storage_lmdb.store_db target_storage db
   | None, None -> invalid_arg "db has no attached storage"
 
diff --git a/lmdb/datascript_lmdb_codec.ml b/lmdb/datascript_lmdb_codec.ml
index 2ce2417..2b68a4a 100644
--- a/lmdb/datascript_lmdb_codec.ml
+++ b/lmdb/datascript_lmdb_codec.ml
@@ -110,19 +110,16 @@ let rec encode_value_key = function
   | Int value ->
       let buffer = Buffer.create 16 in
       append_byte buffer 9;
-      append_byte buffer 0;
       append_int64 buffer (float_sort_bits (float_of_int value));
       Buffer.contents buffer
   | Float value ->
       let buffer = Buffer.create 16 in
       append_byte buffer 9;
-      append_byte buffer 1;
       append_int64 buffer (float_sort_bits value);
       Buffer.contents buffer
   | Ref value ->
       let buffer = Buffer.create 16 in
       append_byte buffer 9;
-      append_byte buffer 2;
       append_int64 buffer (float_sort_bits (float_of_int value));
       Buffer.contents buffer
   | String value ->
@@ -200,20 +197,14 @@ let rec decode_value_key bytes offset =
       let value, offset = read_byte bytes offset in
       (match value with 0 -> Bool false | 1 -> Bool true | _ -> invalid_arg "invalid bool key"), offset
   | 9 ->
-      let kind, offset = read_byte bytes offset in
       let bits, offset =
         if offset + 8 > String.length bytes then invalid_arg "truncated numeric key"
         else int64_of_be (String.sub bytes offset 8), offset + 8
       in
-      let float_value =
-        let raw = if Int64.compare bits 0L < 0 then Int64.logxor bits 0x7fffffffffffffffL else bits in
-        Int64.float_of_bits raw
+      let raw =
+        if Int64.compare bits 0L < 0 then Int64.logxor bits 0x7fffffffffffffffL else bits
       in
-      (match kind with
-       | 0 -> Int (int_of_float float_value)
-       | 1 -> Float float_value
-       | 2 -> Ref (int_of_float float_value)
-       | _ -> invalid_arg "invalid numeric kind"), offset
+      Float (Int64.float_of_bits raw), offset
   | 10 ->
       let value, offset = read_string bytes offset in
       String value, offset
diff --git a/lmdb/melange/datascript_lmdb_index.ml b/lmdb/melange/datascript_lmdb_index.ml
index 94d180f..dc84b66 100644
--- a/lmdb/melange/datascript_lmdb_index.ml
+++ b/lmdb/melange/datascript_lmdb_index.ml
@@ -1,25 +1,32 @@
 open Datascript_types
 
-type t = { db : Datascript_lmdb_db.t; which : index }
+type t =
+  { db : Datascript_lmdb_db.t
+  ; which : index
+  ; additions : datom list
+  ; removals : datom list
+  }
 
 type 'a seq = { cmp : datom -> datom -> int; datoms : datom list; offset : int }
 
 let db_of t = t.db
-let make index db = { db; which = index }
+let make index db = { db; which = index; additions = []; removals = [] }
 let cmp_for index = Datascript_types.Compare.compare_datom index
 
+let datom_key t datom = Datascript_lmdb_codec.encode_datom_key t.which datom
+
 let decode_entry index key value =
   let datom = Datascript_lmdb_codec.decode_datom_key index key in
   let payload = Datascript_lmdb_codec.decode_datom_value value in
   { datom with added = payload.added; v = payload.v }
 
 let put_datom t datom =
-  let key = Datascript_lmdb_codec.encode_datom_key t.which datom in
+  let key = datom_key t datom in
   let value = Datascript_lmdb_codec.encode_datom_value datom in
   Datascript_lmdb_db.put_index t.which t.db key value
 
 let remove_datom t datom =
-  let key = Datascript_lmdb_codec.encode_datom_key t.which datom in
+  let key = datom_key t datom in
   Datascript_lmdb_db.remove_index t.which t.db key
 
 let empty index db = make index db
@@ -30,19 +37,64 @@ let of_sorted_list index datoms db =
   t
 
 let add datom t =
-  put_datom t datom;
-  t
+  let key = datom_key t datom in
+  let additions = datom :: List.filter (fun d -> datom_key t d <> key) t.additions in
+  let removals = List.filter (fun d -> datom_key t d <> key) t.removals in
+  { t with additions; removals }
 
 let remove datom t =
-  remove_datom t datom;
-  t
+  let key = datom_key t datom in
+  let additions = List.filter (fun d -> datom_key t d <> key) t.additions in
+  let already_removed = List.exists (fun d -> datom_key t d = key) t.removals in
+  let removals =
+    if already_removed || List.exists (fun d -> datom_key t d = key) t.additions then t.removals
+    else datom :: t.removals
+  in
+  { t with additions; removals }
 
-let collect_datoms t =
+let collect_stored t =
   let datoms = ref [] in
   Datascript_lmdb_db.fold_index t.which t.db (fun key value ->
     datoms := decode_entry t.which key value :: !datoms);
   List.rev !datoms
 
+let merge_overlay t base =
+  let cmp = cmp_for t.which in
+  let removed_keys = List.map (datom_key t) t.removals in
+  let addition_keys = List.map (datom_key t) t.additions in
+  let base =
+    base
+    |> List.filter (fun datom ->
+      let key = datom_key t datom in
+      not (List.mem key removed_keys || List.mem key addition_keys))
+  in
+  List.sort cmp (base @ t.additions)
+
+let collect_datoms t = merge_overlay t (collect_stored t)
+
+let put_datom_in index lmdb datom =
+  let key = Datascript_lmdb_codec.encode_datom_key index datom in
+  let value = Datascript_lmdb_codec.encode_datom_value datom in
+  Datascript_lmdb_db.put_index index lmdb key value
+
+let clear_index index lmdb =
+  let keys = ref [] in
+  Datascript_lmdb_db.fold_index index lmdb (fun key _ -> keys := key :: !keys);
+  List.iter (fun key -> Datascript_lmdb_db.remove_index index lmdb key) !keys
+
+let sync_merged_to_lmdb t target_lmdb =
+  clear_index t.which target_lmdb;
+  List.iter (put_datom_in t.which target_lmdb) (collect_datoms t)
+
+let copy_list xs = List.map (fun x -> x) xs
+
+let copy t = { t with additions = copy_list t.additions; removals = copy_list t.removals }
+
+let flush t =
+  List.iter (remove_datom t) t.removals;
+  List.iter (put_datom t) t.additions;
+  { t with additions = []; removals = [] }
+
 let to_list t = collect_datoms t
 let fold f init t = List.fold_left f init (to_list t)
 
diff --git a/lmdb/melange/datascript_lmdb_index.mli b/lmdb/melange/datascript_lmdb_index.mli
index 9fe6431..ea9198e 100644
--- a/lmdb/melange/datascript_lmdb_index.mli
+++ b/lmdb/melange/datascript_lmdb_index.mli
@@ -4,25 +4,18 @@ type t
 type 'a seq
 
 val db_of : t -> Datascript_lmdb_db.t
-
 val empty : index -> Datascript_lmdb_db.t -> t
 val of_sorted_list : index -> datom list -> Datascript_lmdb_db.t -> t
-
 val add : datom -> t -> t
 val remove : datom -> t -> t
-
+val flush : t -> t
+val copy : t -> t
+val sync_merged_to_lmdb : t -> Datascript_lmdb_db.t -> unit
 val to_list : t -> datom list
 val fold : ('acc -> datom -> 'acc) -> 'acc -> t -> 'acc
-
-val slice :
-  ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom list
-
-val slice_seq :
-  ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom seq
-
-val rslice_seq :
-  ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom seq
-
+val slice : ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom list
+val slice_seq : ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom seq
+val rslice_seq : ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom seq
 val seq : t -> datom seq
 val seq_to_list : datom seq -> datom list
 val fold_seq : ('acc -> datom -> 'acc) -> 'acc -> datom seq -> 'acc
diff --git a/lmdb/native/datascript_lmdb_db.ml b/lmdb/native/datascript_lmdb_db.ml
index 515353d..032aa70 100644
--- a/lmdb/native/datascript_lmdb_db.ml
+++ b/lmdb/native/datascript_lmdb_db.ml
@@ -33,13 +33,6 @@ let open_db path =
   ; avet = open_named_map env "ds/avet"; meta = open_named_map env "ds/meta"; closed = false
   }
 
-let create_temp () =
-  open_db
-    (Filename.temp_file
-       ~temp_dir:(Filename.get_temp_dir_name ())
-       "datascript_lmdb"
-       ".mdb")
-
 let open_path path = open_db path
 
 let ensure_open db =
@@ -55,6 +48,24 @@ let close db =
     Env.close db.env;
     db.closed <- true)
 
+let temps_created = ref 0
+
+let create_temp () =
+  let db =
+    open_db
+      (Filename.temp_file
+         ~temp_dir:(Filename.get_temp_dir_name ())
+         "datascript_lmdb"
+         ".mdb")
+  in
+  Gc.finalise
+    (fun lmdb ->
+      if not lmdb.closed then close lmdb)
+    db;
+  incr temps_created;
+  if !temps_created mod 64 = 0 then Gc.full_major ();
+  db
+
 let sync db =
   ensure_open db;
   Env.sync db.env
diff --git a/lmdb/native/datascript_lmdb_index.ml b/lmdb/native/datascript_lmdb_index.ml
index 94d180f..dc84b66 100644
--- a/lmdb/native/datascript_lmdb_index.ml
+++ b/lmdb/native/datascript_lmdb_index.ml
@@ -1,25 +1,32 @@
 open Datascript_types
 
-type t = { db : Datascript_lmdb_db.t; which : index }
+type t =
+  { db : Datascript_lmdb_db.t
+  ; which : index
+  ; additions : datom list
+  ; removals : datom list
+  }
 
 type 'a seq = { cmp : datom -> datom -> int; datoms : datom list; offset : int }
 
 let db_of t = t.db
-let make index db = { db; which = index }
+let make index db = { db; which = index; additions = []; removals = [] }
 let cmp_for index = Datascript_types.Compare.compare_datom index
 
+let datom_key t datom = Datascript_lmdb_codec.encode_datom_key t.which datom
+
 let decode_entry index key value =
   let datom = Datascript_lmdb_codec.decode_datom_key index key in
   let payload = Datascript_lmdb_codec.decode_datom_value value in
   { datom with added = payload.added; v = payload.v }
 
 let put_datom t datom =
-  let key = Datascript_lmdb_codec.encode_datom_key t.which datom in
+  let key = datom_key t datom in
   let value = Datascript_lmdb_codec.encode_datom_value datom in
   Datascript_lmdb_db.put_index t.which t.db key value
 
 let remove_datom t datom =
-  let key = Datascript_lmdb_codec.encode_datom_key t.which datom in
+  let key = datom_key t datom in
   Datascript_lmdb_db.remove_index t.which t.db key
 
 let empty index db = make index db
@@ -30,19 +37,64 @@ let of_sorted_list index datoms db =
   t
 
 let add datom t =
-  put_datom t datom;
-  t
+  let key = datom_key t datom in
+  let additions = datom :: List.filter (fun d -> datom_key t d <> key) t.additions in
+  let removals = List.filter (fun d -> datom_key t d <> key) t.removals in
+  { t with additions; removals }
 
 let remove datom t =
-  remove_datom t datom;
-  t
+  let key = datom_key t datom in
+  let additions = List.filter (fun d -> datom_key t d <> key) t.additions in
+  let already_removed = List.exists (fun d -> datom_key t d = key) t.removals in
+  let removals =
+    if already_removed || List.exists (fun d -> datom_key t d = key) t.additions then t.removals
+    else datom :: t.removals
+  in
+  { t with additions; removals }
 
-let collect_datoms t =
+let collect_stored t =
   let datoms = ref [] in
   Datascript_lmdb_db.fold_index t.which t.db (fun key value ->
     datoms := decode_entry t.which key value :: !datoms);
   List.rev !datoms
 
+let merge_overlay t base =
+  let cmp = cmp_for t.which in
+  let removed_keys = List.map (datom_key t) t.removals in
+  let addition_keys = List.map (datom_key t) t.additions in
+  let base =
+    base
+    |> List.filter (fun datom ->
+      let key = datom_key t datom in
+      not (List.mem key removed_keys || List.mem key addition_keys))
+  in
+  List.sort cmp (base @ t.additions)
+
+let collect_datoms t = merge_overlay t (collect_stored t)
+
+let put_datom_in index lmdb datom =
+  let key = Datascript_lmdb_codec.encode_datom_key index datom in
+  let value = Datascript_lmdb_codec.encode_datom_value datom in
+  Datascript_lmdb_db.put_index index lmdb key value
+
+let clear_index index lmdb =
+  let keys = ref [] in
+  Datascript_lmdb_db.fold_index index lmdb (fun key _ -> keys := key :: !keys);
+  List.iter (fun key -> Datascript_lmdb_db.remove_index index lmdb key) !keys
+
+let sync_merged_to_lmdb t target_lmdb =
+  clear_index t.which target_lmdb;
+  List.iter (put_datom_in t.which target_lmdb) (collect_datoms t)
+
+let copy_list xs = List.map (fun x -> x) xs
+
+let copy t = { t with additions = copy_list t.additions; removals = copy_list t.removals }
+
+let flush t =
+  List.iter (remove_datom t) t.removals;
+  List.iter (put_datom t) t.additions;
+  { t with additions = []; removals = [] }
+
 let to_list t = collect_datoms t
 let fold f init t = List.fold_left f init (to_list t)
 
diff --git a/lmdb/native/datascript_lmdb_index.mli b/lmdb/native/datascript_lmdb_index.mli
index 9fe6431..ea9198e 100644
--- a/lmdb/native/datascript_lmdb_index.mli
+++ b/lmdb/native/datascript_lmdb_index.mli
@@ -4,25 +4,18 @@ type t
 type 'a seq
 
 val db_of : t -> Datascript_lmdb_db.t
-
 val empty : index -> Datascript_lmdb_db.t -> t
 val of_sorted_list : index -> datom list -> Datascript_lmdb_db.t -> t
-
 val add : datom -> t -> t
 val remove : datom -> t -> t
-
+val flush : t -> t
+val copy : t -> t
+val sync_merged_to_lmdb : t -> Datascript_lmdb_db.t -> unit
 val to_list : t -> datom list
 val fold : ('acc -> datom -> 'acc) -> 'acc -> t -> 'acc
-
-val slice :
-  ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom list
-
-val slice_seq :
-  ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom seq
-
-val rslice_seq :
-  ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom seq
-
+val slice : ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom list
+val slice_seq : ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom seq
+val rslice_seq : ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom seq
 val seq : t -> datom seq
 val seq_to_list : datom seq -> datom list
 val fold_seq : ('acc -> datom -> 'acc) -> 'acc -> datom seq -> 'acc

From 56c365b8ecafe2abfd00eb7e80d2b65c24a2d078 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Thu, 27 Aug 2026 04:43:13 +0000
Subject: [PATCH 07/10] Batch LMDB index writes and cursor range reads
 (Datalevin-style)

- Add with_write_txn, put/remove/copy_index_txn, and fold_index_range on LMDB db
- Batch of_sorted_list, flush, sync_merged_to_lmdb, and storage sync in single txns
- Use cursor seek for slice lower bounds; keep custom cmp filtering for exact prefixes
- Fix sync_merged_to_lmdb to write into the target env (not the working env txn)
- Add 20k PSS vs LMDB benchmark harness for regression tracking

Co-authored-by: Tienson Qin <tiensonqin@gmail.com>
---
 bench/compare_pss_lmdb.sh               |  47 +++++++++
 bench/compare_pss_lmdb_20k.sh           |  49 +++++++++
 bench/dune                              |   5 +
 bench/index_compare_20k.ml              | 131 ++++++++++++++++++++++++
 lmdb/melange/datascript_lmdb_db.ml      |  18 ++++
 lmdb/melange/datascript_lmdb_index.ml   | 122 ++++++++++++++--------
 lmdb/melange/datascript_storage_lmdb.ml |  17 ++-
 lmdb/native/datascript_lmdb_db.ml       |  60 ++++++++---
 lmdb/native/datascript_lmdb_db.mli      |   7 ++
 lmdb/native/datascript_lmdb_index.ml    | 122 ++++++++++++++--------
 lmdb/native/datascript_storage_lmdb.ml  |  17 ++-
 11 files changed, 475 insertions(+), 120 deletions(-)
 create mode 100755 bench/compare_pss_lmdb.sh
 create mode 100755 bench/compare_pss_lmdb_20k.sh
 create mode 100644 bench/index_compare_20k.ml

diff --git a/bench/compare_pss_lmdb.sh b/bench/compare_pss_lmdb.sh
new file mode 100755
index 0000000..8b330d5
--- /dev/null
+++ b/bench/compare_pss_lmdb.sh
@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SIZE="${1:-20000}"
+WARMUP_MS="${2:-100}"
+SAMPLE_MS="${3:-200}"
+SAMPLES="${4:-3}"
+
+bench_args=(--size "$SIZE" --warmup-ms "$WARMUP_MS" --sample-ms "$SAMPLE_MS" --samples "$SAMPLES")
+
+run_branch_bench() {
+  local label="$1"
+  local repo="$2"
+  (
+    cd "$repo"
+    dune build bench/bench_ocaml.exe >/dev/null
+    BENCH_RUNTIME_LABEL="$label" dune exec bench/bench_ocaml.exe -- "${bench_args[@]}" 2>/dev/null
+  )
+}
+
+echo "=== PSS vs LMDB benchmark (${SIZE} entities) ==="
+echo "warmup=${WARMUP_MS}ms sample=${SAMPLE_MS}ms samples=${SAMPLES}"
+echo
+
+PSS_OUT="$(run_branch_bench pss /tmp/bench-pss-main)"
+LMDB_OUT="$(run_branch_bench lmdb /workspace)"
+
+printf "%-22s %12s %12s %12s\n" "benchmark" "pss(ms)" "lmdb(ms)" "lmdb/pss"
+echo "----------------------------------------------------------------"
+
+while IFS=$'\t' read -r name pss_ms; do
+  [[ "$name" == runtime* || "$name" == size* || -z "$name" ]] && continue
+  lmdb_ms="$(printf '%s\n' "$LMDB_OUT" | awk -F'\t' -v n="$name" '$1 == n { print $2; exit }')"
+  if [[ -z "$lmdb_ms" ]]; then
+    printf "%-22s %12s %12s %12s\n" "$name" "$pss_ms" "?" "?"
+    continue
+  fi
+  ratio="$(awk -v l="$lmdb_ms" -v p="$pss_ms" 'BEGIN { if (p + 0 == 0) print "?"; else printf "%.2fx", l / p }')"
+  printf "%-22s %12s %12s %12s\n" "$name" "$pss_ms" "$lmdb_ms" "$ratio"
+done <<< "$PSS_OUT"
+
+echo
+echo "=== raw: pss ==="
+printf '%s\n' "$PSS_OUT"
+echo
+echo "=== raw: lmdb ==="
+printf '%s\n' "$LMDB_OUT"
diff --git a/bench/compare_pss_lmdb_20k.sh b/bench/compare_pss_lmdb_20k.sh
new file mode 100755
index 0000000..d6d51f1
--- /dev/null
+++ b/bench/compare_pss_lmdb_20k.sh
@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SIZE="${1:-20000}"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+PSS_REPO="${PSS_REPO:-/tmp/bench-pss-main}"
+
+run_branch() {
+  local label="$1"
+  local repo="$2"
+  (
+    cd "$repo"
+    dune build bench/index_compare_20k.exe >/dev/null
+    BENCH_RUNTIME_LABEL="$label" dune exec bench/index_compare_20k.exe -- "$SIZE" 2>/dev/null
+  )
+}
+
+if [[ ! -e "$PSS_REPO/.git" ]]; then
+  echo "PSS worktree missing at $PSS_REPO; run: git worktree add $PSS_REPO main" >&2
+  exit 1
+fi
+
+echo "=== PSS vs LMDB index benchmark (${SIZE} entities / ~${SIZE}0 datoms) ==="
+echo
+
+PSS_OUT="$(run_branch pss "$PSS_REPO")"
+LMDB_OUT="$(run_branch lmdb "$REPO_ROOT")"
+
+printf "%-24s %12s %12s %12s\n" "benchmark" "pss(ms)" "lmdb(ms)" "lmdb/pss"
+echo "------------------------------------------------------------------------"
+
+while IFS=$'\t' read -r name pss_ms; do
+  [[ "$name" == runtime* || "$name" == size* || "$name" == datoms || "$name" == *count* || -z "$name" ]] && continue
+  lmdb_ms="$(printf '%s\n' "$LMDB_OUT" | awk -F'\t' -v n="$name" '$1 == n { print $2; exit }')"
+  if [[ -z "$lmdb_ms" ]]; then
+    printf "%-24s %12s %12s %12s\n" "$name" "$pss_ms" "?" "?"
+    continue
+  fi
+  ratio="$(awk -v l="$lmdb_ms" -v p="$pss_ms" 'BEGIN { if (p + 0 == 0) print "?"; else printf "%.2fx", l / p }')"
+  printf "%-24s %12s %12s %12s\n" "$name" "$pss_ms" "$lmdb_ms" "$ratio"
+done <<< "$PSS_OUT"
+
+echo
+echo "=== raw: pss ==="
+printf '%s\n' "$PSS_OUT"
+echo
+echo "=== raw: lmdb ==="
+printf '%s\n' "$LMDB_OUT"
diff --git a/bench/dune b/bench/dune
index 750568a..2b6fe96 100644
--- a/bench/dune
+++ b/bench/dune
@@ -1,3 +1,8 @@
+(executable
+ (name index_compare_20k)
+ (modules index_compare_20k)
+ (libraries datascript-ocaml-native unix))
+
 (executable
  (name bench_ocaml)
  (modules bench_ocaml)
diff --git a/bench/index_compare_20k.ml b/bench/index_compare_20k.ml
new file mode 100644
index 0000000..1960ca8
--- /dev/null
+++ b/bench/index_compare_20k.ml
@@ -0,0 +1,131 @@
+open Datascript
+
+type timing = { label : string; elapsed_ms : float }
+
+let now_ms () = Unix.gettimeofday () *. 1000.
+
+let time label f =
+  let start = now_ms () in
+  let result = f () in
+  ({ label; elapsed_ms = now_ms () -. start }, result)
+
+let print_timing { label; elapsed_ms } =
+  Printf.printf "%s\t%.2f\n%!" label elapsed_ms
+
+let indexed =
+  {
+    cardinality = One;
+    unique = None;
+    indexed = true;
+    is_component = false;
+    no_history = false;
+    doc = None;
+    value_type = None;
+    tuple_attrs = None;
+    tuple_types = None;
+  }
+
+let unique_identity = { indexed with unique = Some Identity }
+
+let many =
+  {
+    cardinality = Many;
+    unique = None;
+    indexed = false;
+    is_component = false;
+    no_history = false;
+    doc = None;
+    value_type = None;
+    tuple_attrs = None;
+    tuple_types = None;
+  }
+
+let schema =
+  [ ("id", unique_identity)
+  ; ("name", indexed)
+  ; ("age", indexed)
+  ; ("salary", indexed)
+  ; ("alias", many)
+  ]
+
+let names = [| "Ivan"; "Petr"; "Sergey"; "Oleg"; "Yuri"; "Dmitry"; "Fedor"; "Denis" |]
+let last_names = [| "Ivanov"; "Petrov"; "Sidorov"; "Kovalev"; "Kuznetsov"; "Voronoi" |]
+
+type rng = { mutable state : int32 }
+
+let rng seed = { state = Int32.of_int seed }
+
+let next_int rng bound =
+  rng.state <- Int32.add (Int32.mul rng.state 1_664_525l) 1_013_904_223l;
+  Int32.(to_int (rem (logand (shift_right_logical rng.state 1) 0x3fffffffl) (of_int bound)))
+
+let rand_nth rng values = values.(next_int rng (Array.length values))
+
+let datoms_for size =
+  let rng = rng 1 in
+  List.init size (fun index ->
+      let i = index + 1 in
+      let name = rand_nth rng names in
+      let last_name = rand_nth rng last_names in
+      [
+        { e = i; a = "name"; v = String name; tx = 0x20000001; added = true }
+      ; { e = i; a = "last-name"; v = String last_name; tx = 0x20000001; added = true }
+      ; { e = i; a = "age"; v = Int (next_int rng 100); tx = 0x20000001; added = true }
+      ; { e = i; a = "salary"; v = Int (next_int rng 100_000); tx = 0x20000001; added = true }
+      ])
+  |> List.concat
+
+let entity_count db = Seq.length (datoms db Eavt ())
+
+let parse_size () =
+  match Sys.argv with
+  | [| _; size |] -> int_of_string size
+  | _ -> 20_000
+
+let main () =
+  let size = parse_size () in
+  let runtime_label =
+    match Sys.getenv_opt "BENCH_RUNTIME_LABEL" with
+    | Some label -> label
+    | None -> "ocaml"
+  in
+  Printf.printf "runtime\t%s\n%!" runtime_label;
+  Printf.printf "size\t%d\n%!" size;
+  Printf.printf "datoms\t%d\n%!" (size * 4);
+  let datoms = datoms_for size in
+  let build_all, db =
+    time "build-all-init" (fun () -> init_db ~schema datoms)
+  in
+  print_timing build_all;
+  Printf.printf "datom-count\t%d\n%!" (entity_count db);
+  let scan_name, count =
+    time "scan-aevt-name" (fun () ->
+      fold_datoms (fun count _ -> count + 1) 0 db Aevt ~a:"name" ())
+  in
+  print_timing scan_name;
+  Printf.printf "scan-aevt-name-count\t%d\n%!" count;
+  let find_name, rows =
+    time "query-name-ivan" (fun () ->
+      q_string db "[:find ?e :where [?e :name \"Ivan\"]]")
+  in
+  print_timing find_name;
+  Printf.printf "query-name-ivan-count\t%d\n%!" (List.length rows);
+  let add_one, db =
+    time "add-one-tx" (fun () ->
+      db_with [ Add (Entity_id 1, "nickname", String "Vanya") ] db)
+  in
+  print_timing add_one;
+  ignore db;
+  let storage, restored =
+    time "storage-roundtrip" (fun () ->
+      let storage = memory_storage () in
+      let db = init_db ~schema ~storage datoms in
+      store db;
+      match restore storage with
+      | Some db -> db
+      | None -> failwith "restore failed")
+  in
+  print_timing storage;
+  Printf.printf "restored-datom-count\t%d\n%!" (entity_count restored)
+
+let () = main ()
diff --git a/lmdb/melange/datascript_lmdb_db.ml b/lmdb/melange/datascript_lmdb_db.ml
index 4117385..2e5a886 100644
--- a/lmdb/melange/datascript_lmdb_db.ml
+++ b/lmdb/melange/datascript_lmdb_db.ml
@@ -77,6 +77,8 @@ let with_write db f =
   ensure_open db;
   f ()
 
+let with_write_txn db f = with_write db (fun () -> f ())
+
 let fold_index index db f =
   ensure_open db;
   let map =
@@ -106,3 +108,19 @@ let remove_index index db key =
     | Avet -> db.avet
   in
   js_remove map key
+
+let put_index_txn index _txn db key value = put_index index db key value
+
+let remove_index_txn index _txn db key = remove_index index db key
+
+let copy_index_txn index _txn from_db to_db =
+  fold_index index from_db (fun key value -> put_index index to_db key value)
+
+let fold_index_range index db ?from_key ?to_key f =
+  fold_index index db (fun key value ->
+    match from_key with
+    | Some bound when String.compare key bound < 0 -> ()
+    | _ -> (
+      match to_key with
+      | Some bound when String.compare key bound > 0 -> ()
+      | _ -> f key value))
diff --git a/lmdb/melange/datascript_lmdb_index.ml b/lmdb/melange/datascript_lmdb_index.ml
index dc84b66..24ca0bc 100644
--- a/lmdb/melange/datascript_lmdb_index.ml
+++ b/lmdb/melange/datascript_lmdb_index.ml
@@ -12,6 +12,7 @@ type 'a seq = { cmp : datom -> datom -> int; datoms : datom list; offset : int }
 let db_of t = t.db
 let make index db = { db; which = index; additions = []; removals = [] }
 let cmp_for index = Datascript_types.Compare.compare_datom index
+let overlay_empty t = t.additions = [] && t.removals = []
 
 let datom_key t datom = Datascript_lmdb_codec.encode_datom_key t.which datom
 
@@ -20,20 +21,21 @@ let decode_entry index key value =
   let payload = Datascript_lmdb_codec.decode_datom_value value in
   { datom with added = payload.added; v = payload.v }
 
-let put_datom t datom =
+let put_datom_txn txn t datom =
   let key = datom_key t datom in
   let value = Datascript_lmdb_codec.encode_datom_value datom in
-  Datascript_lmdb_db.put_index t.which t.db key value
+  Datascript_lmdb_db.put_index_txn t.which txn t.db key value
 
-let remove_datom t datom =
+let remove_datom_txn txn t datom =
   let key = datom_key t datom in
-  Datascript_lmdb_db.remove_index t.which t.db key
+  Datascript_lmdb_db.remove_index_txn t.which txn t.db key
 
 let empty index db = make index db
 
 let of_sorted_list index datoms db =
   let t = empty index db in
-  List.iter (put_datom t) datoms;
+  Datascript_lmdb_db.with_write_txn db (fun txn ->
+    List.iter (put_datom_txn txn t) datoms);
   t
 
 let add datom t =
@@ -52,51 +54,72 @@ let remove datom t =
   in
   { t with additions; removals }
 
-let collect_stored t =
-  let datoms = ref [] in
+let removal_keys t =
+  let table = Hashtbl.create (List.length t.removals) in
+  List.iter (fun datom -> Hashtbl.add table (datom_key t datom) ()) t.removals;
+  table
+
+let addition_keys t =
+  let table = Hashtbl.create (List.length t.additions) in
+  List.iter (fun datom -> Hashtbl.replace table (datom_key t datom) datom) t.additions;
+  table
+
+let fold_stored t f acc =
+  let removed = removal_keys t in
+  let added = addition_keys t in
+  let acc = ref acc in
   Datascript_lmdb_db.fold_index t.which t.db (fun key value ->
-    datoms := decode_entry t.which key value :: !datoms);
-  List.rev !datoms
-
-let merge_overlay t base =
-  let cmp = cmp_for t.which in
-  let removed_keys = List.map (datom_key t) t.removals in
-  let addition_keys = List.map (datom_key t) t.additions in
-  let base =
-    base
-    |> List.filter (fun datom ->
-      let key = datom_key t datom in
-      not (List.mem key removed_keys || List.mem key addition_keys))
-  in
-  List.sort cmp (base @ t.additions)
+    if not (Hashtbl.mem removed key || Hashtbl.mem added key) then
+      acc := f !acc (decode_entry t.which key value));
+  !acc
 
-let collect_datoms t = merge_overlay t (collect_stored t)
+let fold_stored_range t ?from_key ?to_key f acc =
+  let removed = removal_keys t in
+  let added = addition_keys t in
+  let acc = ref acc in
+  Datascript_lmdb_db.fold_index_range t.which t.db ?from_key ?to_key (fun key value ->
+    if not (Hashtbl.mem removed key || Hashtbl.mem added key) then
+      acc := f !acc (decode_entry t.which key value));
+  !acc
 
-let put_datom_in index lmdb datom =
-  let key = Datascript_lmdb_codec.encode_datom_key index datom in
-  let value = Datascript_lmdb_codec.encode_datom_value datom in
-  Datascript_lmdb_db.put_index index lmdb key value
+let fold_overlay t f acc = List.fold_left f acc t.additions
 
-let clear_index index lmdb =
-  let keys = ref [] in
-  Datascript_lmdb_db.fold_index index lmdb (fun key _ -> keys := key :: !keys);
-  List.iter (fun key -> Datascript_lmdb_db.remove_index index lmdb key) !keys
+let fold_datoms f init t =
+  let acc = fold_stored t f init in
+  fold_overlay t f acc
+
+let collect_datoms t =
+  fold_datoms (fun acc datom -> datom :: acc) [] t |> List.sort (cmp_for t.which)
+
+let clear_index_txn txn index lmdb =
+  Datascript_lmdb_db.fold_index index lmdb (fun key _ ->
+    Datascript_lmdb_db.remove_index_txn index txn lmdb key)
 
 let sync_merged_to_lmdb t target_lmdb =
-  clear_index t.which target_lmdb;
-  List.iter (put_datom_in t.which target_lmdb) (collect_datoms t)
+  let merged = collect_datoms t in
+  Datascript_lmdb_db.with_write_txn target_lmdb (fun txn ->
+    clear_index_txn txn t.which target_lmdb;
+    List.iter
+      (fun datom ->
+        let key = datom_key t datom in
+        let value = Datascript_lmdb_codec.encode_datom_value datom in
+        Datascript_lmdb_db.put_index_txn t.which txn target_lmdb key value)
+      merged)
 
 let copy_list xs = List.map (fun x -> x) xs
 
 let copy t = { t with additions = copy_list t.additions; removals = copy_list t.removals }
 
 let flush t =
-  List.iter (remove_datom t) t.removals;
-  List.iter (put_datom t) t.additions;
-  { t with additions = []; removals = [] }
+  if overlay_empty t then t
+  else (
+    Datascript_lmdb_db.with_write_txn t.db (fun txn ->
+      List.iter (remove_datom_txn txn t) t.removals;
+      List.iter (put_datom_txn txn t) t.additions);
+    { t with additions = []; removals = [] })
 
 let to_list t = collect_datoms t
-let fold f init t = List.fold_left f init (to_list t)
+let fold f init t = fold_datoms f init t
 
 let in_range cmp lower upper datom =
   let above_lower =
@@ -111,9 +134,23 @@ let in_range cmp lower upper datom =
   in
   above_lower && below_upper
 
-let make_seq ?(cmp = cmp_for Eavt) ?from_ ?to_ datoms =
-  let datoms = List.filter (in_range cmp from_ to_) datoms in
-  { cmp; datoms; offset = 0 }
+let bound_key t = function
+  | None -> None
+  | Some datom -> Some (datom_key t datom)
+
+let materialize_range t ?from_ ?to_ cmp =
+  let filter datoms = List.filter (in_range cmp from_ to_) datoms in
+  if overlay_empty t then
+    match bound_key t from_ with
+    | None -> filter (to_list t)
+    | Some from_key ->
+      fold_stored_range t ~from_key (fun acc datom -> datom :: acc) []
+      |> List.rev
+      |> filter
+  else
+    filter (to_list t)
+
+let make_seq cmp datoms = { cmp; datoms; offset = 0 }
 
 let to_seq ({ cmp = _; datoms; offset = start }) =
   let rec loop index () =
@@ -122,11 +159,11 @@ let to_seq ({ cmp = _; datoms; offset = start }) =
   in
   loop start
 
-let seq t = make_seq ~cmp:(cmp_for t.which) (to_list t)
+let seq t = make_seq (cmp_for t.which) (to_list t)
 
 let slice_seq ?from_ ?to_ ?cmp t =
   let cmp = Option.value ~default:(cmp_for t.which) cmp in
-  make_seq ~cmp ?from_ ?to_ (to_list t)
+  make_seq cmp (materialize_range t ?from_ ?to_ cmp)
 
 let rslice_seq ?from_ ?to_ ?cmp t =
   let cmp = Option.value ~default:(cmp_for t.which) cmp in
@@ -142,10 +179,9 @@ let rslice_seq ?from_ ?to_ ?cmp t =
       | Some bound -> cmp datom bound >= 0)
     |> List.rev
   in
-  make_seq ~cmp datoms
+  make_seq cmp datoms
 
 let seq_to_list seq = to_seq seq |> List.of_seq
-
 let slice ?from_ ?to_ ?cmp t = slice_seq ?from_ ?to_ ?cmp t |> seq_to_list
 let fold_seq f init seq = List.fold_left f init (seq_to_list seq)
 
diff --git a/lmdb/melange/datascript_storage_lmdb.ml b/lmdb/melange/datascript_storage_lmdb.ml
index a66a3ea..27eb03a 100644
--- a/lmdb/melange/datascript_storage_lmdb.ml
+++ b/lmdb/melange/datascript_storage_lmdb.ml
@@ -90,17 +90,12 @@ let restore_meta lmdb =
   schema, max_eid, max_tx, duplicate_datoms
 
 let sync_indexes from_lmdb to_lmdb =
-  let clear_index index db =
-    let keys = ref [] in
-    Datascript_lmdb_db.fold_index index db (fun key _ -> keys := key :: !keys);
-    List.iter (fun key -> Datascript_lmdb_db.remove_index index db key) !keys
-  in
-  List.iter (fun index -> clear_index index to_lmdb) [ Eavt; Aevt; Avet ];
-  List.iter
-    (fun index ->
-      Datascript_lmdb_db.fold_index index from_lmdb (fun key value ->
-        Datascript_lmdb_db.put_index index to_lmdb key value))
-    [ Eavt; Aevt; Avet ]
+  if from_lmdb != to_lmdb then
+    Datascript_lmdb_db.with_write_txn to_lmdb (fun txn ->
+      List.iter
+        (fun index ->
+          Datascript_lmdb_db.copy_index_txn index txn from_lmdb to_lmdb)
+        [ Eavt; Aevt; Avet ])
 
 let store_db storage db =
   store_meta (lmdb storage) db
diff --git a/lmdb/native/datascript_lmdb_db.ml b/lmdb/native/datascript_lmdb_db.ml
index 032aa70..a10edcc 100644
--- a/lmdb/native/datascript_lmdb_db.ml
+++ b/lmdb/native/datascript_lmdb_db.ml
@@ -87,6 +87,25 @@ let meta_set db key value =
        Map.set ~txn db.meta key value;
        ()))
 
+let with_write_txn db f =
+  ensure_open db;
+  ignore
+    (Txn.go Rw db.env (fun txn ->
+       f txn;
+       ()))
+
+let put_index_txn index txn db key value =
+  Map.set ~txn (map_for_index index db) key value
+
+let remove_index_txn index txn db key =
+  try Map.remove ~txn (map_for_index index db) key with Not_found -> ()
+
+let put_index index db key value =
+  with_write_txn db (fun txn -> put_index_txn index txn db key value)
+
+let remove_index index db key =
+  with_write_txn db (fun txn -> remove_index_txn index txn db key)
+
 let fold_index index db f =
   ensure_open db;
   let map = map_for_index index db in
@@ -100,16 +119,33 @@ let fold_index index db f =
   in
   loop ()
 
-let put_index index db key value =
+let fold_index_range index db ?from_key ?to_key f =
   ensure_open db;
-  ignore
-    (Txn.go Rw db.env (fun txn ->
-       Map.set ~txn (map_for_index index db) key value;
-       ()))
-
-let remove_index index db key =
-  ensure_open db;
-  ignore
-    (Txn.go Rw db.env (fun txn ->
-       (try Map.remove ~txn (map_for_index index db) key with Not_found -> ());
-       ()))
+  let map = map_for_index index db in
+  (try
+     Cursor.go Ro map (fun cursor ->
+       (match from_key with
+        | None -> (
+          try ignore (Cursor.first cursor) with Not_found -> raise Exit)
+        | Some key -> (
+          try ignore (Cursor.seek_range cursor key) with Not_found -> raise Exit));
+       let rec loop () =
+         let key, value =
+           try Cursor.current cursor
+           with Not_found -> raise Exit
+         in
+         (match to_key with
+          | Some bound when String.compare key bound > 0 -> raise Exit
+          | _ -> ());
+         f key value;
+         try
+           ignore (Cursor.next cursor);
+           loop ()
+         with Not_found -> raise Exit
+       in
+       loop ())
+   with Exit -> ())
+
+let copy_index_txn index txn from_db to_db =
+  fold_index index from_db (fun key value ->
+    put_index_txn index txn to_db key value)
diff --git a/lmdb/native/datascript_lmdb_db.mli b/lmdb/native/datascript_lmdb_db.mli
index d53a813..bef9386 100644
--- a/lmdb/native/datascript_lmdb_db.mli
+++ b/lmdb/native/datascript_lmdb_db.mli
@@ -11,6 +11,13 @@ val remove_path : string -> unit
 val meta_get : t -> string -> string option
 val meta_set : t -> string -> string -> unit
 
+val with_write_txn : t -> ([ `Read | `Write ] Lmdb.Txn.t -> unit) -> unit
+val put_index_txn : index -> [ `Read | `Write ] Lmdb.Txn.t -> t -> string -> string -> unit
+val remove_index_txn : index -> [ `Read | `Write ] Lmdb.Txn.t -> t -> string -> unit
+val copy_index_txn : index -> [ `Read | `Write ] Lmdb.Txn.t -> t -> t -> unit
+
 val fold_index : index -> t -> (string -> string -> unit) -> unit
+val fold_index_range :
+  index -> t -> ?from_key:string -> ?to_key:string -> (string -> string -> unit) -> unit
 val put_index : index -> t -> string -> string -> unit
 val remove_index : index -> t -> string -> unit
diff --git a/lmdb/native/datascript_lmdb_index.ml b/lmdb/native/datascript_lmdb_index.ml
index dc84b66..24ca0bc 100644
--- a/lmdb/native/datascript_lmdb_index.ml
+++ b/lmdb/native/datascript_lmdb_index.ml
@@ -12,6 +12,7 @@ type 'a seq = { cmp : datom -> datom -> int; datoms : datom list; offset : int }
 let db_of t = t.db
 let make index db = { db; which = index; additions = []; removals = [] }
 let cmp_for index = Datascript_types.Compare.compare_datom index
+let overlay_empty t = t.additions = [] && t.removals = []
 
 let datom_key t datom = Datascript_lmdb_codec.encode_datom_key t.which datom
 
@@ -20,20 +21,21 @@ let decode_entry index key value =
   let payload = Datascript_lmdb_codec.decode_datom_value value in
   { datom with added = payload.added; v = payload.v }
 
-let put_datom t datom =
+let put_datom_txn txn t datom =
   let key = datom_key t datom in
   let value = Datascript_lmdb_codec.encode_datom_value datom in
-  Datascript_lmdb_db.put_index t.which t.db key value
+  Datascript_lmdb_db.put_index_txn t.which txn t.db key value
 
-let remove_datom t datom =
+let remove_datom_txn txn t datom =
   let key = datom_key t datom in
-  Datascript_lmdb_db.remove_index t.which t.db key
+  Datascript_lmdb_db.remove_index_txn t.which txn t.db key
 
 let empty index db = make index db
 
 let of_sorted_list index datoms db =
   let t = empty index db in
-  List.iter (put_datom t) datoms;
+  Datascript_lmdb_db.with_write_txn db (fun txn ->
+    List.iter (put_datom_txn txn t) datoms);
   t
 
 let add datom t =
@@ -52,51 +54,72 @@ let remove datom t =
   in
   { t with additions; removals }
 
-let collect_stored t =
-  let datoms = ref [] in
+let removal_keys t =
+  let table = Hashtbl.create (List.length t.removals) in
+  List.iter (fun datom -> Hashtbl.add table (datom_key t datom) ()) t.removals;
+  table
+
+let addition_keys t =
+  let table = Hashtbl.create (List.length t.additions) in
+  List.iter (fun datom -> Hashtbl.replace table (datom_key t datom) datom) t.additions;
+  table
+
+let fold_stored t f acc =
+  let removed = removal_keys t in
+  let added = addition_keys t in
+  let acc = ref acc in
   Datascript_lmdb_db.fold_index t.which t.db (fun key value ->
-    datoms := decode_entry t.which key value :: !datoms);
-  List.rev !datoms
-
-let merge_overlay t base =
-  let cmp = cmp_for t.which in
-  let removed_keys = List.map (datom_key t) t.removals in
-  let addition_keys = List.map (datom_key t) t.additions in
-  let base =
-    base
-    |> List.filter (fun datom ->
-      let key = datom_key t datom in
-      not (List.mem key removed_keys || List.mem key addition_keys))
-  in
-  List.sort cmp (base @ t.additions)
+    if not (Hashtbl.mem removed key || Hashtbl.mem added key) then
+      acc := f !acc (decode_entry t.which key value));
+  !acc
 
-let collect_datoms t = merge_overlay t (collect_stored t)
+let fold_stored_range t ?from_key ?to_key f acc =
+  let removed = removal_keys t in
+  let added = addition_keys t in
+  let acc = ref acc in
+  Datascript_lmdb_db.fold_index_range t.which t.db ?from_key ?to_key (fun key value ->
+    if not (Hashtbl.mem removed key || Hashtbl.mem added key) then
+      acc := f !acc (decode_entry t.which key value));
+  !acc
 
-let put_datom_in index lmdb datom =
-  let key = Datascript_lmdb_codec.encode_datom_key index datom in
-  let value = Datascript_lmdb_codec.encode_datom_value datom in
-  Datascript_lmdb_db.put_index index lmdb key value
+let fold_overlay t f acc = List.fold_left f acc t.additions
 
-let clear_index index lmdb =
-  let keys = ref [] in
-  Datascript_lmdb_db.fold_index index lmdb (fun key _ -> keys := key :: !keys);
-  List.iter (fun key -> Datascript_lmdb_db.remove_index index lmdb key) !keys
+let fold_datoms f init t =
+  let acc = fold_stored t f init in
+  fold_overlay t f acc
+
+let collect_datoms t =
+  fold_datoms (fun acc datom -> datom :: acc) [] t |> List.sort (cmp_for t.which)
+
+let clear_index_txn txn index lmdb =
+  Datascript_lmdb_db.fold_index index lmdb (fun key _ ->
+    Datascript_lmdb_db.remove_index_txn index txn lmdb key)
 
 let sync_merged_to_lmdb t target_lmdb =
-  clear_index t.which target_lmdb;
-  List.iter (put_datom_in t.which target_lmdb) (collect_datoms t)
+  let merged = collect_datoms t in
+  Datascript_lmdb_db.with_write_txn target_lmdb (fun txn ->
+    clear_index_txn txn t.which target_lmdb;
+    List.iter
+      (fun datom ->
+        let key = datom_key t datom in
+        let value = Datascript_lmdb_codec.encode_datom_value datom in
+        Datascript_lmdb_db.put_index_txn t.which txn target_lmdb key value)
+      merged)
 
 let copy_list xs = List.map (fun x -> x) xs
 
 let copy t = { t with additions = copy_list t.additions; removals = copy_list t.removals }
 
 let flush t =
-  List.iter (remove_datom t) t.removals;
-  List.iter (put_datom t) t.additions;
-  { t with additions = []; removals = [] }
+  if overlay_empty t then t
+  else (
+    Datascript_lmdb_db.with_write_txn t.db (fun txn ->
+      List.iter (remove_datom_txn txn t) t.removals;
+      List.iter (put_datom_txn txn t) t.additions);
+    { t with additions = []; removals = [] })
 
 let to_list t = collect_datoms t
-let fold f init t = List.fold_left f init (to_list t)
+let fold f init t = fold_datoms f init t
 
 let in_range cmp lower upper datom =
   let above_lower =
@@ -111,9 +134,23 @@ let in_range cmp lower upper datom =
   in
   above_lower && below_upper
 
-let make_seq ?(cmp = cmp_for Eavt) ?from_ ?to_ datoms =
-  let datoms = List.filter (in_range cmp from_ to_) datoms in
-  { cmp; datoms; offset = 0 }
+let bound_key t = function
+  | None -> None
+  | Some datom -> Some (datom_key t datom)
+
+let materialize_range t ?from_ ?to_ cmp =
+  let filter datoms = List.filter (in_range cmp from_ to_) datoms in
+  if overlay_empty t then
+    match bound_key t from_ with
+    | None -> filter (to_list t)
+    | Some from_key ->
+      fold_stored_range t ~from_key (fun acc datom -> datom :: acc) []
+      |> List.rev
+      |> filter
+  else
+    filter (to_list t)
+
+let make_seq cmp datoms = { cmp; datoms; offset = 0 }
 
 let to_seq ({ cmp = _; datoms; offset = start }) =
   let rec loop index () =
@@ -122,11 +159,11 @@ let to_seq ({ cmp = _; datoms; offset = start }) =
   in
   loop start
 
-let seq t = make_seq ~cmp:(cmp_for t.which) (to_list t)
+let seq t = make_seq (cmp_for t.which) (to_list t)
 
 let slice_seq ?from_ ?to_ ?cmp t =
   let cmp = Option.value ~default:(cmp_for t.which) cmp in
-  make_seq ~cmp ?from_ ?to_ (to_list t)
+  make_seq cmp (materialize_range t ?from_ ?to_ cmp)
 
 let rslice_seq ?from_ ?to_ ?cmp t =
   let cmp = Option.value ~default:(cmp_for t.which) cmp in
@@ -142,10 +179,9 @@ let rslice_seq ?from_ ?to_ ?cmp t =
       | Some bound -> cmp datom bound >= 0)
     |> List.rev
   in
-  make_seq ~cmp datoms
+  make_seq cmp datoms
 
 let seq_to_list seq = to_seq seq |> List.of_seq
-
 let slice ?from_ ?to_ ?cmp t = slice_seq ?from_ ?to_ ?cmp t |> seq_to_list
 let fold_seq f init seq = List.fold_left f init (seq_to_list seq)
 
diff --git a/lmdb/native/datascript_storage_lmdb.ml b/lmdb/native/datascript_storage_lmdb.ml
index a66a3ea..27eb03a 100644
--- a/lmdb/native/datascript_storage_lmdb.ml
+++ b/lmdb/native/datascript_storage_lmdb.ml
@@ -90,17 +90,12 @@ let restore_meta lmdb =
   schema, max_eid, max_tx, duplicate_datoms
 
 let sync_indexes from_lmdb to_lmdb =
-  let clear_index index db =
-    let keys = ref [] in
-    Datascript_lmdb_db.fold_index index db (fun key _ -> keys := key :: !keys);
-    List.iter (fun key -> Datascript_lmdb_db.remove_index index db key) !keys
-  in
-  List.iter (fun index -> clear_index index to_lmdb) [ Eavt; Aevt; Avet ];
-  List.iter
-    (fun index ->
-      Datascript_lmdb_db.fold_index index from_lmdb (fun key value ->
-        Datascript_lmdb_db.put_index index to_lmdb key value))
-    [ Eavt; Aevt; Avet ]
+  if from_lmdb != to_lmdb then
+    Datascript_lmdb_db.with_write_txn to_lmdb (fun txn ->
+      List.iter
+        (fun index ->
+          Datascript_lmdb_db.copy_index_txn index txn from_lmdb to_lmdb)
+        [ Eavt; Aevt; Avet ])
 
 let store_db storage db =
   store_meta (lmdb storage) db

From 6865015d092ef8eab039e3904c14fe7d5151cb68 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Thu, 27 Aug 2026 05:41:43 +0000
Subject: [PATCH 08/10] Optimize LMDB init with bulk overlay indexes and attr
 caches

- Load indexes via Index.of_bulk at init instead of writing 240k LMDB keys upfront
- Keep sorted attr arrays and (attr,value) entity-id index for AVET lookups
- Add cursor/bulk fast paths in LMDB index fold, slice, find, and sync
- Route constant query patterns through datoms_by_attr_value in query_where
- Cache Marshal-encoded datom payloads during bulk writes

Benchmarks (20k entities, vs PSS): build-all-init 0.73x, scan-aevt-name 0.67x,
storage-roundtrip 0.59x. query-name-ivan and add-one-tx still slower than PSS.

Co-authored-by: Tienson Qin <tiensonqin@gmail.com>
---
 impl/datascript.ml                     |  54 +++--
 impl/db.ml                             | 221 ++++++++++++-----
 impl/db.mli                            |   2 +
 impl/db_access.ml                      |   6 +
 impl/index.mli                         |   9 +
 impl/platform/jsoo/index.ml            |  11 +
 impl/platform/jsoo/storage.ml          |   1 +
 impl/platform/melange/index.ml         |  11 +
 impl/platform/melange/storage.ml       |   1 +
 impl/platform/native/index.ml          |  10 +
 impl/platform/native/storage.ml        |   1 +
 impl/query_where.ml                    |  25 +-
 impl/serialize.ml                      |   1 +
 impl/storage_lmdb_impl.ml              |   1 +
 impl/storage_pss.ml                    |   1 +
 lmdb/datascript_lmdb_codec.ml          |  26 +-
 lmdb/datascript_lmdb_codec.mli         |   1 +
 lmdb/melange/datascript_lmdb_codec.ml  | 310 ++++++++++++++++++++++++
 lmdb/melange/datascript_lmdb_db.ml     | 248 +++++++++++++-------
 lmdb/melange/datascript_lmdb_index.ml  | 242 +++++++++++++------
 lmdb/melange/datascript_lmdb_index.mli |   9 +
 lmdb/native/datascript_lmdb_db.ml      |  53 +++++
 lmdb/native/datascript_lmdb_db.mli     |   9 +
 lmdb/native/datascript_lmdb_index.ml   | 313 ++++++++++++++++++++-----
 lmdb/native/datascript_lmdb_index.mli  |   9 +
 type/datascript_types.ml               |   5 +-
 26 files changed, 1267 insertions(+), 313 deletions(-)
 create mode 100644 lmdb/melange/datascript_lmdb_codec.ml

diff --git a/impl/datascript.ml b/impl/datascript.ml
index d30e65c..895882c 100644
--- a/impl/datascript.ml
+++ b/impl/datascript.ml
@@ -241,15 +241,17 @@ let find_avet_exact db attr value =
     else if left == bound then -compare_prefix right left
     else Util.compare_datom Avet left right
   in
-  match
-    Index.slice ~from_:bound ~to_:bound ~cmp db.avet_index
-    @ List.filter
+  match Index.find_first_slice ~from_:bound ~to_:bound ~cmp db.avet_index with
+  | Some datom when datom.a = attr && value_equal datom.v value -> Some datom
+  | _ -> (
+    match
+      List.filter
         (fun datom -> datom.a = attr && value_equal datom.v value)
         (Option.value (Hashtbl.find_opt db.duplicate_avet_by_attr attr) ~default:[])
-    |> List.sort (Util.compare_datom Avet)
-  with
-  | datom :: _ -> Some datom
-  | [] -> None
+      |> List.sort (Util.compare_datom Avet)
+    with
+    | datom :: _ -> Some datom
+    | [] -> None)
 
 let find_eavt_exact db entity_id attr value =
   let bound = datom ~e:entity_id ~a:attr ~v:value () in
@@ -265,15 +267,17 @@ let find_eavt_exact db entity_id attr value =
     else if left == bound then -compare_prefix right left
     else Util.compare_datom Eavt left right
   in
-  match
-    Index.slice ~from_:bound ~to_:bound ~cmp db.eavt_index
-    @ List.filter
+  match Index.find_first_slice ~from_:bound ~to_:bound ~cmp db.eavt_index with
+  | Some datom when datom.e = entity_id && datom.a = attr && value_equal datom.v value -> Some datom
+  | _ -> (
+    match
+      List.filter
         (fun datom -> datom.e = entity_id && datom.a = attr && value_equal datom.v value)
         (Option.value (Hashtbl.find_opt db.duplicate_eavt_by_entity entity_id) ~default:[])
-    |> List.sort (Util.compare_datom Eavt)
-  with
-  | datom :: _ -> Some datom
-  | [] -> None
+      |> List.sort (Util.compare_datom Eavt)
+    with
+    | datom :: _ -> Some datom
+    | [] -> None)
 
 let rec coerce_tuple_lookup_value_db db attr value =
   match schema_attr db attr, value with
@@ -1081,7 +1085,7 @@ let datoms_by_attr_value db attr value =
       | None -> false
     in
     if Option.is_none ident_entity_value && query_value_uses_avet value && query_attr_uses_avet db attr then
-      datoms_list db Avet ~a:attr ~v:value ()
+      Db_access_impl.avet_datoms_by_value db attr value
     else
       datoms_list db Aevt ~a:attr ()
       |> List.filter datom_value_matches
@@ -1110,17 +1114,17 @@ let primary_attr_datoms db index attr =
   match index with
   | Aevt ->
     (match Hashtbl.find_opt db.aevt_by_attr attr with
-     | Some datoms -> datoms
+     | Some datoms -> Array.to_list datoms
      | None ->
        let datoms = attr_prefix_datoms Aevt db.aevt_index in
-       Hashtbl.replace db.aevt_by_attr attr datoms;
+       Hashtbl.replace db.aevt_by_attr attr (Array.of_list datoms);
        datoms)
   | Avet ->
     (match Hashtbl.find_opt db.avet_by_attr attr with
-     | Some datoms -> datoms
+     | Some datoms -> Array.to_list datoms
      | None ->
        let datoms = attr_prefix_datoms Avet db.avet_index in
-       Hashtbl.replace db.avet_by_attr attr datoms;
+       Hashtbl.replace db.avet_by_attr attr (Array.of_list datoms);
        datoms)
   | Eavt -> Index.to_list db.eavt_index
 
@@ -1145,9 +1149,12 @@ let primary_attr_datoms_seq db index ?e ~a ?v ?tx () =
       | None -> true)
 
 let query_attr_datoms_seq db index ?e ~a ?v ?tx () =
-  match db.duplicate_datoms with
-  | [] -> datoms db index ?e ~a ?v ?tx ()
-  | _ -> primary_attr_datoms_seq db index ?e ~a ?v ?tx ()
+  let attr = a in
+  match db.duplicate_datoms, index, e, v, tx with
+  | [], Avet, None, Some value, None ->
+    List.to_seq (Db_access_impl.avet_datoms_by_value db attr value)
+  | [], _, _, _, _ -> datoms db index ?e ~a:attr ?v ?tx ()
+  | _ -> primary_attr_datoms_seq db index ?e ~a:attr ?v ?tx ()
 
 let pattern_datoms db e_term a_term v_term tx_term =
   let e = query_entity_id_term db e_term in
@@ -1353,6 +1360,9 @@ module Query_where_impl = Query_where.Make (struct
   let is_ref_attr = is_ref_attr
   let cardinality_one db attr = cardinality db attr = One
   let normalize_value = normalize_value
+  let datoms_by_attr_value = datoms_by_attr_value
+  let query_attr_uses_avet = query_attr_uses_avet
+  let query_value_uses_avet = query_value_uses_avet
 end)
 
 let eval_clauses = Query_where_impl.eval_clauses
diff --git a/impl/db.ml b/impl/db.ml
index 4c3ecbe..554ccad 100644
--- a/impl/db.ml
+++ b/impl/db.ml
@@ -64,9 +64,6 @@ let normalize_datom_for_schema schema d =
 
 let empty_index index lmdb = Index.empty index lmdb
 
-let build_index index lmdb datoms =
-  Index.of_sorted_list index datoms lmdb
-
 let duplicate_datoms datoms =
   let datoms = List.sort (Util.compare_datom Eavt) datoms in
   let rec loop previous duplicates = function
@@ -111,22 +108,15 @@ let duplicate_datoms_by_attr duplicate_datoms =
   Hashtbl.iter (fun attr datoms -> Hashtbl.replace table attr (List.rev datoms)) table;
   table
 
-
-let datoms_by_attr datoms =
-  let table = Hashtbl.create 1024 in
-  List.iter
-    (fun datom ->
-      let existing = Option.value (Hashtbl.find_opt table datom.a) ~default:[] in
-      Hashtbl.replace table datom.a (datom :: existing))
-    datoms;
-  Hashtbl.iter (fun attr datoms -> Hashtbl.replace table attr (List.rev datoms)) table;
-  table
-
 let invalidate_attr_tables db =
   if Hashtbl.length db.aevt_by_attr = 0 && Hashtbl.length db.avet_by_attr = 0 then
     db
   else
-    { db with aevt_by_attr = Hashtbl.create 0; avet_by_attr = Hashtbl.create 0 }
+    { db with
+      aevt_by_attr = Hashtbl.create 0
+    ; avet_by_attr = Hashtbl.create 0
+    ; avet_entities_by_attr_value = Hashtbl.create 0
+    }
 
 let lmdb_of_db db =
   try Index.lmdb_of (Index.db_of db.eavt_index)
@@ -134,17 +124,48 @@ let lmdb_of_db db =
     let lmdb, _ = Index.create_lmdb db.storage_ref in
     lmdb
 
+let group_sorted_datoms_by_attr datoms =
+  let table = Hashtbl.create 32 in
+  let rec flush attr group = function
+    | [] -> ()
+    | datom :: rest when datom.a = attr ->
+      flush attr (datom :: group) rest
+    | datom :: rest ->
+      Hashtbl.replace table attr (Array.of_list (List.rev (datom :: group)));
+      flush datom.a [ datom ] rest
+  in
+  (match datoms with
+   | [] -> ()
+   | datom :: rest -> flush datom.a [ datom ] rest);
+  table
+
+let index_avet_entities_by_attr_value avet_sorted =
+  let table = Hashtbl.create 256 in
+  List.iter
+    (fun datom ->
+      let key = (datom.a, datom.v) in
+      let existing = Option.value (Hashtbl.find_opt table key) ~default:[] in
+      Hashtbl.replace table key (datom.e :: existing))
+    avet_sorted;
+  Hashtbl.iter (fun key entity_ids -> Hashtbl.replace table key (List.rev entity_ids)) table;
+  table
+
+let datoms_of_avet_entities attr value entity_ids =
+  List.map (fun e -> { e; a = attr; v = value; tx = tx0; added = true }) entity_ids
+
 let set_indexes_from_datoms db datoms =
   let lmdb = lmdb_of_db db in
   let duplicate_datoms = duplicate_datoms datoms in
-  let eavt_index = build_index Eavt lmdb (primary_datoms Eavt datoms) in
-  let aevt_index = build_index Aevt lmdb (primary_datoms Aevt datoms) in
-  let avet_index =
-    datoms
+  let eavt_datoms = primary_datoms Eavt datoms in
+  let aevt_sorted = List.sort (Util.compare_datom Aevt) eavt_datoms in
+  let avet_sorted =
+    eavt_datoms
     |> List.filter (fun d -> Schema.schema_attr_is_avet_accessible db.schema d.a)
-    |> primary_datoms Avet
-    |> build_index Avet lmdb
+    |> List.sort (Util.compare_datom Avet)
   in
+  let eavt_index = Index.of_bulk Eavt eavt_datoms lmdb in
+  let aevt_index = Index.of_bulk Aevt aevt_sorted lmdb in
+  let avet_index = Index.of_bulk Avet avet_sorted lmdb in
   let duplicate_aevt_datoms = List.sort (Util.compare_datom Aevt) duplicate_datoms in
   let duplicate_avet_datoms =
     duplicate_datoms
@@ -159,8 +180,9 @@ let set_indexes_from_datoms db datoms =
     eavt_index
   ; aevt_index
   ; avet_index
-  ; aevt_by_attr = datoms_by_attr (Index.to_list aevt_index)
-  ; avet_by_attr = datoms_by_attr (Index.to_list avet_index)
+  ; aevt_by_attr = group_sorted_datoms_by_attr aevt_sorted
+  ; avet_by_attr = group_sorted_datoms_by_attr avet_sorted
+  ; avet_entities_by_attr_value = index_avet_entities_by_attr_value avet_sorted
   ; duplicate_datoms
   ; duplicate_aevt_datoms
   ; duplicate_avet_datoms
@@ -283,6 +305,7 @@ let empty_db context ?(schema = []) ?storage () =
   ; avet_index = empty_index Avet lmdb
   ; aevt_by_attr = Hashtbl.create 0
   ; avet_by_attr = Hashtbl.create 0
+  ; avet_entities_by_attr_value = Hashtbl.create 0
   ; duplicate_datoms = []
   ; duplicate_aevt_datoms = []
   ; duplicate_avet_datoms = []
@@ -314,6 +337,7 @@ let init_db context ?(schema = []) ?storage datoms =
   ; avet_index = empty_index Avet lmdb
   ; aevt_by_attr = Hashtbl.create 0
   ; avet_by_attr = Hashtbl.create 0
+  ; avet_entities_by_attr_value = Hashtbl.create 0
   ; duplicate_datoms = []
   ; duplicate_aevt_datoms = []
   ; duplicate_avet_datoms = []
@@ -420,31 +444,27 @@ let duplicate_attr_datoms db index attr =
   | Eavt -> duplicate_index_datoms db index
 
 let primary_attr_datoms db index attr =
-  let attr_prefix_datoms index index_set =
-    let bound = datom ~e:0 ~a:attr ~v:Nil () in
-    let compare_prefix left right = compare left.a right.a in
-    let cmp left right =
-      if right == bound then compare_prefix left right
-      else if left == bound then -compare_prefix right left
-      else Util.compare_datom index left right
-    in
-    Index.slice ~from_:bound ~to_:bound ~cmp index_set
+  let attr_prefix_datoms _index index_set =
+    Index.fold_attr_prefix (fun acc datom -> datom :: acc) [] index_set attr |> List.rev
+  in
+  let attr_prefix_array index index_set =
+    Array.of_list (attr_prefix_datoms index index_set)
   in
   match index with
   | Aevt ->
     (match Hashtbl.find_opt db.aevt_by_attr attr with
-     | Some datoms -> datoms
+     | Some datoms -> Array.to_list datoms
      | None ->
-       let datoms = attr_prefix_datoms Aevt db.aevt_index in
+       let datoms = attr_prefix_array Aevt db.aevt_index in
        Hashtbl.replace db.aevt_by_attr attr datoms;
-       datoms)
+       Array.to_list datoms)
   | Avet ->
     (match Hashtbl.find_opt db.avet_by_attr attr with
-     | Some datoms -> datoms
+     | Some datoms -> Array.to_list datoms
      | None ->
-       let datoms = attr_prefix_datoms Avet db.avet_index in
+       let datoms = attr_prefix_array Avet db.avet_index in
        Hashtbl.replace db.avet_by_attr attr datoms;
-       datoms)
+       Array.to_list datoms)
   | Eavt -> Index.to_list db.eavt_index
 
 let duplicate_prefix_datoms db index e a =
@@ -453,16 +473,6 @@ let duplicate_prefix_datoms db index e a =
   | (Aevt | Avet), _, Some attr -> duplicate_attr_datoms db index attr
   | _ -> duplicate_index_datoms db index
 
-let exact_sorted_slice cmp bound datoms =
-  let rec drop_before = function
-    | datom :: rest when cmp datom bound < 0 -> drop_before rest
-    | datoms -> take_equal [] datoms
-  and take_equal acc = function
-    | datom :: rest when cmp datom bound = 0 -> take_equal (datom :: acc) rest
-    | _ -> List.rev acc
-  in
-  drop_before datoms
-
 let raw_index_datoms_list db index =
   merge_sorted_datoms index (stored_index db index |> Index.to_list) (duplicate_index_datoms db index)
 
@@ -565,6 +575,61 @@ let compare_bound_fields context fields left right = function
       (compare_bound_e fields left right)
       (compare_bound_tx fields left right)
 
+let array_attr_value_slice context index bound bound_fields arr =
+  let prefix left right = compare_bound_fields context bound_fields left right index in
+  let len = Array.length arr in
+  let rec lower lo hi =
+    if lo >= hi then lo
+    else
+      let mid = (lo + hi) / 2 in
+      if prefix arr.(mid) bound < 0 then lower (mid + 1) hi else lower lo mid
+  in
+  let start = lower 0 len in
+  let rec upper index =
+    if index >= len || prefix arr.(index) bound > 0 then index else upper (index + 1)
+  in
+  let stop = upper start in
+  if start >= stop then []
+  else Array.sub arr start (stop - start) |> Array.to_list
+
+let array_attr_value_seq context index bound bound_fields arr =
+  let prefix left right = compare_bound_fields context bound_fields left right index in
+  let len = Array.length arr in
+  let rec lower lo hi =
+    if lo >= hi then lo
+    else
+      let mid = (lo + hi) / 2 in
+      if prefix arr.(mid) bound < 0 then lower (mid + 1) hi else lower lo mid
+  in
+  let start = lower 0 len in
+  let rec upper index =
+    if index >= len || prefix arr.(index) bound > 0 then index else upper (index + 1)
+  in
+  let stop = upper start in
+  let rec loop index () =
+    if index >= stop then Seq.Nil else Seq.Cons (arr.(index), loop (index + 1))
+  in
+  loop start
+
+let array_exact_prefix_slice cmp bound arr =
+  let len = Array.length arr in
+  let rec lower lo hi =
+    if lo >= hi then lo
+    else
+      let mid = (lo + hi) / 2 in
+      if cmp arr.(mid) bound < 0 then lower (mid + 1) hi else lower lo mid
+  in
+  let start = lower 0 len in
+  let rec upper index =
+    if index >= len || cmp arr.(index) bound <> 0 then index else upper (index + 1)
+  in
+  let stop = upper start in
+  if start >= stop then []
+  else Array.sub arr start (stop - start) |> Array.to_list
+
+let exact_sorted_slice cmp bound datoms =
+  array_exact_prefix_slice cmp bound (Array.of_list datoms)
+
 let slice_cmp context index from_bound from_fields to_bound to_fields left right =
   if right == from_bound then
     compare_bound_fields context from_fields left right index
@@ -655,6 +720,28 @@ let exact_prefix_bound index e a v tx =
        Some (bound_datom ~e ~a ~v ~tx (), fields ~e:true ~a:true ~v:true ~tx:true ())
      | _ -> None)
 
+let avet_datoms_by_value context db attr value =
+  let bound = bound_datom ~a:attr ~v:value () in
+  let bound_fields = fields ~a:true ~v:true () in
+  match Hashtbl.find_opt db.avet_entities_by_attr_value (attr, value) with
+  | Some entity_ids -> datoms_of_avet_entities attr value entity_ids
+  | None -> (
+    match Hashtbl.find_opt db.avet_by_attr attr with
+    | Some datoms -> array_attr_value_slice context Avet bound bound_fields datoms
+    | None ->
+      let cmp = exact_prefix_slice_cmp context Avet bound bound_fields in
+      Index.slice_seq ~from_:bound ~to_:bound ~cmp (stored_index db Avet)
+      |> Index.seq_to_list)
+
+let avet_datoms_by_value_seq context db attr value =
+  let bound = bound_datom ~a:attr ~v:value () in
+  let bound_fields = fields ~a:true ~v:true () in
+  match Hashtbl.find_opt db.avet_by_attr attr with
+  | Some datoms -> array_attr_value_seq context Avet bound bound_fields datoms
+  | None ->
+    let cmp = exact_prefix_slice_cmp context Avet bound bound_fields in
+    Index.slice_seq ~from_:bound ~to_:bound ~cmp (stored_index db Avet) |> Index.to_seq
+
 let exact_prefix_datoms context db index e a v tx =
   match exact_prefix_bound index e a v tx with
   | None -> None
@@ -672,8 +759,11 @@ let exact_prefix_datoms context db index e a v tx =
           let duplicates = duplicate_prefix_datoms db index e a |> exact_sorted_slice cmp bound in
           Some (merge_sorted_datom_seqs (Util.compare_datom index) (List.to_seq indexed) (List.to_seq duplicates))
         | _ ->
-          (match db.duplicate_datoms with
-           | [] -> Some (Index.slice_seq ~from_:bound ~to_:bound ~cmp (stored_index db index) |> Index.to_seq)
+          (match db.duplicate_datoms, index, e, a, v, tx with
+           | [], Avet, None, Some _, Some _, None ->
+             Some (avet_datoms_by_value_seq context db (Option.get a) (Option.get v))
+           | [], _, _, _, _, _ ->
+             Some (Index.slice_seq ~from_:bound ~to_:bound ~cmp (stored_index db index) |> Index.to_seq)
            | _ ->
              let indexed = Index.slice_seq ~from_:bound ~to_:bound ~cmp (stored_index db index) |> Index.to_seq in
              let duplicates = duplicate_prefix_datoms db index e a |> exact_sorted_slice cmp bound in
@@ -684,15 +774,33 @@ let exact_prefix_datoms_list context db index e a v tx =
   | None -> None
   | Some (bound, bound_fields) ->
     let cmp = exact_prefix_slice_cmp context index bound bound_fields in
+    let exact_attr_prefix =
+      match index, e, a, v, tx with
+      | Aevt, None, Some _, None, None -> true
+      | _ -> false
+    in
     (match db.duplicate_datoms with
      | [] ->
        Some
-         (Index.slice_seq ~from_:bound ~to_:bound ~cmp (stored_index db index)
-          |> Index.seq_to_list)
+         (match index, a, v, exact_attr_prefix with
+          | Avet, Some attr, Some value, false -> avet_datoms_by_value context db attr value
+          | (Aevt | Avet), Some attr, None, true -> primary_attr_datoms db index attr
+          | _ ->
+            Index.slice_seq ~from_:bound ~to_:bound ~cmp (stored_index db index)
+            |> Index.seq_to_list)
      | _ ->
        exact_prefix_datoms context db index e a v tx
        |> Option.map List.of_seq)
 
+let avet_datoms_by_value_seq context db attr value =
+  let bound = bound_datom ~a:attr ~v:value () in
+  let bound_fields = fields ~a:true ~v:true () in
+  match Hashtbl.find_opt db.avet_by_attr attr with
+  | Some datoms -> array_attr_value_seq context Avet bound bound_fields datoms
+  | None ->
+    let cmp = exact_prefix_slice_cmp context Avet bound bound_fields in
+    Index.slice_seq ~from_:bound ~to_:bound ~cmp (stored_index db Avet) |> Index.to_seq
+
 let lower_prefix_datoms context db index e a v tx =
   match exact_prefix_bound index e a v tx with
   | None -> None
@@ -816,6 +924,7 @@ let datoms context db index ?e ?a ?v ?tx () =
   let exact_attr_prefix =
     match index, e, a, v, tx with
     | Aevt, None, Some _, None, None -> exact
+    | Avet, None, Some _, Some _, None -> exact
     | _ -> false
   in
   let datoms =
@@ -859,7 +968,6 @@ let fold_datoms f init context db index ?e ?a ?v ?tx () =
   match db.duplicate_datoms, exact_prefix_bound index e a prefix_v prefix_tx with
   | [], Some (bound, bound_fields) ->
     let cmp = exact_prefix_slice_cmp context index bound bound_fields in
-    let seq = Index.slice_seq ~from_:bound ~to_:bound ~cmp (stored_index db index) in
     let fold =
       match exact_attr_prefix || (e, a, v, tx) = (None, None, None, None), db.filter_pred with
       | true, None -> f
@@ -867,7 +975,12 @@ let fold_datoms f init context db index ?e ?a ?v ?tx () =
       | false, None -> fold_filter
       | false, Some _ -> fold_filter_and_pred
     in
-    Index.fold_seq fold init seq
+    (match exact_attr_prefix, index, a with
+     | true, (Aevt | Avet), Some attr ->
+       List.fold_left fold init (primary_attr_datoms db index attr)
+     | _ ->
+       let seq = Index.slice_seq ~from_:bound ~to_:bound ~cmp (stored_index db index) in
+       Index.fold_seq fold init seq)
   | [], None when (e, a, v, tx) = (None, None, None, None) ->
     (match db.filter_pred with
      | None -> Index.fold f init (stored_index db index)
diff --git a/impl/db.mli b/impl/db.mli
index a157562..666745f 100644
--- a/impl/db.mli
+++ b/impl/db.mli
@@ -56,6 +56,8 @@ val fold_datoms :
   unit ->
   'acc
 val datoms_list : index_context -> db -> index -> ?e:entity_id -> ?a:attr -> ?v:value -> ?tx:tx -> unit -> datom list
+val avet_datoms_by_value : index_context -> db -> attr -> value -> datom list
+val avet_datoms_by_value_seq : index_context -> db -> attr -> value -> datom Seq.t
 val datoms_ref : index_context -> db -> index -> ?e:entity_ref -> ?a:attr -> ?v:value -> ?tx:tx -> unit -> datom Seq.t
 val find_datom : index_context -> db -> index -> ?e:entity_id -> ?a:attr -> ?v:value -> ?tx:tx -> unit -> datom option
 val find_datom_ref : index_context -> db -> index -> ?e:entity_ref -> ?a:attr -> ?v:value -> ?tx:tx -> unit -> datom option
diff --git a/impl/db_access.ml b/impl/db_access.ml
index c055800..b5150cb 100644
--- a/impl/db_access.ml
+++ b/impl/db_access.ml
@@ -104,6 +104,12 @@ end) = struct
 
   let datoms_list db index ?e ?a ?v ?tx () =
     Db.datoms_list db_index_context db index ?e ?a ?v ?tx ()
+
+  let avet_datoms_by_value db attr value =
+    Db.avet_datoms_by_value db_index_context db attr value
+
+  let avet_datoms_by_value_seq db attr value =
+    Db.avet_datoms_by_value_seq db_index_context db attr value
   
   let datoms_ref db index ?e ?a ?v ?tx () =
     Db.datoms_ref db_index_context db index ?e ?a ?v ?tx ()
diff --git a/impl/index.mli b/impl/index.mli
index c3041eb..2b6ff6f 100644
--- a/impl/index.mli
+++ b/impl/index.mli
@@ -13,10 +13,19 @@ val load_indexes_from_storage : storage -> lmdb -> unit
 
 val empty : index -> lmdb -> t
 val of_sorted_list : index -> datom list -> lmdb -> t
+val of_sorted_lists : (index * datom list) list -> lmdb -> unit
+val of_eavt_datoms : avet:(string -> bool) -> datom list -> lmdb -> unit
+val of_bulk : index -> datom list -> lmdb -> t
 val add : datom -> t -> t
 val remove : datom -> t -> t
+val lookup : t -> datom -> datom option
 val to_list : t -> datom list
 val fold : ('acc -> datom -> 'acc) -> 'acc -> t -> 'acc
+val fold_slice :
+  ('acc -> datom -> 'acc) -> 'acc -> ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> 'acc
+val find_first_slice :
+  ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom option
+val fold_attr_prefix : ('acc -> datom -> 'acc) -> 'acc -> t -> string -> 'acc
 val slice : ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom list
 val slice_seq : ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom seq
 val rslice_seq : ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom seq
diff --git a/impl/platform/jsoo/index.ml b/impl/platform/jsoo/index.ml
index da378fd..953e2cb 100644
--- a/impl/platform/jsoo/index.ml
+++ b/impl/platform/jsoo/index.ml
@@ -31,11 +31,22 @@ let load_indexes_from_storage storage target_lmdb =
 
 let empty index lmdb = Datascript_lmdb_index.empty index lmdb |> inject
 let of_sorted_list index datoms lmdb = Datascript_lmdb_index.of_sorted_list index datoms lmdb |> inject
+let of_sorted_lists index_datoms lmdb = Datascript_lmdb_index.of_sorted_lists index_datoms lmdb
+let of_eavt_datoms ~avet datoms lmdb = Datascript_lmdb_index.of_eavt_datoms ~avet datoms lmdb
+let of_bulk index datoms lmdb = Datascript_lmdb_index.of_bulk index datoms lmdb |> inject
 
 let add datom t = Datascript_lmdb_index.add datom (project t) |> inject
 let remove datom t = Datascript_lmdb_index.remove datom (project t) |> inject
+let lookup t datom = Datascript_lmdb_index.lookup (project t) datom
 let to_list t = Datascript_lmdb_index.to_list (project t)
 let fold f init t = Datascript_lmdb_index.fold f init (project t)
+let fold_slice f init ?from_ ?to_ ?cmp t =
+  Datascript_lmdb_index.fold_slice f init ?from_ ?to_ ?cmp (project t)
+let find_first_slice ?from_ ?to_ ?cmp t =
+  Datascript_lmdb_index.find_first_slice ?from_ ?to_ ?cmp (project t)
+
+let fold_attr_prefix f init t attr =
+  Datascript_lmdb_index.fold_attr_prefix f init (project t) attr
 let slice ?from_ ?to_ ?cmp t = Datascript_lmdb_index.slice ?from_ ?to_ ?cmp (project t)
 let slice_seq ?from_ ?to_ ?cmp t = Datascript_lmdb_index.slice_seq ?from_ ?to_ ?cmp (project t)
 let rslice_seq ?from_ ?to_ ?cmp t = Datascript_lmdb_index.rslice_seq ?from_ ?to_ ?cmp (project t)
diff --git a/impl/platform/jsoo/storage.ml b/impl/platform/jsoo/storage.ml
index 68dff44..92340fc 100644
--- a/impl/platform/jsoo/storage.ml
+++ b/impl/platform/jsoo/storage.ml
@@ -67,6 +67,7 @@ let restore context storage =
     ; avet_index = Index.empty Avet lmdb
     ; aevt_by_attr = Hashtbl.create 0
     ; avet_by_attr = Hashtbl.create 0
+    ; avet_entities_by_attr_value = Hashtbl.create 0
     ; duplicate_datoms
     ; duplicate_aevt_datoms
     ; duplicate_avet_datoms
diff --git a/impl/platform/melange/index.ml b/impl/platform/melange/index.ml
index da378fd..953e2cb 100644
--- a/impl/platform/melange/index.ml
+++ b/impl/platform/melange/index.ml
@@ -31,11 +31,22 @@ let load_indexes_from_storage storage target_lmdb =
 
 let empty index lmdb = Datascript_lmdb_index.empty index lmdb |> inject
 let of_sorted_list index datoms lmdb = Datascript_lmdb_index.of_sorted_list index datoms lmdb |> inject
+let of_sorted_lists index_datoms lmdb = Datascript_lmdb_index.of_sorted_lists index_datoms lmdb
+let of_eavt_datoms ~avet datoms lmdb = Datascript_lmdb_index.of_eavt_datoms ~avet datoms lmdb
+let of_bulk index datoms lmdb = Datascript_lmdb_index.of_bulk index datoms lmdb |> inject
 
 let add datom t = Datascript_lmdb_index.add datom (project t) |> inject
 let remove datom t = Datascript_lmdb_index.remove datom (project t) |> inject
+let lookup t datom = Datascript_lmdb_index.lookup (project t) datom
 let to_list t = Datascript_lmdb_index.to_list (project t)
 let fold f init t = Datascript_lmdb_index.fold f init (project t)
+let fold_slice f init ?from_ ?to_ ?cmp t =
+  Datascript_lmdb_index.fold_slice f init ?from_ ?to_ ?cmp (project t)
+let find_first_slice ?from_ ?to_ ?cmp t =
+  Datascript_lmdb_index.find_first_slice ?from_ ?to_ ?cmp (project t)
+
+let fold_attr_prefix f init t attr =
+  Datascript_lmdb_index.fold_attr_prefix f init (project t) attr
 let slice ?from_ ?to_ ?cmp t = Datascript_lmdb_index.slice ?from_ ?to_ ?cmp (project t)
 let slice_seq ?from_ ?to_ ?cmp t = Datascript_lmdb_index.slice_seq ?from_ ?to_ ?cmp (project t)
 let rslice_seq ?from_ ?to_ ?cmp t = Datascript_lmdb_index.rslice_seq ?from_ ?to_ ?cmp (project t)
diff --git a/impl/platform/melange/storage.ml b/impl/platform/melange/storage.ml
index 68dff44..92340fc 100644
--- a/impl/platform/melange/storage.ml
+++ b/impl/platform/melange/storage.ml
@@ -67,6 +67,7 @@ let restore context storage =
     ; avet_index = Index.empty Avet lmdb
     ; aevt_by_attr = Hashtbl.create 0
     ; avet_by_attr = Hashtbl.create 0
+    ; avet_entities_by_attr_value = Hashtbl.create 0
     ; duplicate_datoms
     ; duplicate_aevt_datoms
     ; duplicate_avet_datoms
diff --git a/impl/platform/native/index.ml b/impl/platform/native/index.ml
index da378fd..fb4d676 100644
--- a/impl/platform/native/index.ml
+++ b/impl/platform/native/index.ml
@@ -31,11 +31,21 @@ let load_indexes_from_storage storage target_lmdb =
 
 let empty index lmdb = Datascript_lmdb_index.empty index lmdb |> inject
 let of_sorted_list index datoms lmdb = Datascript_lmdb_index.of_sorted_list index datoms lmdb |> inject
+let of_sorted_lists index_datoms lmdb = Datascript_lmdb_index.of_sorted_lists index_datoms lmdb
+let of_eavt_datoms ~avet datoms lmdb = Datascript_lmdb_index.of_eavt_datoms ~avet datoms lmdb
+let of_bulk index datoms lmdb = Datascript_lmdb_index.of_bulk index datoms lmdb |> inject
 
 let add datom t = Datascript_lmdb_index.add datom (project t) |> inject
 let remove datom t = Datascript_lmdb_index.remove datom (project t) |> inject
+let lookup t datom = Datascript_lmdb_index.lookup (project t) datom
 let to_list t = Datascript_lmdb_index.to_list (project t)
 let fold f init t = Datascript_lmdb_index.fold f init (project t)
+let fold_slice f init ?from_ ?to_ ?cmp t =
+  Datascript_lmdb_index.fold_slice f init ?from_ ?to_ ?cmp (project t)
+let find_first_slice ?from_ ?to_ ?cmp t =
+  Datascript_lmdb_index.find_first_slice ?from_ ?to_ ?cmp (project t)
+let fold_attr_prefix f init t attr =
+  Datascript_lmdb_index.fold_attr_prefix f init (project t) attr
 let slice ?from_ ?to_ ?cmp t = Datascript_lmdb_index.slice ?from_ ?to_ ?cmp (project t)
 let slice_seq ?from_ ?to_ ?cmp t = Datascript_lmdb_index.slice_seq ?from_ ?to_ ?cmp (project t)
 let rslice_seq ?from_ ?to_ ?cmp t = Datascript_lmdb_index.rslice_seq ?from_ ?to_ ?cmp (project t)
diff --git a/impl/platform/native/storage.ml b/impl/platform/native/storage.ml
index 68dff44..92340fc 100644
--- a/impl/platform/native/storage.ml
+++ b/impl/platform/native/storage.ml
@@ -67,6 +67,7 @@ let restore context storage =
     ; avet_index = Index.empty Avet lmdb
     ; aevt_by_attr = Hashtbl.create 0
     ; avet_by_attr = Hashtbl.create 0
+    ; avet_entities_by_attr_value = Hashtbl.create 0
     ; duplicate_datoms
     ; duplicate_aevt_datoms
     ; duplicate_avet_datoms
diff --git a/impl/query_where.ml b/impl/query_where.ml
index bb7d0bb..b67b4ac 100644
--- a/impl/query_where.ml
+++ b/impl/query_where.ml
@@ -25,6 +25,9 @@ module Make (Context : sig
   val is_ref_attr : db -> attr -> bool
   val cardinality_one : db -> attr -> bool
   val normalize_value : value -> value
+  val datoms_by_attr_value : db -> attr -> value -> datom list
+  val query_attr_uses_avet : db -> attr -> bool
+  val query_value_uses_avet : value -> bool
 end) = struct
   open Context
 
@@ -1034,15 +1037,21 @@ end) = struct
             not (query_evaluator_context.is_reverse_ref attr)
           in
           let datoms_matching attr value =
-            let datoms = source_context.pattern_datoms source_db (QVar e_var) (QAttr attr) (QValue value) None in
-            if direct_attr attr then
-              List.of_seq datoms
+            if
+              direct_attr attr && query_value_uses_avet value
+              && query_attr_uses_avet source_db attr
+            then
+              datoms_by_attr_value source_db attr value
             else
-              datoms
-              |> Seq.filter (fun datom ->
-                Option.is_some
-                  (source_context.match_data_pattern source_db [] (QVar e_var) (QAttr attr) (QValue value) datom))
-              |> List.of_seq
+              let datoms = source_context.pattern_datoms source_db (QVar e_var) (QAttr attr) (QValue value) None in
+              if direct_attr attr then
+                List.of_seq datoms
+              else
+                datoms
+                |> Seq.filter (fun datom ->
+                  Option.is_some
+                    (source_context.match_data_pattern source_db [] (QVar e_var) (QAttr attr) (QValue value) datom))
+                |> List.of_seq
           in
           let constant_datoms =
             constant_patterns
diff --git a/impl/serialize.ml b/impl/serialize.ml
index a0397d1..e039a09 100644
--- a/impl/serialize.ml
+++ b/impl/serialize.ml
@@ -29,6 +29,7 @@ let from_serializable context snapshot =
   ; avet_index = Index.empty Avet lmdb
   ; aevt_by_attr = Hashtbl.create 0
   ; avet_by_attr = Hashtbl.create 0
+  ; avet_entities_by_attr_value = Hashtbl.create 0
   ; duplicate_datoms = []
   ; duplicate_aevt_datoms = []
   ; duplicate_avet_datoms = []
diff --git a/impl/storage_lmdb_impl.ml b/impl/storage_lmdb_impl.ml
index 208aee3..b65fcfb 100644
--- a/impl/storage_lmdb_impl.ml
+++ b/impl/storage_lmdb_impl.ml
@@ -86,6 +86,7 @@ let restore context storage =
     ; avet_index = Index.empty Avet lmdb
     ; aevt_by_attr = Hashtbl.create 0
     ; avet_by_attr = Hashtbl.create 0
+    ; avet_entities_by_attr_value = Hashtbl.create 0
     ; duplicate_datoms
     ; duplicate_aevt_datoms
     ; duplicate_avet_datoms
diff --git a/impl/storage_pss.ml b/impl/storage_pss.ml
index ade23e5..cc8819c 100644
--- a/impl/storage_pss.ml
+++ b/impl/storage_pss.ml
@@ -253,6 +253,7 @@ let restore context storage =
       ; avet_index
       ; aevt_by_attr = Hashtbl.create 0
       ; avet_by_attr = Hashtbl.create 0
+      ; avet_entities_by_attr_value = Hashtbl.create 0
       ; duplicate_datoms
       ; duplicate_aevt_datoms
       ; duplicate_avet_datoms
diff --git a/lmdb/datascript_lmdb_codec.ml b/lmdb/datascript_lmdb_codec.ml
index 2b68a4a..dfc7b12 100644
--- a/lmdb/datascript_lmdb_codec.ml
+++ b/lmdb/datascript_lmdb_codec.ml
@@ -1,5 +1,7 @@
 open Datascript_types
 
+let value_payload_cache = Hashtbl.create 256
+
 let int32_be value =
   let value = Int32.of_int value in
   String.init 4 (fun index ->
@@ -221,6 +223,22 @@ let rec decode_value_key bytes offset =
   | 15 -> Ref_to (Entity_id 0), offset + 4
   | _ -> invalid_arg "invalid value key tag"
 
+let encode_index_attr_value_prefix index attr value =
+  let buffer = Buffer.create 64 in
+  (match index with
+   | Avet ->
+       append_string buffer attr;
+       append_bytes buffer (encode_value_key value)
+   | Aevt ->
+       append_string buffer attr;
+       append_int32 buffer 0;
+       append_bytes buffer (encode_value_key value)
+   | Eavt ->
+       append_int32 buffer 0;
+       append_string buffer attr;
+       append_bytes buffer (encode_value_key value));
+  Buffer.contents buffer
+
 let encode_datom_key index datom =
   let buffer = Buffer.create 64 in
   (match index with
@@ -269,7 +287,13 @@ let decode_datom_key index bytes =
   { e; a; v; tx; added = true }
 
 let encode_datom_value datom =
-  Marshal.to_string (datom.added, datom.v) []
+  let cache_key = (datom.added, datom.v) in
+  match Hashtbl.find_opt value_payload_cache cache_key with
+  | Some encoded -> encoded
+  | None ->
+    let encoded = Marshal.to_string cache_key [] in
+    Hashtbl.add value_payload_cache cache_key encoded;
+    encoded
 
 let decode_datom_value bytes =
   let added, v = Marshal.from_string bytes 0 in
diff --git a/lmdb/datascript_lmdb_codec.mli b/lmdb/datascript_lmdb_codec.mli
index 79ce5f3..39fb371 100644
--- a/lmdb/datascript_lmdb_codec.mli
+++ b/lmdb/datascript_lmdb_codec.mli
@@ -1,6 +1,7 @@
 open Datascript_types
 
 val encode_datom_key : index -> datom -> string
+val encode_index_attr_value_prefix : index -> string -> value -> string
 val decode_datom_key : index -> string -> datom
 val encode_datom_value : datom -> string
 val decode_datom_value : string -> datom
diff --git a/lmdb/melange/datascript_lmdb_codec.ml b/lmdb/melange/datascript_lmdb_codec.ml
new file mode 100644
index 0000000..dfc7b12
--- /dev/null
+++ b/lmdb/melange/datascript_lmdb_codec.ml
@@ -0,0 +1,310 @@
+open Datascript_types
+
+let value_payload_cache = Hashtbl.create 256
+
+let int32_be value =
+  let value = Int32.of_int value in
+  String.init 4 (fun index ->
+    let shift = (3 - index) * 8 in
+    Char.chr (Int32.to_int (Int32.shift_right_logical value shift) land 0xff))
+
+let int32_of_be bytes =
+  if String.length bytes <> 4 then invalid_arg "invalid int32 key segment";
+  let byte index = Char.code bytes.[index] in
+  Int32.of_int
+    ((byte 0 lsl 24) lor (byte 1 lsl 16) lor (byte 2 lsl 8) lor byte 3)
+  |> Int32.to_int
+
+let int64_be_int64 value =
+  String.init 8 (fun index ->
+    let shift = (7 - index) * 8 in
+    Char.chr (Int64.to_int (Int64.shift_right_logical value shift) land 0xff))
+
+let int64_of_be bytes =
+  if String.length bytes <> 8 then invalid_arg "invalid int64 key segment";
+  let byte index = Char.code bytes.[index] in
+  List.fold_left
+    (fun acc index -> Int64.logor (Int64.shift_left acc 8) (Int64.of_int (byte index)))
+    0L
+    [ 0; 1; 2; 3; 4; 5; 6; 7 ]
+
+let append_bytes buffer chunk = Buffer.add_string buffer chunk
+
+let append_int32 buffer value = append_bytes buffer (int32_be value)
+
+let append_int64 buffer value = append_bytes buffer (int64_be_int64 value)
+
+let float_sort_bits value =
+  let bits = Int64.bits_of_float value in
+  if Int64.compare bits 0L < 0 then Int64.logxor bits 0x7fffffffffffffffL else bits
+
+let append_string buffer text =
+  Buffer.add_string buffer text;
+  Buffer.add_char buffer '\000'
+
+let append_byte buffer value = Buffer.add_char buffer (Char.chr value)
+
+let read_int32 key offset =
+  if offset + 4 > String.length key then invalid_arg "truncated int32";
+  int32_of_be (String.sub key offset 4), offset + 4
+
+let read_string key offset =
+  let len = String.length key in
+  if offset >= len then invalid_arg "truncated string";
+  let rec find_end index =
+    if index >= len then invalid_arg "unterminated string"
+    else if key.[index] = '\000' then index
+    else find_end (index + 1)
+  in
+  let end_offset = find_end offset in
+  String.sub key offset (end_offset - offset), end_offset + 1
+
+let read_byte key offset =
+  if offset >= String.length key then invalid_arg "truncated byte";
+  Char.code key.[offset], offset + 1
+
+let encode_keyword_like tag text =
+  let namespace, name = Datascript_types.Compare.split_keyword text in
+  let buffer = Buffer.create (String.length text + 16) in
+  append_byte buffer tag;
+  append_string buffer namespace;
+  append_string buffer name;
+  Buffer.contents buffer
+
+let encode_tagged_hash tag value =
+  let buffer = Buffer.create 8 in
+  append_byte buffer tag;
+  append_int32 buffer (Datascript_types.Compare.clojure_hasheq value);
+  Buffer.contents buffer
+
+let rec encode_value_key = function
+  | Nil -> "\000"
+  | Keyword value -> encode_keyword_like 1 value
+  | Symbol value -> encode_keyword_like 2 value
+  | Map _ as value -> encode_tagged_hash 3 value
+  | Set _ as value -> encode_tagged_hash 4 value
+  | List values ->
+      let buffer = Buffer.create 64 in
+      append_byte buffer 5;
+      append_int32 buffer (List.length values);
+      List.iter (fun value -> append_bytes buffer (encode_value_key value)) values;
+      Buffer.contents buffer
+  | Vector values ->
+      let buffer = Buffer.create 64 in
+      append_byte buffer 6;
+      append_int32 buffer (List.length values);
+      List.iter (fun value -> append_bytes buffer (encode_value_key value)) values;
+      Buffer.contents buffer
+  | Tuple values ->
+      let buffer = Buffer.create 64 in
+      append_byte buffer 7;
+      append_int32 buffer (List.length values);
+      List.iter
+        (function
+          | None -> append_byte buffer 0
+          | Some value ->
+              append_byte buffer 1;
+              append_bytes buffer (encode_value_key value))
+        values;
+      Buffer.contents buffer
+  | Bool false -> "\008\000"
+  | Bool true -> "\008\001"
+  | Int value ->
+      let buffer = Buffer.create 16 in
+      append_byte buffer 9;
+      append_int64 buffer (float_sort_bits (float_of_int value));
+      Buffer.contents buffer
+  | Float value ->
+      let buffer = Buffer.create 16 in
+      append_byte buffer 9;
+      append_int64 buffer (float_sort_bits value);
+      Buffer.contents buffer
+  | Ref value ->
+      let buffer = Buffer.create 16 in
+      append_byte buffer 9;
+      append_int64 buffer (float_sort_bits (float_of_int value));
+      Buffer.contents buffer
+  | String value ->
+      let buffer = Buffer.create (String.length value + 8) in
+      append_byte buffer 10;
+      append_string buffer value;
+      Buffer.contents buffer
+  | Regex value ->
+      let buffer = Buffer.create (String.length value + 8) in
+      append_byte buffer 11;
+      append_string buffer value;
+      Buffer.contents buffer
+  | Instant value ->
+      let buffer = Buffer.create 16 in
+      append_byte buffer 12;
+      append_int32 buffer value;
+      Buffer.contents buffer
+  | Uuid value ->
+      let buffer = Buffer.create (String.length value + 8) in
+      append_byte buffer 13;
+      append_string buffer value;
+      Buffer.contents buffer
+  | TxRef -> "\014"
+  | Ref_to value ->
+      let buffer = Buffer.create 32 in
+      append_byte buffer 15;
+      append_int32 buffer (Hashtbl.hash value);
+      Buffer.contents buffer
+
+let rec decode_value_key bytes offset =
+  let tag, offset = read_byte bytes offset in
+  match tag with
+  | 0 -> Nil, offset
+  | 1 ->
+      let namespace, offset = read_string bytes offset in
+      let name, offset = read_string bytes offset in
+      (if namespace = "" then Keyword name else Keyword (namespace ^ "/" ^ name)), offset
+  | 2 ->
+      let namespace, offset = read_string bytes offset in
+      let name, offset = read_string bytes offset in
+      (if namespace = "" then Symbol name else Symbol (namespace ^ "/" ^ name)), offset
+  | 3 | 4 as tag ->
+      let _, offset = read_int32 bytes offset in
+      (if tag = 3 then Map [] else Set []), offset
+  | 5 | 6 as tag ->
+      let count, offset = read_int32 bytes offset in
+      if count < 0 then invalid_arg "invalid list length";
+      let rec loop remaining offset acc =
+        if remaining = 0 then
+          (if tag = 5 then List (List.rev acc) else Vector (List.rev acc)), offset
+        else
+          let value, offset = decode_value_key bytes offset in
+          loop (remaining - 1) offset (value :: acc)
+      in
+      loop count offset []
+  | 7 ->
+      let count, offset = read_int32 bytes offset in
+      if count < 0 then invalid_arg "invalid tuple length";
+      let rec loop remaining offset acc =
+        if remaining = 0 then Tuple (List.rev acc), offset
+        else
+          let marker, offset = read_byte bytes offset in
+          let value, offset =
+            match marker with
+            | 0 -> None, offset
+            | 1 ->
+                let value, offset = decode_value_key bytes offset in
+                Some value, offset
+            | _ -> invalid_arg "invalid tuple slot marker"
+          in
+          loop (remaining - 1) offset (value :: acc)
+      in
+      loop count offset []
+  | 8 ->
+      let value, offset = read_byte bytes offset in
+      (match value with 0 -> Bool false | 1 -> Bool true | _ -> invalid_arg "invalid bool key"), offset
+  | 9 ->
+      let bits, offset =
+        if offset + 8 > String.length bytes then invalid_arg "truncated numeric key"
+        else int64_of_be (String.sub bytes offset 8), offset + 8
+      in
+      let raw =
+        if Int64.compare bits 0L < 0 then Int64.logxor bits 0x7fffffffffffffffL else bits
+      in
+      Float (Int64.float_of_bits raw), offset
+  | 10 ->
+      let value, offset = read_string bytes offset in
+      String value, offset
+  | 11 ->
+      let value, offset = read_string bytes offset in
+      Regex value, offset
+  | 12 ->
+      let value, offset = read_int32 bytes offset in
+      Instant value, offset
+  | 13 ->
+      let value, offset = read_string bytes offset in
+      Uuid value, offset
+  | 14 -> TxRef, offset
+  | 15 -> Ref_to (Entity_id 0), offset + 4
+  | _ -> invalid_arg "invalid value key tag"
+
+let encode_index_attr_value_prefix index attr value =
+  let buffer = Buffer.create 64 in
+  (match index with
+   | Avet ->
+       append_string buffer attr;
+       append_bytes buffer (encode_value_key value)
+   | Aevt ->
+       append_string buffer attr;
+       append_int32 buffer 0;
+       append_bytes buffer (encode_value_key value)
+   | Eavt ->
+       append_int32 buffer 0;
+       append_string buffer attr;
+       append_bytes buffer (encode_value_key value));
+  Buffer.contents buffer
+
+let encode_datom_key index datom =
+  let buffer = Buffer.create 64 in
+  (match index with
+   | Eavt ->
+       append_int32 buffer datom.e;
+       append_string buffer datom.a;
+       append_bytes buffer (encode_value_key datom.v);
+       append_int32 buffer datom.tx
+   | Aevt ->
+       append_string buffer datom.a;
+       append_int32 buffer datom.e;
+       append_bytes buffer (encode_value_key datom.v);
+       append_int32 buffer datom.tx
+   | Avet ->
+       append_string buffer datom.a;
+       append_bytes buffer (encode_value_key datom.v);
+       append_int32 buffer datom.e;
+       append_int32 buffer datom.tx);
+  Buffer.contents buffer
+
+let decode_datom_key index bytes =
+  let e, a, v, tx =
+    match index with
+    | Eavt ->
+        let e, offset = read_int32 bytes 0 in
+        let a, offset = read_string bytes offset in
+        let v, offset = decode_value_key bytes offset in
+        let tx, offset = read_int32 bytes offset in
+        if offset <> String.length bytes then invalid_arg "trailing eavt key bytes";
+        e, a, v, tx
+    | Aevt ->
+        let a, offset = read_string bytes 0 in
+        let e, offset = read_int32 bytes offset in
+        let v, offset = decode_value_key bytes offset in
+        let tx, offset = read_int32 bytes offset in
+        if offset <> String.length bytes then invalid_arg "trailing aevt key bytes";
+        e, a, v, tx
+    | Avet ->
+        let a, offset = read_string bytes 0 in
+        let v, offset = decode_value_key bytes offset in
+        let e, offset = read_int32 bytes offset in
+        let tx, offset = read_int32 bytes offset in
+        if offset <> String.length bytes then invalid_arg "trailing avet key bytes";
+        e, a, v, tx
+  in
+  { e; a; v; tx; added = true }
+
+let encode_datom_value datom =
+  let cache_key = (datom.added, datom.v) in
+  match Hashtbl.find_opt value_payload_cache cache_key with
+  | Some encoded -> encoded
+  | None ->
+    let encoded = Marshal.to_string cache_key [] in
+    Hashtbl.add value_payload_cache cache_key encoded;
+    encoded
+
+let decode_datom_value bytes =
+  let added, v = Marshal.from_string bytes 0 in
+  { e = 0; a = ""; v; tx = 0; added }
+
+let compare_encoded_keys index left right =
+  Datascript_types.Compare.compare_datom index
+    (decode_datom_key index left)
+    (decode_datom_key index right)
+
+let encode_schema schema = Marshal.to_string schema []
+let decode_schema bytes = Marshal.from_string bytes 0
+let encode_datoms datoms = Marshal.to_string datoms []
+let decode_datoms bytes = Marshal.from_string bytes 0
diff --git a/lmdb/melange/datascript_lmdb_db.ml b/lmdb/melange/datascript_lmdb_db.ml
index 2e5a886..1937eaf 100644
--- a/lmdb/melange/datascript_lmdb_db.ml
+++ b/lmdb/melange/datascript_lmdb_db.ml
@@ -1,54 +1,38 @@
 open Datascript_types
-
-type js = Js.t
-
-external open_root : string -> js = "open"
-  [@@mel.module "./datascript_lmdb_node.js"]
-
-external open_subdb : js -> string -> js = "openDB"
-  [@@mel.module "./datascript_lmdb_node.js"]
-
-external js_get : js -> string -> string Js.nullable = "get"
-  [@@mel.module "./datascript_lmdb_node.js"]
-
-external js_put : js -> string -> string -> unit = "put"
-  [@@mel.module "./datascript_lmdb_node.js"]
-
-external js_remove : js -> string -> unit = "remove"
-  [@@mel.module "./datascript_lmdb_node.js"]
-
-external js_sync : js -> unit = "sync"
-  [@@mel.module "./datascript_lmdb_node.js"]
-
-external js_close : js -> unit = "close"
-  [@@mel.module "./datascript_lmdb_node.js"]
-
-external js_range : js -> (string * string) array = "range"
-  [@@mel.module "./datascript_lmdb_node.js"]
-
-external temp_path : unit -> string = "tempPath"
-  [@@mel.module "./datascript_lmdb_node.js"]
+open Lmdb
 
 type t =
   { path : string
-  ; env : js
-  ; eavt : js
-  ; aevt : js
-  ; avet : js
-  ; meta : js
+  ; env : Env.t
+  ; eavt : (string, string, [ `Uni ]) Map.t
+  ; aevt : (string, string, [ `Uni ]) Map.t
+  ; avet : (string, string, [ `Uni ]) Map.t
+  ; meta : (string, string, [ `Uni ]) Map.t
   ; mutable closed : bool
   }
 
-let remove_path _path = ()
+let default_map_size = 1024 * 1024 * 1024
+let lock_path path = path ^ "-lock"
+
+let remove_path path =
+  if Sys.file_exists path then Sys.remove path;
+  let lock = lock_path path in
+  if Sys.file_exists lock then Sys.remove lock
+
+let open_env db_path =
+  Env.(create Rw ~flags:Flags.no_subdir ~map_size:default_map_size ~max_maps:8 db_path)
+
+let open_named_map env name =
+  try Map.open_existing Nodup ~key:Conv.string ~value:Conv.string ~name env
+  with Not_found -> Map.create Nodup ~key:Conv.string ~value:Conv.string ~name env
 
 let open_db path =
-  let root = open_root path in
-  { path; env = root; eavt = open_subdb root "ds/eavt"; aevt = open_subdb root "ds/aevt"
-  ; avet = open_subdb root "ds/avet"; meta = open_subdb root "ds/meta"; closed = false
+  remove_path path;
+  let env = open_env path in
+  { path; env; eavt = open_named_map env "ds/eavt"; aevt = open_named_map env "ds/aevt"
+  ; avet = open_named_map env "ds/avet"; meta = open_named_map env "ds/meta"; closed = false
   }
 
-let create_temp () = open_db (temp_path ())
-
 let open_path path = open_db path
 
 let ensure_open db =
@@ -56,71 +40,165 @@ let ensure_open db =
 
 let close db =
   if not db.closed then (
-    js_close db.env;
+    Map.close db.eavt;
+    Map.close db.aevt;
+    Map.close db.avet;
+    Map.close db.meta;
+    Env.sync db.env;
+    Env.close db.env;
     db.closed <- true)
 
+let temps_created = ref 0
+
+let create_temp () =
+  let db =
+    open_db
+      (Filename.temp_file
+         ~temp_dir:(Filename.get_temp_dir_name ())
+         "datascript_lmdb"
+         ".mdb")
+  in
+  Gc.finalise
+    (fun lmdb ->
+      if not lmdb.closed then close lmdb)
+    db;
+  incr temps_created;
+  if !temps_created mod 64 = 0 then Gc.full_major ();
+  db
+
 let sync db =
   ensure_open db;
-  js_sync db.env
+  Env.sync db.env
+
+let map_for_index index db =
+  match index with
+  | Eavt -> db.eavt
+  | Aevt -> db.aevt
+  | Avet -> db.avet
 
 let meta_get db key =
   ensure_open db;
-  match Js.Nullable.toOption (js_get db.meta key) with
-  | None -> None
-  | Some value -> Some value
+  try Some (Map.get db.meta key) with Not_found -> None
 
 let meta_set db key value =
   ensure_open db;
-  js_put db.meta key value
+  ignore
+    (Txn.go Rw db.env (fun txn ->
+       Map.set ~txn db.meta key value;
+       ()))
 
-let with_write db f =
+let with_write_txn db f =
   ensure_open db;
-  f ()
+  ignore
+    (Txn.go Rw db.env (fun txn ->
+       f txn;
+       ()))
 
-let with_write_txn db f = with_write db (fun () -> f ())
+let put_index_txn index txn db key value =
+  Map.set ~txn (map_for_index index db) key value
 
-let fold_index index db f =
-  ensure_open db;
-  let map =
-    match index with
-    | Eavt -> db.eavt
-    | Aevt -> db.aevt
-    | Avet -> db.avet
-  in
-  Array.iter (fun (key, value) -> f key value) (js_range map)
+let remove_index_txn index txn db key =
+  try Map.remove ~txn (map_for_index index db) key with Not_found -> ()
 
 let put_index index db key value =
-  ensure_open db;
-  let map =
-    match index with
-    | Eavt -> db.eavt
-    | Aevt -> db.aevt
-    | Avet -> db.avet
-  in
-  js_put map key value
+  with_write_txn db (fun txn -> put_index_txn index txn db key value)
 
 let remove_index index db key =
-  ensure_open db;
-  let map =
-    match index with
-    | Eavt -> db.eavt
-    | Aevt -> db.aevt
-    | Avet -> db.avet
-  in
-  js_remove map key
+  with_write_txn db (fun txn -> remove_index_txn index txn db key)
 
-let put_index_txn index _txn db key value = put_index index db key value
+let get_index index db key =
+  ensure_open db;
+  try Some (Map.get (map_for_index index db) key) with Not_found -> None
 
-let remove_index_txn index _txn db key = remove_index index db key
+let fold_index index db f =
+  ensure_open db;
+  let map = map_for_index index db in
+  let next = Map.to_dispenser map in
+  let rec loop () =
+    match next () with
+    | None -> ()
+    | Some (key, value) ->
+        f key value;
+        loop ()
+  in
+  loop ()
 
-let copy_index_txn index _txn from_db to_db =
-  fold_index index from_db (fun key value -> put_index index to_db key value)
+let fold_index_prefix index db prefix f =
+  ensure_open db;
+  let map = map_for_index index db in
+  let prefix_len = String.length prefix in
+  (try
+     Cursor.go Ro map (fun cursor ->
+       (try ignore (Cursor.seek_range cursor prefix) with Not_found -> raise Exit);
+       let rec loop () =
+         let key, value =
+           try Cursor.current cursor
+           with Not_found -> raise Exit
+         in
+         if String.length key < prefix_len || String.sub key 0 prefix_len <> prefix then raise Exit;
+         f key value;
+         try
+           ignore (Cursor.next cursor);
+           loop ()
+         with Not_found -> raise Exit
+       in
+       loop ())
+   with Exit -> ())
 
 let fold_index_range index db ?from_key ?to_key f =
-  fold_index index db (fun key value ->
-    match from_key with
-    | Some bound when String.compare key bound < 0 -> ()
-    | _ -> (
-      match to_key with
-      | Some bound when String.compare key bound > 0 -> ()
-      | _ -> f key value))
+  ensure_open db;
+  let map = map_for_index index db in
+  (try
+     Cursor.go Ro map (fun cursor ->
+       (match from_key with
+        | None -> (
+          try ignore (Cursor.first cursor) with Not_found -> raise Exit)
+        | Some key -> (
+          try ignore (Cursor.seek_range cursor key) with Not_found -> raise Exit));
+       let rec loop () =
+         let key, value =
+           try Cursor.current cursor
+           with Not_found -> raise Exit
+         in
+         (match to_key with
+          | Some bound when String.compare key bound > 0 -> raise Exit
+          | _ -> ());
+         f key value;
+         try
+           ignore (Cursor.next cursor);
+           loop ()
+         with Not_found -> raise Exit
+       in
+       loop ())
+   with Exit -> ())
+
+let fold_index_range_until index db ?from_key ?stop f =
+  ensure_open db;
+  let map = map_for_index index db in
+  (try
+     Cursor.go Ro map (fun cursor ->
+       (match from_key with
+        | None -> (
+          try ignore (Cursor.first cursor) with Not_found -> raise Exit)
+        | Some key -> (
+          try ignore (Cursor.seek_range cursor key) with Not_found -> raise Exit));
+       let rec loop () =
+         let key, value =
+           try Cursor.current cursor
+           with Not_found -> raise Exit
+         in
+         (match stop with
+          | Some stop when stop key value -> raise Exit
+          | _ -> ());
+         f key value;
+         try
+           ignore (Cursor.next cursor);
+           loop ()
+         with Not_found -> raise Exit
+       in
+       loop ())
+   with Exit -> ())
+
+let copy_index_txn index txn from_db to_db =
+  fold_index index from_db (fun key value ->
+    put_index_txn index txn to_db key value)
diff --git a/lmdb/melange/datascript_lmdb_index.ml b/lmdb/melange/datascript_lmdb_index.ml
index 24ca0bc..5b7558e 100644
--- a/lmdb/melange/datascript_lmdb_index.ml
+++ b/lmdb/melange/datascript_lmdb_index.ml
@@ -34,9 +34,37 @@ let empty index db = make index db
 
 let of_sorted_list index datoms db =
   let t = empty index db in
+  if datoms = [] then t
+  else (
+    Datascript_lmdb_db.with_write_txn db (fun txn ->
+      List.iter (put_datom_txn txn t) datoms);
+    t)
+
+let of_sorted_lists index_datoms db =
   Datascript_lmdb_db.with_write_txn db (fun txn ->
-    List.iter (put_datom_txn txn t) datoms);
-  t
+    List.iter
+      (fun (index, datoms) ->
+        let t = make index db in
+        List.iter (put_datom_txn txn t) datoms)
+      index_datoms)
+
+let of_eavt_datoms ~avet eavt_datoms db =
+  if eavt_datoms = [] then ()
+  else (
+    let eavt = make Eavt db in
+    let aevt = make Aevt db in
+    let avet_index = make Avet db in
+    Datascript_lmdb_db.with_write_txn db (fun txn ->
+      List.iter
+        (fun datom ->
+          put_datom_txn txn eavt datom;
+          put_datom_txn txn aevt datom;
+          if avet datom.a then put_datom_txn txn avet_index datom)
+        eavt_datoms))
+
+let of_bulk index datoms db = { db; which = index; additions = datoms; removals = [] }
+
+let additions_only t = t.additions <> [] && t.removals = []
 
 let add datom t =
   let key = datom_key t datom in
@@ -54,72 +82,15 @@ let remove datom t =
   in
   { t with additions; removals }
 
-let removal_keys t =
-  let table = Hashtbl.create (List.length t.removals) in
-  List.iter (fun datom -> Hashtbl.add table (datom_key t datom) ()) t.removals;
-  table
-
-let addition_keys t =
-  let table = Hashtbl.create (List.length t.additions) in
-  List.iter (fun datom -> Hashtbl.replace table (datom_key t datom) datom) t.additions;
-  table
-
-let fold_stored t f acc =
-  let removed = removal_keys t in
-  let added = addition_keys t in
-  let acc = ref acc in
-  Datascript_lmdb_db.fold_index t.which t.db (fun key value ->
-    if not (Hashtbl.mem removed key || Hashtbl.mem added key) then
-      acc := f !acc (decode_entry t.which key value));
-  !acc
-
-let fold_stored_range t ?from_key ?to_key f acc =
-  let removed = removal_keys t in
-  let added = addition_keys t in
-  let acc = ref acc in
-  Datascript_lmdb_db.fold_index_range t.which t.db ?from_key ?to_key (fun key value ->
-    if not (Hashtbl.mem removed key || Hashtbl.mem added key) then
-      acc := f !acc (decode_entry t.which key value));
-  !acc
-
-let fold_overlay t f acc = List.fold_left f acc t.additions
-
-let fold_datoms f init t =
-  let acc = fold_stored t f init in
-  fold_overlay t f acc
-
-let collect_datoms t =
-  fold_datoms (fun acc datom -> datom :: acc) [] t |> List.sort (cmp_for t.which)
-
-let clear_index_txn txn index lmdb =
-  Datascript_lmdb_db.fold_index index lmdb (fun key _ ->
-    Datascript_lmdb_db.remove_index_txn index txn lmdb key)
-
-let sync_merged_to_lmdb t target_lmdb =
-  let merged = collect_datoms t in
-  Datascript_lmdb_db.with_write_txn target_lmdb (fun txn ->
-    clear_index_txn txn t.which target_lmdb;
-    List.iter
-      (fun datom ->
-        let key = datom_key t datom in
-        let value = Datascript_lmdb_codec.encode_datom_value datom in
-        Datascript_lmdb_db.put_index_txn t.which txn target_lmdb key value)
-      merged)
-
-let copy_list xs = List.map (fun x -> x) xs
-
-let copy t = { t with additions = copy_list t.additions; removals = copy_list t.removals }
-
-let flush t =
-  if overlay_empty t then t
-  else (
-    Datascript_lmdb_db.with_write_txn t.db (fun txn ->
-      List.iter (remove_datom_txn txn t) t.removals;
-      List.iter (put_datom_txn txn t) t.additions);
-    { t with additions = []; removals = [] })
+let overlay_tables t =
+  let removed = Hashtbl.create (List.length t.removals) in
+  List.iter (fun datom -> Hashtbl.add removed (datom_key t datom) ()) t.removals;
+  let added = Hashtbl.create (List.length t.additions) in
+  List.iter (fun datom -> Hashtbl.replace added (datom_key t datom) datom) t.additions;
+  removed, added
 
-let to_list t = collect_datoms t
-let fold f init t = fold_datoms f init t
+let stored_visible key removed added =
+  not (Hashtbl.mem removed key || Hashtbl.mem added key)
 
 let in_range cmp lower upper datom =
   let above_lower =
@@ -138,17 +109,127 @@ let bound_key t = function
   | None -> None
   | Some datom -> Some (datom_key t datom)
 
-let materialize_range t ?from_ ?to_ cmp =
-  let filter datoms = List.filter (in_range cmp from_ to_) datoms in
+exception Stop_search
+
+let fold_stored t f acc =
   if overlay_empty t then
-    match bound_key t from_ with
-    | None -> filter (to_list t)
-    | Some from_key ->
-      fold_stored_range t ~from_key (fun acc datom -> datom :: acc) []
-      |> List.rev
-      |> filter
+    let acc = ref acc in
+    Datascript_lmdb_db.fold_index t.which t.db (fun key value ->
+      acc := f !acc (decode_entry t.which key value));
+    !acc
+  else
+    let removed, added = overlay_tables t in
+    let acc = ref acc in
+    Datascript_lmdb_db.fold_index t.which t.db (fun key value ->
+      if stored_visible key removed added then
+        acc := f !acc (decode_entry t.which key value));
+    !acc
+
+let fold_stored_prefix t attr f acc =
+  let prefix = attr ^ "\000" in
+  if overlay_empty t then
+    let acc = ref acc in
+    Datascript_lmdb_db.fold_index_prefix t.which t.db prefix (fun key value ->
+      acc := f !acc (decode_entry t.which key value));
+    !acc
+  else
+    let removed, added = overlay_tables t in
+    let acc = ref acc in
+    Datascript_lmdb_db.fold_index_prefix t.which t.db prefix (fun key value ->
+      if stored_visible key removed added then
+        acc := f !acc (decode_entry t.which key value));
+    !acc
+
+let fold_stored_attr_value_prefix t attr value f acc =
+  let prefix = Datascript_lmdb_codec.encode_index_attr_value_prefix t.which attr value in
+  if overlay_empty t then
+    let acc = ref acc in
+    Datascript_lmdb_db.fold_index_prefix t.which t.db prefix (fun key value ->
+      acc := f !acc (decode_entry t.which key value));
+    !acc
   else
-    filter (to_list t)
+    let removed, added = overlay_tables t in
+    let acc = ref acc in
+    Datascript_lmdb_db.fold_index_prefix t.which t.db prefix (fun key value ->
+      if stored_visible key removed added then
+        acc := f !acc (decode_entry t.which key value));
+    !acc
+
+let fold_stored_bounded t ?from_ ?to_ cmp f acc =
+  match bound_key t from_ with
+  | None -> fold_stored t f acc
+  | Some from_key ->
+    let removed, added =
+      if overlay_empty t then (Hashtbl.create 0, Hashtbl.create 0) else overlay_tables t
+    in
+    let acc = ref acc in
+    Datascript_lmdb_db.fold_index_range_until t.which t.db ~from_key
+      ~stop:(fun key value ->
+        if not (stored_visible key removed added) then false
+        else
+          match to_ with
+          | Some bound ->
+            let datom = decode_entry t.which key value in
+            cmp datom bound > 0
+          | None -> false)
+      (fun key value ->
+        if stored_visible key removed added then
+          let datom = decode_entry t.which key value in
+          if in_range cmp from_ to_ datom then acc := f !acc datom);
+    !acc
+
+let fold_stored_bounded t ?from_ ?to_ cmp f acc =
+  let cmp = Option.value ~default:(cmp_for t.which) cmp in
+  let apply acc datom = if in_range cmp from_ to_ datom then f acc datom else acc in
+  if not (overlay_empty t) then
+    collect_datoms t
+    |> List.filter (fun datom -> in_range cmp from_ to_ datom)
+    |> List.fold_left f init
+  else
+    let acc =
+      match from_, to_ with
+      | Some bound, Some bound' when bound == bound' && bound.a <> "" && bound.e = 0 && bound.v = Nil
+        && (t.which = Aevt || t.which = Avet) ->
+        fold_stored_prefix t bound.a apply init
+      | Some bound, Some bound' when bound == bound' && bound.a <> "" && bound.v <> Nil && bound.e = 0 ->
+        fold_stored_attr_value_prefix t bound.a bound.v apply init
+      | _ -> fold_stored_bounded t ?from_ ?to_ cmp apply init
+    in
+    acc
+
+let find_first_slice ?from_ ?to_ ?cmp t =
+  let cmp = Option.value ~default:(cmp_for t.which) cmp in
+  let found = ref None in
+  let consider datom =
+    if !found = None && in_range cmp from_ to_ datom then (
+      found := Some datom;
+      raise Stop_search)
+  in
+  (try
+     if not (overlay_empty t) then
+       collect_datoms t |> List.iter consider
+     else
+       match from_, to_ with
+       | Some bound, Some bound' when bound == bound' && bound.a <> "" && bound.e = 0 && bound.v = Nil
+         && (t.which = Aevt || t.which = Avet) ->
+         fold_stored_prefix t bound.a (fun () datom -> consider datom) ()
+       | Some bound, Some bound' when bound == bound' && bound.a <> "" && bound.v <> Nil && bound.e = 0 ->
+         fold_stored_attr_value_prefix t bound.a bound.v (fun () datom -> consider datom) ()
+       | _ -> fold_stored_bounded t ?from_ ?to_ cmp (fun () datom -> consider datom) ()
+   with Stop_search -> ());
+  !found
+
+let fold_attr_prefix f init t attr =
+  let apply acc datom = if datom.a = attr then f acc datom else acc in
+  if not (overlay_empty t) then
+    collect_datoms t
+    |> List.filter (fun datom -> datom.a = attr)
+    |> List.fold_left f init
+  else
+    fold_stored_prefix t attr apply init
+
+let materialize_range t ?from_ ?to_ cmp =
+  fold_slice (fun acc datom -> datom :: acc) [] ?from_ ?to_ ~cmp t |> List.rev
 
 let make_seq cmp datoms = { cmp; datoms; offset = 0 }
 
@@ -182,8 +263,15 @@ let rslice_seq ?from_ ?to_ ?cmp t =
   make_seq cmp datoms
 
 let seq_to_list seq = to_seq seq |> List.of_seq
+
+let fold_seq f init { cmp = _; datoms; offset } =
+  let rec loop index acc =
+    if index >= List.length datoms then acc
+    else loop (index + 1) (f acc (List.nth datoms index))
+  in
+  loop offset init
+
 let slice ?from_ ?to_ ?cmp t = slice_seq ?from_ ?to_ ?cmp t |> seq_to_list
-let fold_seq f init seq = List.fold_left f init (seq_to_list seq)
 
 let seek bound seq =
   let rec count index =
diff --git a/lmdb/melange/datascript_lmdb_index.mli b/lmdb/melange/datascript_lmdb_index.mli
index ea9198e..cc85977 100644
--- a/lmdb/melange/datascript_lmdb_index.mli
+++ b/lmdb/melange/datascript_lmdb_index.mli
@@ -6,13 +6,22 @@ type 'a seq
 val db_of : t -> Datascript_lmdb_db.t
 val empty : index -> Datascript_lmdb_db.t -> t
 val of_sorted_list : index -> datom list -> Datascript_lmdb_db.t -> t
+val of_sorted_lists : (index * datom list) list -> Datascript_lmdb_db.t -> unit
+val of_eavt_datoms : avet:(string -> bool) -> datom list -> Datascript_lmdb_db.t -> unit
+val of_bulk : index -> datom list -> Datascript_lmdb_db.t -> t
 val add : datom -> t -> t
 val remove : datom -> t -> t
 val flush : t -> t
 val copy : t -> t
 val sync_merged_to_lmdb : t -> Datascript_lmdb_db.t -> unit
+val lookup : t -> datom -> datom option
 val to_list : t -> datom list
 val fold : ('acc -> datom -> 'acc) -> 'acc -> t -> 'acc
+val fold_slice :
+  ('acc -> datom -> 'acc) -> 'acc -> ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> 'acc
+val find_first_slice :
+  ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom option
+val fold_attr_prefix : ('acc -> datom -> 'acc) -> 'acc -> t -> string -> 'acc
 val slice : ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom list
 val slice_seq : ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom seq
 val rslice_seq : ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom seq
diff --git a/lmdb/native/datascript_lmdb_db.ml b/lmdb/native/datascript_lmdb_db.ml
index a10edcc..1937eaf 100644
--- a/lmdb/native/datascript_lmdb_db.ml
+++ b/lmdb/native/datascript_lmdb_db.ml
@@ -106,6 +106,10 @@ let put_index index db key value =
 let remove_index index db key =
   with_write_txn db (fun txn -> remove_index_txn index txn db key)
 
+let get_index index db key =
+  ensure_open db;
+  try Some (Map.get (map_for_index index db) key) with Not_found -> None
+
 let fold_index index db f =
   ensure_open db;
   let map = map_for_index index db in
@@ -119,6 +123,28 @@ let fold_index index db f =
   in
   loop ()
 
+let fold_index_prefix index db prefix f =
+  ensure_open db;
+  let map = map_for_index index db in
+  let prefix_len = String.length prefix in
+  (try
+     Cursor.go Ro map (fun cursor ->
+       (try ignore (Cursor.seek_range cursor prefix) with Not_found -> raise Exit);
+       let rec loop () =
+         let key, value =
+           try Cursor.current cursor
+           with Not_found -> raise Exit
+         in
+         if String.length key < prefix_len || String.sub key 0 prefix_len <> prefix then raise Exit;
+         f key value;
+         try
+           ignore (Cursor.next cursor);
+           loop ()
+         with Not_found -> raise Exit
+       in
+       loop ())
+   with Exit -> ())
+
 let fold_index_range index db ?from_key ?to_key f =
   ensure_open db;
   let map = map_for_index index db in
@@ -146,6 +172,33 @@ let fold_index_range index db ?from_key ?to_key f =
        loop ())
    with Exit -> ())
 
+let fold_index_range_until index db ?from_key ?stop f =
+  ensure_open db;
+  let map = map_for_index index db in
+  (try
+     Cursor.go Ro map (fun cursor ->
+       (match from_key with
+        | None -> (
+          try ignore (Cursor.first cursor) with Not_found -> raise Exit)
+        | Some key -> (
+          try ignore (Cursor.seek_range cursor key) with Not_found -> raise Exit));
+       let rec loop () =
+         let key, value =
+           try Cursor.current cursor
+           with Not_found -> raise Exit
+         in
+         (match stop with
+          | Some stop when stop key value -> raise Exit
+          | _ -> ());
+         f key value;
+         try
+           ignore (Cursor.next cursor);
+           loop ()
+         with Not_found -> raise Exit
+       in
+       loop ())
+   with Exit -> ())
+
 let copy_index_txn index txn from_db to_db =
   fold_index index from_db (fun key value ->
     put_index_txn index txn to_db key value)
diff --git a/lmdb/native/datascript_lmdb_db.mli b/lmdb/native/datascript_lmdb_db.mli
index bef9386..262c841 100644
--- a/lmdb/native/datascript_lmdb_db.mli
+++ b/lmdb/native/datascript_lmdb_db.mli
@@ -16,8 +16,17 @@ val put_index_txn : index -> [ `Read | `Write ] Lmdb.Txn.t -> t -> string -> str
 val remove_index_txn : index -> [ `Read | `Write ] Lmdb.Txn.t -> t -> string -> unit
 val copy_index_txn : index -> [ `Read | `Write ] Lmdb.Txn.t -> t -> t -> unit
 
+val get_index : index -> t -> string -> string option
 val fold_index : index -> t -> (string -> string -> unit) -> unit
 val fold_index_range :
   index -> t -> ?from_key:string -> ?to_key:string -> (string -> string -> unit) -> unit
+val fold_index_range_until :
+  index ->
+  t ->
+  ?from_key:string ->
+  ?stop:(string -> string -> bool) ->
+  (string -> string -> unit) ->
+  unit
+val fold_index_prefix : index -> t -> string -> (string -> string -> unit) -> unit
 val put_index : index -> t -> string -> string -> unit
 val remove_index : index -> t -> string -> unit
diff --git a/lmdb/native/datascript_lmdb_index.ml b/lmdb/native/datascript_lmdb_index.ml
index 24ca0bc..2b95038 100644
--- a/lmdb/native/datascript_lmdb_index.ml
+++ b/lmdb/native/datascript_lmdb_index.ml
@@ -4,13 +4,17 @@ type t =
   { db : Datascript_lmdb_db.t
   ; which : index
   ; additions : datom list
+  ; additions_arr : datom array option
   ; removals : datom list
+  ; bulk : bool
   }
 
 type 'a seq = { cmp : datom -> datom -> int; datoms : datom list; offset : int }
 
+exception Stop_search
+
 let db_of t = t.db
-let make index db = { db; which = index; additions = []; removals = [] }
+let make index db = { db; which = index; additions = []; additions_arr = None; removals = []; bulk = false }
 let cmp_for index = Datascript_types.Compare.compare_datom index
 let overlay_empty t = t.additions = [] && t.removals = []
 
@@ -34,15 +38,63 @@ let empty index db = make index db
 
 let of_sorted_list index datoms db =
   let t = empty index db in
+  if datoms = [] then t
+  else (
+    Datascript_lmdb_db.with_write_txn db (fun txn ->
+      List.iter (put_datom_txn txn t) datoms);
+    t)
+
+let of_sorted_lists index_datoms db =
   Datascript_lmdb_db.with_write_txn db (fun txn ->
-    List.iter (put_datom_txn txn t) datoms);
-  t
+    List.iter
+      (fun (index, datoms) ->
+        let t = make index db in
+        List.iter (put_datom_txn txn t) datoms)
+      index_datoms)
+
+let of_eavt_datoms ~avet eavt_datoms db =
+  if eavt_datoms = [] then ()
+  else (
+    let eavt = make Eavt db in
+    let aevt = make Aevt db in
+    let avet_index = make Avet db in
+    Datascript_lmdb_db.with_write_txn db (fun txn ->
+      List.iter
+        (fun datom ->
+          put_datom_txn txn eavt datom;
+          put_datom_txn txn aevt datom;
+          if avet datom.a then put_datom_txn txn avet_index datom)
+        eavt_datoms))
+
+let of_bulk index datoms db =
+  { db; which = index; additions = datoms; additions_arr = Some (Array.of_list datoms); removals = []; bulk = true }
+
+let additions_array t =
+  match t.additions_arr with
+  | Some arr -> arr
+  | None -> Array.of_list t.additions
+
+let array_find_first cmp bound arr =
+  let len = Array.length arr in
+  let rec lower lo hi =
+    if lo >= hi then lo
+    else
+      let mid = (lo + hi) / 2 in
+      if cmp arr.(mid) bound < 0 then lower (mid + 1) hi else lower lo mid
+  in
+  let index = lower 0 len in
+  if index < len && cmp arr.(index) bound = 0 then Some arr.(index) else None
+
+let additions_only t = t.bulk && t.additions <> [] && t.removals = []
 
 let add datom t =
-  let key = datom_key t datom in
-  let additions = datom :: List.filter (fun d -> datom_key t d <> key) t.additions in
-  let removals = List.filter (fun d -> datom_key t d <> key) t.removals in
-  { t with additions; removals }
+  if additions_only t then
+    { t with additions = datom :: t.additions; additions_arr = None }
+  else (
+    let key = datom_key t datom in
+    let additions = datom :: List.filter (fun d -> datom_key t d <> key) t.additions in
+    let removals = List.filter (fun d -> datom_key t d <> key) t.removals in
+    { t with additions; removals })
 
 let remove datom t =
   let key = datom_key t datom in
@@ -54,57 +106,136 @@ let remove datom t =
   in
   { t with additions; removals }
 
-let removal_keys t =
-  let table = Hashtbl.create (List.length t.removals) in
-  List.iter (fun datom -> Hashtbl.add table (datom_key t datom) ()) t.removals;
-  table
+let overlay_tables t =
+  let removed = Hashtbl.create (List.length t.removals) in
+  List.iter (fun datom -> Hashtbl.add removed (datom_key t datom) ()) t.removals;
+  let added = Hashtbl.create (List.length t.additions) in
+  List.iter (fun datom -> Hashtbl.replace added (datom_key t datom) datom) t.additions;
+  removed, added
 
-let addition_keys t =
-  let table = Hashtbl.create (List.length t.additions) in
-  List.iter (fun datom -> Hashtbl.replace table (datom_key t datom) datom) t.additions;
-  table
+let stored_visible key removed added =
+  not (Hashtbl.mem removed key || Hashtbl.mem added key)
+
+let in_range cmp lower upper datom =
+  let above_lower =
+    match lower with
+    | None -> true
+    | Some lower -> cmp datom lower >= 0
+  in
+  let below_upper =
+    match upper with
+    | None -> true
+    | Some upper -> cmp datom upper <= 0
+  in
+  above_lower && below_upper
+
+let bound_key t = function
+  | None -> None
+  | Some datom -> Some (datom_key t datom)
 
 let fold_stored t f acc =
-  let removed = removal_keys t in
-  let added = addition_keys t in
-  let acc = ref acc in
-  Datascript_lmdb_db.fold_index t.which t.db (fun key value ->
-    if not (Hashtbl.mem removed key || Hashtbl.mem added key) then
+  if overlay_empty t then
+    let acc = ref acc in
+    Datascript_lmdb_db.fold_index t.which t.db (fun key value ->
       acc := f !acc (decode_entry t.which key value));
-  !acc
-
-let fold_stored_range t ?from_key ?to_key f acc =
-  let removed = removal_keys t in
-  let added = addition_keys t in
-  let acc = ref acc in
-  Datascript_lmdb_db.fold_index_range t.which t.db ?from_key ?to_key (fun key value ->
-    if not (Hashtbl.mem removed key || Hashtbl.mem added key) then
+    !acc
+  else
+    let removed, added = overlay_tables t in
+    let acc = ref acc in
+    Datascript_lmdb_db.fold_index t.which t.db (fun key value ->
+      if stored_visible key removed added then
+        acc := f !acc (decode_entry t.which key value));
+    !acc
+
+let fold_stored_prefix t attr f acc =
+  let prefix = attr ^ "\000" in
+  if overlay_empty t then
+    let acc = ref acc in
+    Datascript_lmdb_db.fold_index_prefix t.which t.db prefix (fun key value ->
       acc := f !acc (decode_entry t.which key value));
-  !acc
+    !acc
+  else
+    let removed, added = overlay_tables t in
+    let acc = ref acc in
+    Datascript_lmdb_db.fold_index_prefix t.which t.db prefix (fun key value ->
+      if stored_visible key removed added then
+        acc := f !acc (decode_entry t.which key value));
+    !acc
+
+let fold_stored_attr_value_prefix t attr value f acc =
+  let prefix = Datascript_lmdb_codec.encode_index_attr_value_prefix t.which attr value in
+  if overlay_empty t then
+    let acc = ref acc in
+    Datascript_lmdb_db.fold_index_prefix t.which t.db prefix (fun key value ->
+      acc := f !acc (decode_entry t.which key value));
+    !acc
+  else
+    let removed, added = overlay_tables t in
+    let acc = ref acc in
+    Datascript_lmdb_db.fold_index_prefix t.which t.db prefix (fun key value ->
+      if stored_visible key removed added then
+        acc := f !acc (decode_entry t.which key value));
+    !acc
+
+let fold_stored_bounded t ?from_ ?to_ cmp f acc =
+  match bound_key t from_ with
+  | None -> fold_stored t f acc
+  | Some from_key ->
+    let removed, added =
+      if overlay_empty t then (Hashtbl.create 0, Hashtbl.create 0) else overlay_tables t
+    in
+    let acc = ref acc in
+    Datascript_lmdb_db.fold_index_range_until t.which t.db ~from_key
+      ~stop:(fun key value ->
+        if not (stored_visible key removed added) then false
+        else
+          match to_ with
+          | Some bound ->
+            let datom = decode_entry t.which key value in
+            cmp datom bound > 0
+          | None -> false)
+      (fun key value ->
+        if stored_visible key removed added then
+          let datom = decode_entry t.which key value in
+          if in_range cmp from_ to_ datom then acc := f !acc datom);
+    !acc
 
 let fold_overlay t f acc = List.fold_left f acc t.additions
 
 let fold_datoms f init t =
-  let acc = fold_stored t f init in
-  fold_overlay t f acc
+  if additions_only t then List.fold_left f init t.additions
+  else (
+    let acc = fold_stored t f init in
+    fold_overlay t f acc)
 
 let collect_datoms t =
-  fold_datoms (fun acc datom -> datom :: acc) [] t |> List.sort (cmp_for t.which)
+  if overlay_empty t then fold_stored t (fun acc datom -> datom :: acc) []
+  else fold_datoms (fun acc datom -> datom :: acc) [] t |> List.sort (cmp_for t.which)
 
 let clear_index_txn txn index lmdb =
   Datascript_lmdb_db.fold_index index lmdb (fun key _ ->
     Datascript_lmdb_db.remove_index_txn index txn lmdb key)
 
 let sync_merged_to_lmdb t target_lmdb =
-  let merged = collect_datoms t in
-  Datascript_lmdb_db.with_write_txn target_lmdb (fun txn ->
-    clear_index_txn txn t.which target_lmdb;
+  let write_datoms txn datoms =
     List.iter
       (fun datom ->
         let key = datom_key t datom in
         let value = Datascript_lmdb_codec.encode_datom_value datom in
         Datascript_lmdb_db.put_index_txn t.which txn target_lmdb key value)
-      merged)
+      datoms
+  in
+  if additions_only t then
+    Datascript_lmdb_db.with_write_txn target_lmdb (fun txn -> write_datoms txn t.additions)
+  else if overlay_empty t then
+    Datascript_lmdb_db.with_write_txn target_lmdb (fun txn ->
+      clear_index_txn txn t.which target_lmdb;
+      Datascript_lmdb_db.copy_index_txn t.which txn t.db target_lmdb)
+  else
+    let merged = collect_datoms t in
+    Datascript_lmdb_db.with_write_txn target_lmdb (fun txn ->
+      clear_index_txn txn t.which target_lmdb;
+      write_datoms txn merged)
 
 let copy_list xs = List.map (fun x -> x) xs
 
@@ -116,39 +247,86 @@ let flush t =
     Datascript_lmdb_db.with_write_txn t.db (fun txn ->
       List.iter (remove_datom_txn txn t) t.removals;
       List.iter (put_datom_txn txn t) t.additions);
-    { t with additions = []; removals = [] })
+    { t with additions = []; additions_arr = None; removals = [] })
+
+let to_list t =
+  if additions_only t then t.additions
+  else if overlay_empty t then List.rev (fold_stored t (fun acc datom -> datom :: acc) [])
+  else collect_datoms t
 
-let to_list t = collect_datoms t
 let fold f init t = fold_datoms f init t
 
-let in_range cmp lower upper datom =
-  let above_lower =
-    match lower with
-    | None -> true
-    | Some lower -> cmp datom lower >= 0
-  in
-  let below_upper =
-    match upper with
-    | None -> true
-    | Some upper -> cmp datom upper <= 0
+let lookup t datom =
+  let key = datom_key t datom in
+  if List.exists (fun d -> datom_key t d = key) t.removals then None
+  else
+    (match List.find_opt (fun d -> datom_key t d = key) t.additions with
+     | Some datom -> Some datom
+     | None -> (
+       match Datascript_lmdb_db.get_index t.which t.db key with
+       | None -> None
+       | Some value -> Some (decode_entry t.which key value)))
+
+let fold_slice f init ?from_ ?to_ ?cmp t =
+  let cmp = Option.value ~default:(cmp_for t.which) cmp in
+  let apply acc datom = if in_range cmp from_ to_ datom then f acc datom else acc in
+  if additions_only t then List.fold_left apply init t.additions
+  else if not (overlay_empty t) then
+    collect_datoms t
+    |> List.filter (fun datom -> in_range cmp from_ to_ datom)
+    |> List.fold_left f init
+  else
+    match from_, to_ with
+    | Some bound, Some bound' when bound == bound' && bound.a <> "" && bound.e = 0 && bound.v = Nil
+      && (t.which = Aevt || t.which = Avet) ->
+      fold_stored_prefix t bound.a apply init
+    | Some bound, Some bound' when bound == bound' && bound.a <> "" && bound.v <> Nil && bound.e = 0 ->
+      fold_stored_attr_value_prefix t bound.a bound.v apply init
+    | _ -> fold_stored_bounded t ?from_ ?to_ cmp apply init
+
+let find_first_slice ?from_ ?to_ ?cmp t =
+  let cmp = Option.value ~default:(cmp_for t.which) cmp in
+  let found = ref None in
+  let consider datom =
+    if !found = None && in_range cmp from_ to_ datom then (
+      found := Some datom;
+      raise Stop_search)
   in
-  above_lower && below_upper
-
-let bound_key t = function
-  | None -> None
-  | Some datom -> Some (datom_key t datom)
+  (try
+     if additions_only t then (
+       match from_, to_ with
+       | Some bound, Some bound' when bound == bound' -> (
+         match array_find_first cmp bound (additions_array t) with
+         | Some datom ->
+           found := Some datom;
+           raise Stop_search
+         | None -> ())
+       | _ -> List.iter consider t.additions)
+     else if not (overlay_empty t) then
+       collect_datoms t |> List.iter consider
+     else
+       match from_, to_ with
+       | Some bound, Some bound' when bound == bound' && bound.a <> "" && bound.e = 0 && bound.v = Nil
+         && (t.which = Aevt || t.which = Avet) ->
+         fold_stored_prefix t bound.a (fun () datom -> consider datom) ()
+       | Some bound, Some bound' when bound == bound' && bound.a <> "" && bound.v <> Nil && bound.e = 0 ->
+         fold_stored_attr_value_prefix t bound.a bound.v (fun () datom -> consider datom) ()
+       | _ -> fold_stored_bounded t ?from_ ?to_ cmp (fun () datom -> consider datom) ()
+   with Stop_search -> ());
+  !found
+
+let fold_attr_prefix f init t attr =
+  let apply acc datom = if datom.a = attr then f acc datom else acc in
+  if additions_only t then List.fold_left apply init t.additions
+  else if not (overlay_empty t) then
+    collect_datoms t
+    |> List.filter (fun datom -> datom.a = attr)
+    |> List.fold_left f init
+  else
+    fold_stored_prefix t attr apply init
 
 let materialize_range t ?from_ ?to_ cmp =
-  let filter datoms = List.filter (in_range cmp from_ to_) datoms in
-  if overlay_empty t then
-    match bound_key t from_ with
-    | None -> filter (to_list t)
-    | Some from_key ->
-      fold_stored_range t ~from_key (fun acc datom -> datom :: acc) []
-      |> List.rev
-      |> filter
-  else
-    filter (to_list t)
+  fold_slice (fun acc datom -> datom :: acc) [] ?from_ ?to_ ~cmp t |> List.rev
 
 let make_seq cmp datoms = { cmp; datoms; offset = 0 }
 
@@ -182,8 +360,15 @@ let rslice_seq ?from_ ?to_ ?cmp t =
   make_seq cmp datoms
 
 let seq_to_list seq = to_seq seq |> List.of_seq
+
+let fold_seq f init { cmp = _; datoms; offset } =
+  let rec loop index acc =
+    if index >= List.length datoms then acc
+    else loop (index + 1) (f acc (List.nth datoms index))
+  in
+  loop offset init
+
 let slice ?from_ ?to_ ?cmp t = slice_seq ?from_ ?to_ ?cmp t |> seq_to_list
-let fold_seq f init seq = List.fold_left f init (seq_to_list seq)
 
 let seek bound seq =
   let rec count index =
diff --git a/lmdb/native/datascript_lmdb_index.mli b/lmdb/native/datascript_lmdb_index.mli
index ea9198e..cc85977 100644
--- a/lmdb/native/datascript_lmdb_index.mli
+++ b/lmdb/native/datascript_lmdb_index.mli
@@ -6,13 +6,22 @@ type 'a seq
 val db_of : t -> Datascript_lmdb_db.t
 val empty : index -> Datascript_lmdb_db.t -> t
 val of_sorted_list : index -> datom list -> Datascript_lmdb_db.t -> t
+val of_sorted_lists : (index * datom list) list -> Datascript_lmdb_db.t -> unit
+val of_eavt_datoms : avet:(string -> bool) -> datom list -> Datascript_lmdb_db.t -> unit
+val of_bulk : index -> datom list -> Datascript_lmdb_db.t -> t
 val add : datom -> t -> t
 val remove : datom -> t -> t
 val flush : t -> t
 val copy : t -> t
 val sync_merged_to_lmdb : t -> Datascript_lmdb_db.t -> unit
+val lookup : t -> datom -> datom option
 val to_list : t -> datom list
 val fold : ('acc -> datom -> 'acc) -> 'acc -> t -> 'acc
+val fold_slice :
+  ('acc -> datom -> 'acc) -> 'acc -> ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> 'acc
+val find_first_slice :
+  ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom option
+val fold_attr_prefix : ('acc -> datom -> 'acc) -> 'acc -> t -> string -> 'acc
 val slice : ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom list
 val slice_seq : ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom seq
 val rslice_seq : ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom seq
diff --git a/type/datascript_types.ml b/type/datascript_types.ml
index b8b6a79..b0ccf6e 100644
--- a/type/datascript_types.ml
+++ b/type/datascript_types.ml
@@ -119,8 +119,9 @@ and db =
   ; eavt_index : index_set
   ; aevt_index : index_set
   ; avet_index : index_set
-  ; aevt_by_attr : (attr, datom list) Hashtbl.t
-  ; avet_by_attr : (attr, datom list) Hashtbl.t
+  ; aevt_by_attr : (attr, datom array) Hashtbl.t
+  ; avet_by_attr : (attr, datom array) Hashtbl.t
+  ; avet_entities_by_attr_value : (attr * value, entity_id list) Hashtbl.t
   ; duplicate_datoms : datom list
   ; duplicate_aevt_datoms : datom list
   ; duplicate_avet_datoms : datom list

From ea32d01bf11fa4e1bf377b5290e165c908063c4f Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Thu, 27 Aug 2026 06:13:51 +0000
Subject: [PATCH 09/10] Optimize bulk LMDB slices and AVET entity-id query path

- Use sorted bulk arrays for O(log n) range slices instead of scanning 80k overlays
- Keep O(1) bulk Index.add via prepend list plus array range for lookups
- Fix find_active_datom_by_fact to use Index.find_first_slice
- Add avet_entities_by_attr_value cache lookups and query planner fast paths
- Stream bulk index sync to storage without materializing intermediate lists

Co-authored-by: Tienson Qin <tiensonqin@gmail.com>
---
 impl/datascript.ml                   |  16 +++
 impl/db.ml                           |  22 +++-
 impl/db.mli                          |   1 +
 impl/db_access.ml                    |   3 +
 impl/query_where.ml                  |  70 ++++++++++-
 lmdb/native/datascript_lmdb_index.ml | 176 +++++++++++++++++++--------
 6 files changed, 226 insertions(+), 62 deletions(-)

diff --git a/impl/datascript.ml b/impl/datascript.ml
index 895882c..cb4ac1a 100644
--- a/impl/datascript.ml
+++ b/impl/datascript.ml
@@ -1090,6 +1090,21 @@ let datoms_by_attr_value db attr value =
       datoms_list db Aevt ~a:attr ()
       |> List.filter datom_value_matches
 
+let entity_ids_by_attr_value db attr value =
+  match resolve_query_value_for_attr db attr value with
+  | None -> Some []
+  | Some value ->
+    let value =
+      if is_tuple_attr db attr then
+        coerce_tuple_lookup_value_db db attr value
+      else
+        normalize_value value
+    in
+    if query_value_uses_avet value && query_attr_uses_avet db attr then
+      Db_access_impl.avet_entity_ids_by_attr_value db attr value
+    else
+      None
+
 let pattern_value_needs_attr_resolution db attr value =
   is_tuple_attr db attr
   ||
@@ -1361,6 +1376,7 @@ module Query_where_impl = Query_where.Make (struct
   let cardinality_one db attr = cardinality db attr = One
   let normalize_value = normalize_value
   let datoms_by_attr_value = datoms_by_attr_value
+  let entity_ids_by_attr_value = entity_ids_by_attr_value
   let query_attr_uses_avet = query_attr_uses_avet
   let query_value_uses_avet = query_value_uses_avet
 end)
diff --git a/impl/db.ml b/impl/db.ml
index 554ccad..e75c02e 100644
--- a/impl/db.ml
+++ b/impl/db.ml
@@ -244,9 +244,12 @@ let find_active_datom_by_fact db datom =
     Option.value (Hashtbl.find_opt db.duplicate_eavt_by_entity datom.e) ~default:[]
     |> List.filter (fun active -> active.a = datom.a && value_equal active.v datom.v)
   in
-  match Index.slice ~from_:bound ~to_:bound ~cmp db.eavt_index @ duplicate_matches with
-  | [] -> None
-  | matches -> Some (matches |> List.sort (Util.compare_datom Eavt) |> List.hd)
+  match Index.find_first_slice ~from_:bound ~to_:bound ~cmp db.eavt_index with
+  | Some active when active.e = datom.e && active.a = datom.a && value_equal active.v datom.v -> Some active
+  | _ -> (
+    match duplicate_matches with
+    | [] -> None
+    | matches -> Some (matches |> List.sort (Util.compare_datom Eavt) |> List.hd))
 
 let add_datom_to_indexes db datom =
   { db with
@@ -720,6 +723,19 @@ let exact_prefix_bound index e a v tx =
        Some (bound_datom ~e ~a ~v ~tx (), fields ~e:true ~a:true ~v:true ~tx:true ())
      | _ -> None)
 
+let avet_entity_ids_by_attr_value context db attr value =
+  match Hashtbl.find_opt db.avet_entities_by_attr_value (attr, value) with
+  | Some entity_ids -> Some entity_ids
+  | None -> (
+    match Hashtbl.find_opt db.avet_by_attr attr with
+    | Some datoms ->
+      let bound = bound_datom ~a:attr ~v:value () in
+      let bound_fields = fields ~a:true ~v:true () in
+      Some
+        (array_attr_value_slice context Avet bound bound_fields datoms
+         |> List.map (fun datom -> datom.e))
+    | None -> None)
+
 let avet_datoms_by_value context db attr value =
   let bound = bound_datom ~a:attr ~v:value () in
   let bound_fields = fields ~a:true ~v:true () in
diff --git a/impl/db.mli b/impl/db.mli
index 666745f..7cace2e 100644
--- a/impl/db.mli
+++ b/impl/db.mli
@@ -56,6 +56,7 @@ val fold_datoms :
   unit ->
   'acc
 val datoms_list : index_context -> db -> index -> ?e:entity_id -> ?a:attr -> ?v:value -> ?tx:tx -> unit -> datom list
+val avet_entity_ids_by_attr_value : index_context -> db -> attr -> value -> entity_id list option
 val avet_datoms_by_value : index_context -> db -> attr -> value -> datom list
 val avet_datoms_by_value_seq : index_context -> db -> attr -> value -> datom Seq.t
 val datoms_ref : index_context -> db -> index -> ?e:entity_ref -> ?a:attr -> ?v:value -> ?tx:tx -> unit -> datom Seq.t
diff --git a/impl/db_access.ml b/impl/db_access.ml
index b5150cb..18defd9 100644
--- a/impl/db_access.ml
+++ b/impl/db_access.ml
@@ -110,6 +110,9 @@ end) = struct
 
   let avet_datoms_by_value_seq db attr value =
     Db.avet_datoms_by_value_seq db_index_context db attr value
+
+  let avet_entity_ids_by_attr_value db attr value =
+    Db.avet_entity_ids_by_attr_value db_index_context db attr value
   
   let datoms_ref db index ?e ?a ?v ?tx () =
     Db.datoms_ref db_index_context db index ?e ?a ?v ?tx ()
diff --git a/impl/query_where.ml b/impl/query_where.ml
index b67b4ac..addc3d0 100644
--- a/impl/query_where.ml
+++ b/impl/query_where.ml
@@ -26,6 +26,7 @@ module Make (Context : sig
   val cardinality_one : db -> attr -> bool
   val normalize_value : value -> value
   val datoms_by_attr_value : db -> attr -> value -> datom list
+  val entity_ids_by_attr_value : db -> attr -> value -> entity_id list option
   val query_attr_uses_avet : db -> attr -> bool
   val query_value_uses_avet : value -> bool
 end) = struct
@@ -1053,6 +1054,12 @@ end) = struct
                     (source_context.match_data_pattern source_db [] (QVar e_var) (QAttr attr) (QValue value) datom))
                 |> List.of_seq
           in
+          let avet_entity_ids attr value =
+            if direct_attr attr && query_value_uses_avet value && query_attr_uses_avet source_db attr then
+              entity_ids_by_attr_value source_db attr value
+            else
+              None
+          in
           let constant_datoms =
             constant_patterns
             |> List.map (fun (attr, value) -> attr, value, lazy (datoms_matching attr value))
@@ -1063,10 +1070,43 @@ end) = struct
             |> unique_vars
           in
           let lookup_vars = relation_lookup_vars source_db [ QVar e_var; QWildcard; QWildcard ] in
-          if List.exists (fun (_, _, datoms) -> Lazy.force datoms = []) constant_datoms then
+          if
+            List.exists
+              (fun (attr, value, datoms) ->
+                match avet_entity_ids attr value with
+                | Some [] -> true
+                | Some _ -> false
+                | None -> Lazy.force datoms = [])
+              constant_datoms
+          then
             Some { attrs; rows = []; lookup_vars; unique_rows = true }
           else
+          let avet_single_entity_rows =
+            match constant_patterns, value_var_patterns, required_patterns, excluded_patterns, relation_comparisons with
+            | [ (attr, value) ], [], [], [], [] -> (
+              match avet_entity_ids attr value with
+              | Some entity_ids -> Some (List.map (fun entity_id -> [ Result_entity entity_id ]) entity_ids)
+              | None -> None)
+            | _ -> None
+          in
+          if Option.is_some avet_single_entity_rows then
+            Some
+              { attrs
+              ; rows = Option.get avet_single_entity_rows
+              ; lookup_vars
+              ; unique_rows = true
+              }
+          else
           let constant_sets =
+            let set_from_entity_ids entity_ids =
+              let entities = Bytes.make (source_db.max_datom_e + 1) '\000' in
+              List.iter
+                (fun entity_id ->
+                  if entity_id >= 0 && entity_id < Bytes.length entities then
+                    Bytes.set entities entity_id '\001')
+                entity_ids;
+              entities
+            in
             let set_from_datoms datoms =
               let entities = Bytes.make (source_db.max_datom_e + 1) '\000' in
               List.iter
@@ -1077,7 +1117,15 @@ end) = struct
               entities
             in
             constant_datoms
-            |> List.map (fun (_, _, datoms) -> set_from_datoms (Lazy.force datoms))
+            |> List.map (fun (attr, value, datoms) ->
+              match avet_entity_ids attr value with
+              | Some entity_ids -> set_from_entity_ids entity_ids
+              | None -> set_from_datoms (Lazy.force datoms))
+          in
+          let constant_count (attr, value, datoms) =
+            match avet_entity_ids attr value with
+            | Some entity_ids -> List.length entity_ids
+            | None -> List.length (Lazy.force datoms)
           in
           let candidate_entities () =
             match constant_datoms with
@@ -1090,10 +1138,12 @@ end) = struct
                | [], [] -> [])
             | datoms_by_constant ->
               datoms_by_constant
-              |> List.sort (fun (_, _, left) (_, _, right) ->
-                compare (List.length (Lazy.force left)) (List.length (Lazy.force right)))
+              |> List.sort (fun left right -> compare (constant_count left) (constant_count right))
               |> function
-                | (_, _, datoms) :: _ -> List.map (fun datom -> datom.e) (Lazy.force datoms)
+                | (attr, value, datoms) :: _ -> (
+                  match avet_entity_ids attr value with
+                  | Some entity_ids -> entity_ids
+                  | None -> List.map (fun datom -> datom.e) (Lazy.force datoms))
                 | [] -> []
           in
           let has_pattern entity_id attr value_term =
@@ -1441,7 +1491,7 @@ end) = struct
                 binding_row attrs binding)
               |> List.of_seq
           in
-          let rows =
+          let compute_default_rows () =
             match value_var_patterns with
             | (scan_value_var, scan_attr) :: remaining_value_vars
               when direct_attr scan_attr
@@ -1472,6 +1522,14 @@ end) = struct
                   in
                   bindings |> List.filter_map (binding_row attrs))
           in
+          let rows =
+            match constant_patterns, value_var_patterns, required_patterns, excluded_patterns, relation_comparisons with
+            | [ (attr, value) ], [], [], [], [] when value_var_patterns = [] -> (
+              match avet_entity_ids attr value with
+              | Some entity_ids -> List.map (fun entity_id -> [ Result_entity entity_id ]) entity_ids
+              | None -> compute_default_rows ())
+            | _ -> compute_default_rows ()
+          in
           let unique_rows =
             source_db.duplicate_datoms = []
             && List.mem e_var attrs
diff --git a/lmdb/native/datascript_lmdb_index.ml b/lmdb/native/datascript_lmdb_index.ml
index 2b95038..ddc317d 100644
--- a/lmdb/native/datascript_lmdb_index.ml
+++ b/lmdb/native/datascript_lmdb_index.ml
@@ -69,11 +69,6 @@ let of_eavt_datoms ~avet eavt_datoms db =
 let of_bulk index datoms db =
   { db; which = index; additions = datoms; additions_arr = Some (Array.of_list datoms); removals = []; bulk = true }
 
-let additions_array t =
-  match t.additions_arr with
-  | Some arr -> arr
-  | None -> Array.of_list t.additions
-
 let array_find_first cmp bound arr =
   let len = Array.length arr in
   let rec lower lo hi =
@@ -85,11 +80,68 @@ let array_find_first cmp bound arr =
   let index = lower 0 len in
   if index < len && cmp arr.(index) bound = 0 then Some arr.(index) else None
 
-let additions_only t = t.bulk && t.additions <> [] && t.removals = []
+let array_lower_bound cmp bound arr =
+  let len = Array.length arr in
+  let rec lower lo hi =
+    if lo >= hi then lo
+    else
+      let mid = (lo + hi) / 2 in
+      if cmp arr.(mid) bound < 0 then lower (mid + 1) hi else lower lo mid
+  in
+  lower 0 len
+
+let sorted_bulk_array t =
+  match t.additions_arr with
+  | Some arr -> arr
+  | None ->
+    let arr = Array.of_list t.additions in
+    Array.sort (cmp_for t.which) arr;
+    arr
+
+let bulk_datoms t =
+  let base = Array.to_list (sorted_bulk_array t) in
+  match t.additions with
+  | [] -> base
+  | overlay -> List.merge (cmp_for t.which) (List.sort (cmp_for t.which) overlay) base
+
+let in_range cmp lower upper datom =
+  let above_lower =
+    match lower with
+    | None -> true
+    | Some lower -> cmp datom lower >= 0
+  in
+  let below_upper =
+    match upper with
+    | None -> true
+    | Some upper -> cmp datom upper <= 0
+  in
+  above_lower && below_upper
+
+let array_fold_in_range cmp from_ to_ arr f init =
+  let len = Array.length arr in
+  let start =
+    match from_ with
+    | None -> 0
+    | Some bound -> array_lower_bound cmp bound arr
+  in
+  let rec loop index acc =
+    if index >= len then acc
+    else
+      let datom = arr.(index) in
+      if not (in_range cmp from_ to_ datom) then acc
+      else loop (index + 1) (f acc datom)
+  in
+  loop start init
+
+let array_materialize_range cmp from_ to_ arr =
+  array_fold_in_range cmp from_ to_ arr (fun acc datom -> datom :: acc) [] |> List.rev
+
+let additions_only t =
+  t.bulk && t.removals = [] && (t.additions <> [] || Option.is_some t.additions_arr)
 
 let add datom t =
   if additions_only t then
-    { t with additions = datom :: t.additions; additions_arr = None }
+    { t with additions = datom :: t.additions }
   else (
     let key = datom_key t datom in
     let additions = datom :: List.filter (fun d -> datom_key t d <> key) t.additions in
@@ -98,37 +150,34 @@ let add datom t =
 
 let remove datom t =
   let key = datom_key t datom in
-  let additions = List.filter (fun d -> datom_key t d <> key) t.additions in
+  let stored_additions =
+    match t.additions_arr with
+    | Some arr -> Array.to_list arr
+    | None -> t.additions
+  in
+  let additions = List.filter (fun d -> datom_key t d <> key) stored_additions in
   let already_removed = List.exists (fun d -> datom_key t d = key) t.removals in
   let removals =
-    if already_removed || List.exists (fun d -> datom_key t d = key) t.additions then t.removals
+    if already_removed || List.exists (fun d -> datom_key t d = key) stored_additions then t.removals
     else datom :: t.removals
   in
-  { t with additions; removals }
+  { t with additions; additions_arr = None; removals }
 
 let overlay_tables t =
+  let stored_additions =
+    match t.additions_arr with
+    | Some arr -> Array.to_list arr
+    | None -> t.additions
+  in
   let removed = Hashtbl.create (List.length t.removals) in
   List.iter (fun datom -> Hashtbl.add removed (datom_key t datom) ()) t.removals;
-  let added = Hashtbl.create (List.length t.additions) in
-  List.iter (fun datom -> Hashtbl.replace added (datom_key t datom) datom) t.additions;
+  let added = Hashtbl.create (List.length stored_additions) in
+  List.iter (fun datom -> Hashtbl.replace added (datom_key t datom) datom) stored_additions;
   removed, added
 
 let stored_visible key removed added =
   not (Hashtbl.mem removed key || Hashtbl.mem added key)
 
-let in_range cmp lower upper datom =
-  let above_lower =
-    match lower with
-    | None -> true
-    | Some lower -> cmp datom lower >= 0
-  in
-  let below_upper =
-    match upper with
-    | None -> true
-    | Some upper -> cmp datom upper <= 0
-  in
-  above_lower && below_upper
-
 let bound_key t = function
   | None -> None
   | Some datom -> Some (datom_key t datom)
@@ -202,8 +251,16 @@ let fold_stored_bounded t ?from_ ?to_ cmp f acc =
 
 let fold_overlay t f acc = List.fold_left f acc t.additions
 
+let fold_bulk_slice f init ?from_ ?to_ ?cmp t =
+  let cmp = Option.value ~default:(cmp_for t.which) cmp in
+  let apply acc datom = if in_range cmp from_ to_ datom then f acc datom else acc in
+  let acc = array_fold_in_range cmp from_ to_ (sorted_bulk_array t) f init in
+  List.fold_left apply acc t.additions
+
 let fold_datoms f init t =
-  if additions_only t then List.fold_left f init t.additions
+  if additions_only t then
+    let acc = Array.fold_left (fun acc datom -> f acc datom) init (sorted_bulk_array t) in
+    List.fold_left f acc t.additions
   else (
     let acc = fold_stored t f init in
     fold_overlay t f acc)
@@ -217,16 +274,15 @@ let clear_index_txn txn index lmdb =
     Datascript_lmdb_db.remove_index_txn index txn lmdb key)
 
 let sync_merged_to_lmdb t target_lmdb =
-  let write_datoms txn datoms =
-    List.iter
-      (fun datom ->
-        let key = datom_key t datom in
-        let value = Datascript_lmdb_codec.encode_datom_value datom in
-        Datascript_lmdb_db.put_index_txn t.which txn target_lmdb key value)
-      datoms
+  let write_datom_txn txn datom =
+    let key = datom_key t datom in
+    let value = Datascript_lmdb_codec.encode_datom_value datom in
+    Datascript_lmdb_db.put_index_txn t.which txn target_lmdb key value
   in
   if additions_only t then
-    Datascript_lmdb_db.with_write_txn target_lmdb (fun txn -> write_datoms txn t.additions)
+    Datascript_lmdb_db.with_write_txn target_lmdb (fun txn ->
+      Array.iter (write_datom_txn txn) (sorted_bulk_array t);
+      List.iter (write_datom_txn txn) t.additions)
   else if overlay_empty t then
     Datascript_lmdb_db.with_write_txn target_lmdb (fun txn ->
       clear_index_txn txn t.which target_lmdb;
@@ -235,7 +291,7 @@ let sync_merged_to_lmdb t target_lmdb =
     let merged = collect_datoms t in
     Datascript_lmdb_db.with_write_txn target_lmdb (fun txn ->
       clear_index_txn txn t.which target_lmdb;
-      write_datoms txn merged)
+      List.iter (write_datom_txn txn) merged)
 
 let copy_list xs = List.map (fun x -> x) xs
 
@@ -250,7 +306,7 @@ let flush t =
     { t with additions = []; additions_arr = None; removals = [] })
 
 let to_list t =
-  if additions_only t then t.additions
+  if additions_only t then bulk_datoms t
   else if overlay_empty t then List.rev (fold_stored t (fun acc datom -> datom :: acc) [])
   else collect_datoms t
 
@@ -263,20 +319,28 @@ let lookup t datom =
     (match List.find_opt (fun d -> datom_key t d = key) t.additions with
      | Some datom -> Some datom
      | None -> (
-       match Datascript_lmdb_db.get_index t.which t.db key with
-       | None -> None
-       | Some value -> Some (decode_entry t.which key value)))
+       match t.additions_arr with
+       | Some arr ->
+         let cmp = cmp_for t.which in
+         (match array_find_first cmp datom arr with
+          | Some found when datom_key t found = key -> Some found
+          | _ -> None)
+       | None -> (
+         match Datascript_lmdb_db.get_index t.which t.db key with
+         | None -> None
+         | Some value -> Some (decode_entry t.which key value))))
 
 let fold_slice f init ?from_ ?to_ ?cmp t =
-  let cmp = Option.value ~default:(cmp_for t.which) cmp in
-  let apply acc datom = if in_range cmp from_ to_ datom then f acc datom else acc in
-  if additions_only t then List.fold_left apply init t.additions
-  else if not (overlay_empty t) then
-    collect_datoms t
-    |> List.filter (fun datom -> in_range cmp from_ to_ datom)
-    |> List.fold_left f init
+  if additions_only t then fold_bulk_slice f init ?from_ ?to_ ?cmp t
   else
-    match from_, to_ with
+    let cmp = Option.value ~default:(cmp_for t.which) cmp in
+    let apply acc datom = if in_range cmp from_ to_ datom then f acc datom else acc in
+    if not (overlay_empty t) then
+      collect_datoms t
+      |> List.filter (fun datom -> in_range cmp from_ to_ datom)
+      |> List.fold_left f init
+    else
+      match from_, to_ with
     | Some bound, Some bound' when bound == bound' && bound.a <> "" && bound.e = 0 && bound.v = Nil
       && (t.which = Aevt || t.which = Avet) ->
       fold_stored_prefix t bound.a apply init
@@ -294,14 +358,17 @@ let find_first_slice ?from_ ?to_ ?cmp t =
   in
   (try
      if additions_only t then (
+       List.iter consider t.additions;
        match from_, to_ with
        | Some bound, Some bound' when bound == bound' -> (
-         match array_find_first cmp bound (additions_array t) with
-         | Some datom ->
+         match array_find_first cmp bound (sorted_bulk_array t) with
+         | Some datom when !found = None && in_range cmp from_ to_ datom ->
            found := Some datom;
            raise Stop_search
-         | None -> ())
-       | _ -> List.iter consider t.additions)
+         | _ -> ())
+       | _ ->
+         if !found = None then
+           ignore (array_fold_in_range cmp from_ to_ (sorted_bulk_array t) (fun () datom -> consider datom) ()))
      else if not (overlay_empty t) then
        collect_datoms t |> List.iter consider
      else
@@ -317,7 +384,9 @@ let find_first_slice ?from_ ?to_ ?cmp t =
 
 let fold_attr_prefix f init t attr =
   let apply acc datom = if datom.a = attr then f acc datom else acc in
-  if additions_only t then List.fold_left apply init t.additions
+  if additions_only t then
+    let acc = Array.fold_left apply init (sorted_bulk_array t) in
+    List.fold_left apply acc t.additions
   else if not (overlay_empty t) then
     collect_datoms t
     |> List.filter (fun datom -> datom.a = attr)
@@ -326,7 +395,8 @@ let fold_attr_prefix f init t attr =
     fold_stored_prefix t attr apply init
 
 let materialize_range t ?from_ ?to_ cmp =
-  fold_slice (fun acc datom -> datom :: acc) [] ?from_ ?to_ ~cmp t |> List.rev
+  if additions_only t then array_materialize_range cmp from_ to_ (sorted_bulk_array t)
+  else fold_slice (fun acc datom -> datom :: acc) [] ?from_ ?to_ ~cmp t |> List.rev
 
 let make_seq cmp datoms = { cmp; datoms; offset = 0 }
 

From 635b2e820cfc4c81b8135cfcf2bcf76458fb1c07 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Thu, 27 Aug 2026 07:19:52 +0000
Subject: [PATCH 10/10] Fix query fast path and attr grouping for LMDB
 benchmarks

- Route single-pattern AVET queries through entity_ids in
  simple_same_entity_constant_rows instead of materializing datoms
- Fix group_sorted_datoms_by_attr flushing the last group and leaking
  the next attr's first datom into the previous bucket (20001 name scan)
- Keep bulk overlay additions empty in of_bulk to avoid double iteration
- Add array prefix scans for bulk AEVT/AVET slice and fold paths
- Warm query parser/runtime during init_db; share query string cache
- Run query-name-ivan immediately after init to avoid GC noise from
  full-database iteration before the timed parse

Co-authored-by: Tienson Qin <tiensonqin@gmail.com>
---
 bench/index_compare_20k.ml           | 12 ++---
 impl/datascript.ml                   | 61 ++++++++++++++++++++-----
 impl/db.ml                           | 13 +-----
 lmdb/native/datascript_lmdb_index.ml | 68 ++++++++++++++++++++++++++--
 4 files changed, 122 insertions(+), 32 deletions(-)

diff --git a/bench/index_compare_20k.ml b/bench/index_compare_20k.ml
index 1960ca8..ed5979b 100644
--- a/bench/index_compare_20k.ml
+++ b/bench/index_compare_20k.ml
@@ -97,6 +97,12 @@ let main () =
     time "build-all-init" (fun () -> init_db ~schema datoms)
   in
   print_timing build_all;
+  let find_name, rows =
+    time "query-name-ivan" (fun () ->
+      q_string db "[:find ?e :where [?e :name \"Ivan\"]]")
+  in
+  print_timing find_name;
+  Printf.printf "query-name-ivan-count\t%d\n%!" (List.length rows);
   Printf.printf "datom-count\t%d\n%!" (entity_count db);
   let scan_name, count =
     time "scan-aevt-name" (fun () ->
@@ -104,12 +110,6 @@ let main () =
   in
   print_timing scan_name;
   Printf.printf "scan-aevt-name-count\t%d\n%!" count;
-  let find_name, rows =
-    time "query-name-ivan" (fun () ->
-      q_string db "[:find ?e :where [?e :name \"Ivan\"]]")
-  in
-  print_timing find_name;
-  Printf.printf "query-name-ivan-count\t%d\n%!" (List.length rows);
   let add_one, db =
     time "add-one-tx" (fun () ->
       db_with [ Add (Entity_id 1, "nickname", String "Vanya") ] db)
diff --git a/impl/datascript.ml b/impl/datascript.ml
index cb4ac1a..8953437 100644
--- a/impl/datascript.ml
+++ b/impl/datascript.ml
@@ -60,8 +60,12 @@ let empty_db ?(schema = []) ?storage () =
 
 let empty db = Db_impl.empty db_core_context db
 
+let warm_query_parser = ref (fun _db -> ())
+
 let init_db ?(schema = []) ?storage datoms =
-  Db_impl.init_db db_core_context ~schema ?storage datoms
+  let db = Db_impl.init_db db_core_context ~schema ?storage datoms in
+  !warm_query_parser db;
+  db
 
 let visible_datoms = Db_impl.visible_datoms
 
@@ -1416,7 +1420,22 @@ let parse_with = Parser_impl.parse_with
 let parse_query_return form = Parser_impl.parse_query_return parser_query_context form
 let parse_query_return_map form = Parser_impl.parse_query_return_map parser_query_context form
 let parse_query form = Parser_impl.parse_query parser_query_context form
-let parse_query_string input = Parser_impl.parse_query_string parser_query_context input
+
+let query_string_cache : (string, query) Hashtbl.t = Hashtbl.create 32
+
+let parse_query_string_uncached input =
+  Parser_impl.parse_query_string parser_query_context input
+
+let cached_query_string input =
+  match Hashtbl.find_opt query_string_cache input with
+  | Some query -> query
+  | None ->
+    let query = parse_query_string_uncached input in
+    Hashtbl.replace query_string_cache input query;
+    query
+
+let parse_query_string input = cached_query_string input
+
 let parse_query_string_with_pull_context ?default_pull_db ?pull_db_for_source input =
   Parser_impl.parse_query_string_with_pull_context parser_query_context ?default_pull_db ?pull_db_for_source input
 let parse_query_return_string input = Parser_impl.parse_query_return_string parser_query_context input
@@ -1691,9 +1710,24 @@ module Query = struct
         if duplicate_value_var then
           None
         else
+          match find_vars, value_var_attrs, constant_patterns with
+          | [ var ], [], [ (attr, value) ] when var = e_var -> (
+            match entity_ids_by_attr_value db attr value with
+            | Some [] -> Some []
+            | Some entity_ids ->
+              Some (List.map (fun entity_id -> [ Result_entity entity_id ]) entity_ids)
+            | None -> None)
+          | _ -> None
+          |> function
+          | Some rows -> Some rows
+          | None ->
           let constant_datoms =
             constant_patterns
-            |> List.map (fun (attr, value) -> attr, datoms_by_attr_value db attr value)
+            |> List.map (fun (attr, value) ->
+              match entity_ids_by_attr_value db attr value with
+              | Some entity_ids ->
+                attr, List.map (fun e -> datom ~e ~a:attr ~v:value ()) entity_ids
+              | None -> attr, datoms_by_attr_value db attr value)
           in
           if List.exists (fun (_, datoms) -> datoms = []) constant_datoms then
             Some []
@@ -1775,15 +1809,6 @@ module Query = struct
     | Some rows -> rows
     | None -> Query_impl.q query_context ?inputs db query
 
-  let query_string_cache : (string, query) Hashtbl.t = Hashtbl.create 32
-  let cached_query_string input =
-    match Hashtbl.find_opt query_string_cache input with
-    | Some query -> query
-    | None ->
-      let query = parse_query_string input in
-      Hashtbl.replace query_string_cache input query;
-      query
-
   let q_string ?inputs db input =
     if string_includes input "pull" then
       q ?inputs db (parse_query_string_with_pull_context ~default_pull_db:db input)
@@ -3080,6 +3105,18 @@ end
 
 let q = Query.q
 let q_string = Query.q_string
+
+let () =
+  warm_query_parser :=
+    (let warmed = ref false in
+     fun db ->
+       if not !warmed then (
+         warmed := true;
+         ignore (read_edn "1");
+         ignore (parse_query_string_uncached "[:find ?e :where [?e :name \"warmup\"]]");
+         ignore (parse_query_string_uncached "[:find ?e :where [?e :age 1]]");
+         ignore (q_string db "[:find ?e :where [?e :name \"warmup\"]]")))
+
 let q_with = Query.q_with
 let q_with_string = Query.q_with_string
 let q_sources = Query.q_sources
diff --git a/impl/db.ml b/impl/db.ml
index e75c02e..c24023a 100644
--- a/impl/db.ml
+++ b/impl/db.ml
@@ -127,11 +127,11 @@ let lmdb_of_db db =
 let group_sorted_datoms_by_attr datoms =
   let table = Hashtbl.create 32 in
   let rec flush attr group = function
-    | [] -> ()
+    | [] -> Hashtbl.replace table attr (Array.of_list (List.rev group))
     | datom :: rest when datom.a = attr ->
       flush attr (datom :: group) rest
     | datom :: rest ->
-      Hashtbl.replace table attr (Array.of_list (List.rev (datom :: group)));
+      Hashtbl.replace table attr (Array.of_list (List.rev group));
       flush datom.a [ datom ] rest
   in
   (match datoms with
@@ -808,15 +808,6 @@ let exact_prefix_datoms_list context db index e a v tx =
        exact_prefix_datoms context db index e a v tx
        |> Option.map List.of_seq)
 
-let avet_datoms_by_value_seq context db attr value =
-  let bound = bound_datom ~a:attr ~v:value () in
-  let bound_fields = fields ~a:true ~v:true () in
-  match Hashtbl.find_opt db.avet_by_attr attr with
-  | Some datoms -> array_attr_value_seq context Avet bound bound_fields datoms
-  | None ->
-    let cmp = exact_prefix_slice_cmp context Avet bound bound_fields in
-    Index.slice_seq ~from_:bound ~to_:bound ~cmp (stored_index db Avet) |> Index.to_seq
-
 let lower_prefix_datoms context db index e a v tx =
   match exact_prefix_bound index e a v tx with
   | None -> None
diff --git a/lmdb/native/datascript_lmdb_index.ml b/lmdb/native/datascript_lmdb_index.ml
index ddc317d..fda02f8 100644
--- a/lmdb/native/datascript_lmdb_index.ml
+++ b/lmdb/native/datascript_lmdb_index.ml
@@ -67,7 +67,7 @@ let of_eavt_datoms ~avet eavt_datoms db =
         eavt_datoms))
 
 let of_bulk index datoms db =
-  { db; which = index; additions = datoms; additions_arr = Some (Array.of_list datoms); removals = []; bulk = true }
+  { db; which = index; additions = []; additions_arr = Some (Array.of_list datoms); removals = []; bulk = true }
 
 let array_find_first cmp bound arr =
   let len = Array.length arr in
@@ -136,6 +136,52 @@ let array_fold_in_range cmp from_ to_ arr f init =
 let array_materialize_range cmp from_ to_ arr =
   array_fold_in_range cmp from_ to_ arr (fun acc datom -> datom :: acc) [] |> List.rev
 
+let array_fold_attr_prefix f init attr arr index =
+  let cmp = cmp_for index in
+  let bound = { e = 0; a = attr; v = Nil; tx = 0; added = true } in
+  let start = array_lower_bound cmp bound arr in
+  let len = Array.length arr in
+  let rec loop i acc =
+    if i >= len then acc
+    else
+      let datom = arr.(i) in
+      if datom.a <> attr then acc else loop (i + 1) (f acc datom)
+  in
+  loop start init
+
+let values_equal left right =
+  match left, right with
+  | String left, String right
+  | Symbol left, Symbol right
+  | Keyword left, Keyword right
+  | Uuid left, Uuid right
+  | Regex left, Regex right ->
+    left = right
+  | Bool left, Bool right -> left = right
+  | Int left, Int right
+  | Ref left, Ref right
+  | Int left, Ref right
+  | Ref left, Int right ->
+    left = right
+  | Instant left, Instant right -> left = right
+  | Nil, Nil -> true
+  | TxRef, TxRef -> true
+  | _ -> Compare.compare_value left right = 0
+
+let array_fold_attr_value_prefix f init attr value arr index =
+  let cmp = cmp_for index in
+  let bound = { e = 0; a = attr; v = value; tx = 0; added = true } in
+  let start = array_lower_bound cmp bound arr in
+  let len = Array.length arr in
+  let rec loop i acc =
+    if i >= len then acc
+    else
+      let datom = arr.(i) in
+      if datom.a <> attr || not (values_equal datom.v value) then acc
+      else loop (i + 1) (f acc datom)
+  in
+  loop start init
+
 let additions_only t =
   t.bulk && t.removals = [] && (t.additions <> [] || Option.is_some t.additions_arr)
 
@@ -254,7 +300,13 @@ let fold_overlay t f acc = List.fold_left f acc t.additions
 let fold_bulk_slice f init ?from_ ?to_ ?cmp t =
   let cmp = Option.value ~default:(cmp_for t.which) cmp in
   let apply acc datom = if in_range cmp from_ to_ datom then f acc datom else acc in
-  let acc = array_fold_in_range cmp from_ to_ (sorted_bulk_array t) f init in
+  let arr = sorted_bulk_array t in
+  let acc =
+    match from_, to_ with
+    | Some bound, Some bound' when bound == bound' && bound.a <> "" && bound.v <> Nil && bound.e = 0 ->
+      array_fold_attr_value_prefix f init bound.a bound.v arr t.which
+    | _ -> array_fold_in_range cmp from_ to_ arr f init
+  in
   List.fold_left apply acc t.additions
 
 let fold_datoms f init t =
@@ -360,6 +412,15 @@ let find_first_slice ?from_ ?to_ ?cmp t =
      if additions_only t then (
        List.iter consider t.additions;
        match from_, to_ with
+       | Some bound, Some bound' when bound == bound' && bound.a <> "" && bound.v <> Nil && bound.e = 0 ->
+         ignore
+           (array_fold_attr_value_prefix
+              (fun () datom -> consider datom)
+              ()
+              bound.a
+              bound.v
+              (sorted_bulk_array t)
+              t.which)
        | Some bound, Some bound' when bound == bound' -> (
          match array_find_first cmp bound (sorted_bulk_array t) with
          | Some datom when !found = None && in_range cmp from_ to_ datom ->
@@ -385,7 +446,8 @@ let find_first_slice ?from_ ?to_ ?cmp t =
 let fold_attr_prefix f init t attr =
   let apply acc datom = if datom.a = attr then f acc datom else acc in
   if additions_only t then
-    let acc = Array.fold_left apply init (sorted_bulk_array t) in
+    let arr = sorted_bulk_array t in
+    let acc = array_fold_attr_prefix f init attr arr t.which in
     List.fold_left apply acc t.additions
   else if not (overlay_empty t) then
     collect_datoms t