diff --git a/.cursor/Dockerfile b/.cursor/Dockerfile new file mode 100644 index 0000000..4374c88 --- /dev/null +++ b/.cursor/Dockerfile @@ -0,0 +1,35 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive +ENV OPAMYES=1 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + git \ + build-essential \ + pkg-config \ + libsqlite3-dev \ + liblmdb-dev \ + && rm -rf /var/lib/apt/lists/* + +# Node.js 24 for js_of_ocaml smoke tests and cross-runtime helpers. +RUN curl -fsSL https://deb.nodesource.com/setup_24.x | bash - \ + && apt-get install -y --no-install-recommends nodejs \ + && rm -rf /var/lib/apt/lists/* + +RUN useradd -m -s /bin/bash ubuntu 2>/dev/null || true + +USER ubuntu +WORKDIR /home/ubuntu + +RUN curl -fsSL https://raw.githubusercontent.com/ocaml/opam/master/shell/install.sh \ + | bash -s -- --disable-sandboxing \ + && opam init --disable-sandboxing -y \ + && opam switch create 5.5 ocaml-base-compiler.5.5.0 -y \ + && eval "$(opam env --switch=5.5)" \ + && opam update -a + +RUN echo 'test -r ~/.opam/opam-init/init.sh && . ~/.opam/opam-init/init.sh > /dev/null 2> /dev/null || true' >> ~/.bashrc \ + && echo 'eval $(opam env --switch=5.5 2>/dev/null)' >> ~/.bashrc diff --git a/.cursor/cloud-agent-install.sh b/.cursor/cloud-agent-install.sh new file mode 100755 index 0000000..798992e --- /dev/null +++ b/.cursor/cloud-agent-install.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(git rev-parse --show-toplevel)" +cd "$repo_root" + +eval "$(opam env --switch=5.5)" + +export OPAMYES=1 +opam install . --deps-only --with-test -y +dune build diff --git a/.cursor/environment.json b/.cursor/environment.json new file mode 100644 index 0000000..491f294 --- /dev/null +++ b/.cursor/environment.json @@ -0,0 +1,9 @@ +{ + "name": "DataScript OCaml (OCaml 5.5)", + "user": "ubuntu", + "build": { + "dockerfile": "Dockerfile", + "context": ".." + }, + "install": ".cursor/cloud-agent-install.sh" +} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f883c22..cbfbd2b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,7 +48,7 @@ jobs: lein: latest - name: Install system dependencies - run: sudo apt-get update && sudo apt-get install -y libsqlite3-dev pkg-config + run: sudo apt-get update && sudo apt-get install -y libsqlite3-dev liblmdb-dev pkg-config - name: Install OCaml dependencies run: opam install . --deps-only --with-test -y diff --git a/bench/compare_pss_lmdb.sh b/bench/compare_pss_lmdb.sh new file mode 100755 index 0000000..8b330d5 --- /dev/null +++ b/bench/compare_pss_lmdb.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +set -euo pipefail + +SIZE="${1:-20000}" +WARMUP_MS="${2:-100}" +SAMPLE_MS="${3:-200}" +SAMPLES="${4:-3}" + +bench_args=(--size "$SIZE" --warmup-ms "$WARMUP_MS" --sample-ms "$SAMPLE_MS" --samples "$SAMPLES") + +run_branch_bench() { + local label="$1" + local repo="$2" + ( + cd "$repo" + dune build bench/bench_ocaml.exe >/dev/null + BENCH_RUNTIME_LABEL="$label" dune exec bench/bench_ocaml.exe -- "${bench_args[@]}" 2>/dev/null + ) +} + +echo "=== PSS vs LMDB benchmark (${SIZE} entities) ===" +echo "warmup=${WARMUP_MS}ms sample=${SAMPLE_MS}ms samples=${SAMPLES}" +echo + +PSS_OUT="$(run_branch_bench pss /tmp/bench-pss-main)" +LMDB_OUT="$(run_branch_bench lmdb /workspace)" + +printf "%-22s %12s %12s %12s\n" "benchmark" "pss(ms)" "lmdb(ms)" "lmdb/pss" +echo "----------------------------------------------------------------" + +while IFS=$'\t' read -r name pss_ms; do + [[ "$name" == runtime* || "$name" == size* || -z "$name" ]] && continue + lmdb_ms="$(printf '%s\n' "$LMDB_OUT" | awk -F'\t' -v n="$name" '$1 == n { print $2; exit }')" + if [[ -z "$lmdb_ms" ]]; then + printf "%-22s %12s %12s %12s\n" "$name" "$pss_ms" "?" "?" + continue + fi + ratio="$(awk -v l="$lmdb_ms" -v p="$pss_ms" 'BEGIN { if (p + 0 == 0) print "?"; else printf "%.2fx", l / p }')" + printf "%-22s %12s %12s %12s\n" "$name" "$pss_ms" "$lmdb_ms" "$ratio" +done <<< "$PSS_OUT" + +echo +echo "=== raw: pss ===" +printf '%s\n' "$PSS_OUT" +echo +echo "=== raw: lmdb ===" +printf '%s\n' "$LMDB_OUT" diff --git a/bench/compare_pss_lmdb_20k.sh b/bench/compare_pss_lmdb_20k.sh new file mode 100755 index 0000000..d6d51f1 --- /dev/null +++ b/bench/compare_pss_lmdb_20k.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +set -euo pipefail + +SIZE="${1:-20000}" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +PSS_REPO="${PSS_REPO:-/tmp/bench-pss-main}" + +run_branch() { + local label="$1" + local repo="$2" + ( + cd "$repo" + dune build bench/index_compare_20k.exe >/dev/null + BENCH_RUNTIME_LABEL="$label" dune exec bench/index_compare_20k.exe -- "$SIZE" 2>/dev/null + ) +} + +if [[ ! -e "$PSS_REPO/.git" ]]; then + echo "PSS worktree missing at $PSS_REPO; run: git worktree add $PSS_REPO main" >&2 + exit 1 +fi + +echo "=== PSS vs LMDB index benchmark (${SIZE} entities / ~${SIZE}0 datoms) ===" +echo + +PSS_OUT="$(run_branch pss "$PSS_REPO")" +LMDB_OUT="$(run_branch lmdb "$REPO_ROOT")" + +printf "%-24s %12s %12s %12s\n" "benchmark" "pss(ms)" "lmdb(ms)" "lmdb/pss" +echo "------------------------------------------------------------------------" + +while IFS=$'\t' read -r name pss_ms; do + [[ "$name" == runtime* || "$name" == size* || "$name" == datoms || "$name" == *count* || -z "$name" ]] && continue + lmdb_ms="$(printf '%s\n' "$LMDB_OUT" | awk -F'\t' -v n="$name" '$1 == n { print $2; exit }')" + if [[ -z "$lmdb_ms" ]]; then + printf "%-24s %12s %12s %12s\n" "$name" "$pss_ms" "?" "?" + continue + fi + ratio="$(awk -v l="$lmdb_ms" -v p="$pss_ms" 'BEGIN { if (p + 0 == 0) print "?"; else printf "%.2fx", l / p }')" + printf "%-24s %12s %12s %12s\n" "$name" "$pss_ms" "$lmdb_ms" "$ratio" +done <<< "$PSS_OUT" + +echo +echo "=== raw: pss ===" +printf '%s\n' "$PSS_OUT" +echo +echo "=== raw: lmdb ===" +printf '%s\n' "$LMDB_OUT" diff --git a/bench/dune b/bench/dune index 67c4a83..2b6fe96 100644 --- a/bench/dune +++ b/bench/dune @@ -1,3 +1,8 @@ +(executable + (name index_compare_20k) + (modules index_compare_20k) + (libraries datascript-ocaml-native unix)) + (executable (name bench_ocaml) (modules bench_ocaml) @@ -33,6 +38,12 @@ (modes exe) (libraries datascript-ocaml-native datascript_sqlite unix sqlite3)) +(executable + (name persistent_storage_bench) + (modules persistent_storage_bench) + (modes exe) + (libraries datascript-ocaml-native datascript_sqlite datascript_lmdb unix sqlite3)) + (executable (name outliner_insert_ocaml) (modules outliner_insert_ocaml) diff --git a/bench/index_compare_20k.ml b/bench/index_compare_20k.ml new file mode 100644 index 0000000..ed5979b --- /dev/null +++ b/bench/index_compare_20k.ml @@ -0,0 +1,131 @@ +open Datascript + +type timing = { label : string; elapsed_ms : float } + +let now_ms () = Unix.gettimeofday () *. 1000. + +let time label f = + let start = now_ms () in + let result = f () in + ({ label; elapsed_ms = now_ms () -. start }, result) + +let print_timing { label; elapsed_ms } = + Printf.printf "%s\t%.2f\n%!" label elapsed_ms + +let indexed = + { + cardinality = One; + unique = None; + indexed = true; + is_component = false; + no_history = false; + doc = None; + value_type = None; + tuple_attrs = None; + tuple_types = None; + } + +let unique_identity = { indexed with unique = Some Identity } + +let many = + { + cardinality = Many; + unique = None; + indexed = false; + is_component = false; + no_history = false; + doc = None; + value_type = None; + tuple_attrs = None; + tuple_types = None; + } + +let schema = + [ ("id", unique_identity) + ; ("name", indexed) + ; ("age", indexed) + ; ("salary", indexed) + ; ("alias", many) + ] + +let names = [| "Ivan"; "Petr"; "Sergey"; "Oleg"; "Yuri"; "Dmitry"; "Fedor"; "Denis" |] +let last_names = [| "Ivanov"; "Petrov"; "Sidorov"; "Kovalev"; "Kuznetsov"; "Voronoi" |] + +type rng = { mutable state : int32 } + +let rng seed = { state = Int32.of_int seed } + +let next_int rng bound = + rng.state <- Int32.add (Int32.mul rng.state 1_664_525l) 1_013_904_223l; + Int32.(to_int (rem (logand (shift_right_logical rng.state 1) 0x3fffffffl) (of_int bound))) + +let rand_nth rng values = values.(next_int rng (Array.length values)) + +let datoms_for size = + let rng = rng 1 in + List.init size (fun index -> + let i = index + 1 in + let name = rand_nth rng names in + let last_name = rand_nth rng last_names in + [ + { e = i; a = "name"; v = String name; tx = 0x20000001; added = true } + ; { e = i; a = "last-name"; v = String last_name; tx = 0x20000001; added = true } + ; { e = i; a = "age"; v = Int (next_int rng 100); tx = 0x20000001; added = true } + ; { e = i; a = "salary"; v = Int (next_int rng 100_000); tx = 0x20000001; added = true } + ]) + |> List.concat + +let entity_count db = Seq.length (datoms db Eavt ()) + +let parse_size () = + match Sys.argv with + | [| _; size |] -> int_of_string size + | _ -> 20_000 + +let main () = + let size = parse_size () in + let runtime_label = + match Sys.getenv_opt "BENCH_RUNTIME_LABEL" with + | Some label -> label + | None -> "ocaml" + in + Printf.printf "runtime\t%s\n%!" runtime_label; + Printf.printf "size\t%d\n%!" size; + Printf.printf "datoms\t%d\n%!" (size * 4); + let datoms = datoms_for size in + let build_all, db = + time "build-all-init" (fun () -> init_db ~schema datoms) + in + print_timing build_all; + let find_name, rows = + time "query-name-ivan" (fun () -> + q_string db "[:find ?e :where [?e :name \"Ivan\"]]") + in + print_timing find_name; + Printf.printf "query-name-ivan-count\t%d\n%!" (List.length rows); + Printf.printf "datom-count\t%d\n%!" (entity_count db); + let scan_name, count = + time "scan-aevt-name" (fun () -> + fold_datoms (fun count _ -> count + 1) 0 db Aevt ~a:"name" ()) + in + print_timing scan_name; + Printf.printf "scan-aevt-name-count\t%d\n%!" count; + let add_one, db = + time "add-one-tx" (fun () -> + db_with [ Add (Entity_id 1, "nickname", String "Vanya") ] db) + in + print_timing add_one; + ignore db; + let storage, restored = + time "storage-roundtrip" (fun () -> + let storage = memory_storage () in + let db = init_db ~schema ~storage datoms in + store db; + match restore storage with + | Some db -> db + | None -> failwith "restore failed") + in + print_timing storage; + Printf.printf "restored-datom-count\t%d\n%!" (entity_count restored) + +let () = main () diff --git a/bench/persistent_storage_bench.ml b/bench/persistent_storage_bench.ml new file mode 100644 index 0000000..1dd101d --- /dev/null +++ b/bench/persistent_storage_bench.ml @@ -0,0 +1,263 @@ +open Datascript + +type timing = { label : string; elapsed_ms : float } + +let now_ms () = Unix.gettimeofday () *. 1000. + +let time label f = + let start = now_ms () in + let result = f () in + ({ label; elapsed_ms = now_ms () -. start }, result) + +let print_timing prefix { label; elapsed_ms } = + Printf.printf "%s%s\t%.2f\n%!" prefix label elapsed_ms + +let indexed = + { + cardinality = One; + unique = None; + indexed = true; + is_component = false; + no_history = false; + doc = None; + value_type = None; + tuple_attrs = None; + tuple_types = None; + } + +let unique_identity = { indexed with unique = Some Identity } + +let schema = + [ + ("block/id", unique_identity); + ("block/journal-day", indexed); + ("block/content", indexed); + ("block/order", indexed); + ("block/collapsed", indexed); + ] + +let block_tx count = + List.init count (fun index -> + let i = index + 1 in + Entity + { + db_id = Some (Temp_id (Printf.sprintf "block-%05d" i)); + attrs = + [ + ("block/id", One_value (String (Printf.sprintf "block-%05d" i))); + ("block/journal-day", One_value (String "2026-06-27")); + ("block/content", One_value (String (Printf.sprintf "Block %05d" i))); + ("block/order", One_value (Float (Float.of_int i))); + ("block/collapsed", One_value (Bool false)); + ]; + }) + +let add_block_tx id order = + [ + Entity + { + db_id = Some (Temp_id id); + attrs = + [ + ("block/id", One_value (String id)); + ("block/journal-day", One_value (String "2026-06-27")); + ("block/content", One_value (String id)); + ("block/order", One_value (Float order)); + ("block/collapsed", One_value (Bool false)); + ]; + }; + ] + +let update_content_tx id content = + [ Add (Lookup_ref ("block/id", String id), "block/content", String content) ] + +let seq_length seq = Seq.fold_left (fun count _ -> count + 1) 0 seq + +let file_size path = + if Sys.file_exists path then (Unix.stat path).st_size else 0 + +let remove_if_exists path = if Sys.file_exists path then Sys.remove path + +let row_count storage = List.length (storage_addresses storage) + +module type BACKEND = sig + val name : string + val extension : string + type session + val open_session : string -> session + val close_session : session -> unit + val storage : session -> storage + val cleanup : string -> unit +end + +module Sqlite_backend : BACKEND = struct + type session = Datascript_sqlite.session + + let name = "sqlite" + let extension = "sqlite3" + let open_session = Datascript_sqlite.open_session + let close_session = Datascript_sqlite.close + let storage = Datascript_sqlite.storage + let cleanup _path = () +end + +module Lmdb_backend : BACKEND = struct + type session = Datascript_lmdb.session + + let name = "lmdb" + let extension = "lmdb" + let open_session = Datascript_lmdb.open_session + let close_session = Datascript_lmdb.close + let storage = Datascript_lmdb.storage + + let cleanup path = + let lock = path ^ "-lock" in + if Sys.file_exists lock then Sys.remove lock +end + +let run_backend (module B : BACKEND) size tx = + let prefix = B.name ^ "-" in + let db_path = + Filename.concat + (Filename.get_temp_dir_name ()) + (Printf.sprintf "datascript-persistent-%s-%d.%s" B.name size B.extension) + in + remove_if_exists db_path; + let session = B.open_session db_path in + Fun.protect + ~finally:(fun () -> + B.close_session session; + remove_if_exists db_path; + B.cleanup db_path) + (fun () -> + let storage = B.storage session in + let persistent_build, persistent_db = + time "snapshot-build-and-store" (fun () -> + let db = db_with tx (empty_db ~schema ~storage ()) in + store db; + db) + in + print_timing prefix persistent_build; + Printf.printf "%ssnapshot-build-datoms\t%d\n%!" prefix + (seq_length (datoms persistent_db Eavt ())); + Printf.printf "%ssnapshot-kvs-rows-after-build\t%d\n%!" prefix (row_count storage); + Printf.printf "%ssnapshot-file-size-after-build\t%d\n%!" prefix (file_size db_path); + let restore_timing, restored_db = + time "snapshot-restore" (fun () -> + match restore storage with + | Some db -> db + | None -> failwith (B.name ^ " persistent db should restore")) + in + print_timing prefix restore_timing; + let persistent_add, restored_db = + time "snapshot-add-one-and-store-after-restore" (fun () -> + let db = + db_with + (add_block_tx "persistent-new" (Float.of_int (size + 1))) + restored_db + in + store db; + db) + in + print_timing prefix persistent_add; + Printf.printf "%ssnapshot-kvs-rows-after-add\t%d\n%!" prefix (row_count storage); + Printf.printf "%ssnapshot-file-size-after-add\t%d\n%!" prefix (file_size db_path); + let persistent_update, restored_db = + time "snapshot-update-one-and-store-after-add" (fun () -> + let db = db_with (update_content_tx "block-00001" "Edited") restored_db in + store db; + db) + in + print_timing prefix persistent_update; + Printf.printf "%ssnapshot-kvs-rows-after-update\t%d\n%!" prefix (row_count storage); + Printf.printf "%ssnapshot-file-size-after-update\t%d\n%!" prefix (file_size db_path); + Printf.printf "%ssnapshot-datoms\t%d\n%!" prefix + (seq_length (datoms restored_db Eavt ())); + let conn_db_path = + Filename.concat + (Filename.get_temp_dir_name ()) + (Printf.sprintf "datascript-persistent-%s-conn-%d.%s" B.name size B.extension) + in + remove_if_exists conn_db_path; + let session = B.open_session conn_db_path in + Fun.protect + ~finally:(fun () -> + B.close_session session; + remove_if_exists conn_db_path; + B.cleanup conn_db_path) + (fun () -> + let storage = B.storage session in + let conn_build, conn = + time "conn-build" (fun () -> + let conn = create_conn ~schema ~storage () in + ignore (transact_conn conn tx); + conn) + in + print_timing prefix conn_build; + Printf.printf "%sconn-build-datoms\t%d\n%!" prefix + (seq_length (datoms (db conn) Eavt ())); + Printf.printf "%sconn-kvs-rows-after-build\t%d\n%!" prefix (row_count storage); + Printf.printf "%sconn-file-size-after-build\t%d\n%!" prefix (file_size conn_db_path); + let conn_restore, conn = + time "conn-restore" (fun () -> + match restore_conn storage with + | Some conn -> conn + | None -> failwith (B.name ^ " persistent conn should restore")) + in + print_timing prefix conn_restore; + let conn_add, _report = + time "conn-add-one-after-restore" (fun () -> + transact_conn conn (add_block_tx "conn-new" (Float.of_int (size + 1)))) + in + print_timing prefix conn_add; + Printf.printf "%sconn-kvs-rows-after-add\t%d\n%!" prefix (row_count storage); + Printf.printf "%sconn-file-size-after-add\t%d\n%!" prefix (file_size conn_db_path); + let conn_update, _report = + time "conn-update-one-after-add" (fun () -> + transact_conn conn (update_content_tx "block-00001" "Edited")) + in + print_timing prefix conn_update; + Printf.printf "%sconn-kvs-rows-after-update\t%d\n%!" prefix (row_count storage); + Printf.printf "%sconn-file-size-after-update\t%d\n%!" prefix (file_size conn_db_path); + Printf.printf "%sconn-datoms\t%d\n%!" prefix (seq_length (datoms (db conn) Eavt ())))) + +let run_size size = + Printf.printf "size\t%d\n%!" size; + let tx = block_tx size in + let memory_build, memory_db = + time "memory-build" (fun () -> db_with tx (empty_db ~schema ())) + in + print_timing "" memory_build; + let memory_add, memory_db = + time "memory-add-one" (fun () -> + db_with (add_block_tx "memory-new" (Float.of_int (size + 1))) memory_db) + in + print_timing "" memory_add; + let _memory_update, memory_db = + time "memory-update-one" (fun () -> + db_with (update_content_tx "block-00001" "Edited") memory_db) + in + print_timing "" _memory_update; + Printf.printf "memory-datoms\t%d\n%!" (seq_length (datoms memory_db Eavt ())); + run_backend (module Sqlite_backend) size tx; + run_backend (module Lmdb_backend) size tx + +let parse_sizes () = + let rec loop sizes = function + | [] -> List.rev sizes + | "--size" :: size :: rest -> loop (int_of_string size :: sizes) rest + | "--sizes" :: value :: rest -> + let parsed = + value + |> String.split_on_char ',' + |> List.filter (fun value -> String.length value > 0) + |> List.map int_of_string + in + loop (List.rev_append parsed sizes) rest + | arg :: _ -> invalid_arg ("unknown benchmark argument: " ^ arg) + in + match loop [] (Sys.argv |> Array.to_list |> List.tl) with + | [] -> [ 100; 1000; 5000 ] + | sizes -> sizes + +let () = List.iter run_size (parse_sizes ()) diff --git a/datascript-ocaml-melange.opam b/datascript-ocaml-melange.opam index 3f574a1..7d8532e 100644 --- a/datascript-ocaml-melange.opam +++ b/datascript-ocaml-melange.opam @@ -8,12 +8,10 @@ depends: [ "ocaml" {>= "5.1.1"} "dune" {>= "3.17"} "datascript_ocaml" {= version} - "persistent_sorted_set_ocaml" {= "dev"} "melange" "melange-transit-melange" {= "0.1.0"} ] pin-depends: [ - ["persistent_sorted_set_ocaml.dev" "git+https://github.com/logseq/persistent-sorted-set-ocaml.git#main"] ["melange-edn-core.0.5.0" "git+https://github.com/RCmerci/melange-edn.git#main"] ["melange-edn-melange.0.5.0" "git+https://github.com/RCmerci/melange-edn.git#main"] ["melange-transit-core.0.1.0" "git+https://github.com/RCmerci/melange-transit.git#main"] diff --git a/datascript-ocaml-native.opam b/datascript-ocaml-native.opam index 9fb78c0..3858445 100644 --- a/datascript-ocaml-native.opam +++ b/datascript-ocaml-native.opam @@ -8,13 +8,12 @@ depends: [ "ocaml" {>= "5.1.1"} "dune" {>= "3.17"} "datascript_ocaml" {= version} - "persistent_sorted_set_ocaml" {= "dev"} "sqlite3" + "lmdb" "melange-transit-native" {= "0.1.0"} "yojson" ] pin-depends: [ - ["persistent_sorted_set_ocaml.dev" "git+https://github.com/logseq/persistent-sorted-set-ocaml.git#main"] ["melange-edn-core.0.5.0" "git+https://github.com/RCmerci/melange-edn.git#main"] ["melange-edn-native.0.5.0" "git+https://github.com/RCmerci/melange-edn.git#main"] ["melange-transit-core.0.1.0" "git+https://github.com/RCmerci/melange-transit.git#main"] diff --git a/datascript_ocaml.opam b/datascript_ocaml.opam index 72cf185..4130cd0 100644 --- a/datascript_ocaml.opam +++ b/datascript_ocaml.opam @@ -7,12 +7,8 @@ license: "MIT" depends: [ "ocaml" {>= "5.1.1"} "dune" {>= "3.17"} - "persistent_sorted_set_ocaml" {= "dev"} "melange" ] -pin-depends: [ - ["persistent_sorted_set_ocaml.dev" "git+https://github.com/logseq/persistent-sorted-set-ocaml.git#main"] -] build: [ ["dune" "build" "-p" name "-j" jobs] ] diff --git a/docs/design-non-pss.md b/docs/design-non-pss.md new file mode 100644 index 0000000..c9ef052 --- /dev/null +++ b/docs/design-non-pss.md @@ -0,0 +1,153 @@ +# Non-PSS LMDB Index Design + +Branch: `feat/non-pss` + +This document records the architecture for replacing `Persistent_sorted_set` indexes +with native LMDB indexes. Queries and writes go directly to LMDB. Datoms keep `tx`, +`added`, and DataScript `value` types (not Datalevin AVG/aid encoding). + +## Type strategy: Scheme A + +Replace PSS field types in `db` with an explicit `Index.t`. Do not alias +`Persistent_sorted_set` to LMDB. + +### New public types (`type/datascript_types.ml`) + +```ocaml +module Index : sig + type t + type 'a seq + val to_seq : 'a seq -> 'a Seq.t +end + +type index = Index.t + +and db = { + db_uid : int; + schema : schema; + eavt_index : index; + aevt_index : index; + avet_index : index; + (* caches and duplicate side tables unchanged *) + ... + storage_ref : storage option; + ... +} +``` + +`Index.t` is abstract at the type level. Native builds use LMDB; the type does not +mention PSS or LMDB in `datascript_types`. + +### Index module API (`lmdb/datascript_lmdb_index.mli`) + +Surface area required by `impl/db.ml` (PSS-free subset): + +```ocaml +type t +type 'a seq + +val empty : index -> t +val of_sorted_list : index -> datom list -> t + +val add : datom -> t -> t +val remove : datom -> t -> t + +val to_list : t -> datom list +val fold : (acc -> datom -> acc) -> acc -> t -> acc + +val slice : ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom list +val slice_seq : ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom seq +val rslice_seq : ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom seq +val seq : t -> datom seq +val seq_to_list : datom seq -> datom list +val fold_seq : (acc -> datom -> acc) -> acc -> datom seq -> acc +val to_seq : datom seq -> datom Seq.t +``` + +Notes: + +- `index` parameter is `Eavt | Aevt | Avet` (`Datascript_types.index`). +- Default comparator is `Util.compare_datom index`; bound slices pass custom `cmp` + (same as PSS today). +- `t` holds a reference to the shared LMDB environment and the DBI name for that + logical index (`ds/eavt`, `ds/aevt`, `ds/avet`). + +### Shared LMDB database handle + +Introduce `Lmdb_db.t` (one env per storage session): + +```ocaml +type t = { + env : Lmdb.Env.t; + eavt : (bytes, bytes, [`Uni]) Lmdb.Map.t; + aevt : (bytes, bytes, [`Uni]) Lmdb.Map.t; + avet : (bytes, bytes, [`Uni]) Lmdb.Map.t; + meta : (string, bytes, [`Uni]) Lmdb.Map.t; + path : string; +} +``` + +Each `Index.t` is `{ db : Lmdb_db.t; which : index }` or three dedicated handles +created at `empty_db` / `restore`. + +`storage` on `feat/non-pss` simplifies to wrapping `Lmdb_db.t`: + +```ocaml +type storage = Lmdb_db.t +``` + +Remove PSS snapshot payloads (`Storage_root`, `Storage_node`, tail groups) from the +native non-PSS path. Logseq SQLite reader code stays on other branches. + +## LMDB key/value encoding + +Keys are order-preserving binary tuples matching `Util.compare_datom`: + +| Index | Key field order | +| --- | --- | +| EAVT | e, a, v, tx | +| AEVT | a, e, v, tx | +| AVET | a, v, e, tx | + +- `v` uses a typed order-preserving encoder aligned with `Util.compare_value`. +- `tx` is included in the key (unlike Datalevin). +- Value payload stores at least `added : bool` and may mirror `v` for simpler decode. + +Duplicate numeric facts (`Int 1` vs `Float 1.0` comparator-equal but distinct) stay +in side tables (`duplicate_*` fields on `db`) with merge at read time, same as today. + +## Code migration map (Scheme A) + +| Area | Change | +| --- | --- | +| `type/datascript_types.ml` | `eavt_index` etc. become `Index.t`; drop PSS from `db` | +| `impl/db.ml` | `module PSet = ...` removed; call `Lmdb_index.*` | +| `impl/datascript.ml` | Replace direct `PSet.*` on `db.*_index` | +| `impl/serialize.ml` | Iterate LMDB or `to_list`; no PSS builders | +| `impl/storage.ml` | Open/sync `Lmdb_db`; delete PSS store/restore/tail | +| `impl/conn.ml` | Commit LMDB txn per transact; no tail compaction | +| `lmdb/*` | `datascript_lmdb_codec.ml`, `datascript_lmdb_index.ml`, `datascript_lmdb_db.ml` | +| `impl/dune` | Native links `datascript_lmdb_index`; drop `persistent_sorted_set_ocaml` on this branch | +| `test/test_db.ml` | Replace `assert_uses_persistent_sorted_set` with LMDB index checks | +| `bench/*` | Update labels; expect no snapshot full-tree rewrite | + +## What stays unchanged + +- Public query semantics: lazy `datoms`, bound slices, filtered DB order. +- `datom` record including `tx` and `added`. +- `value` algebra and `Util.compare_value` rules. +- Three indexes: EAVT, AEVT, AVET. + +## Implementation phases + +1. **Codec + Index unit tests** — key order matches `compare_datom`. +2. **`Lmdb_db` lifecycle** — temp env for `empty_db ()`, path-backed for conn. +3. **Rewire `db.ml`** — Scheme A types throughout. +4. **Storage + conn** — remove PSS snapshot/tail. +5. **Tests + bench** — parity vs PSS branch on small fixtures; benchmark tables. + +## Explicit non-goals (this branch) + +- js_of_ocaml / melange non-PSS. +- Logseq PSS `db.sqlite` compatibility. +- Datalevin-style aid/AVG encoding or dropping `tx` from storage. diff --git a/impl/conn.ml b/impl/conn.ml index cac0400..66a36aa 100644 --- a/impl/conn.ml +++ b/impl/conn.ml @@ -5,7 +5,6 @@ type t = ; mutable listeners : (string * (tx_report -> unit)) list ; mutable next_listener_id : int ; storage : storage option - ; mutable storage_tail : datom list list } type creation_context = @@ -19,33 +18,24 @@ type schema_context = ; with_schema : db -> schema -> db } -type restore_context = - { restore : storage -> db option - ; restore_tail_groups : storage -> datom list list - } +type restore_context = { restore : storage -> db option } type transact_context = { store : ?storage:storage -> db -> unit - ; store_tail : storage -> datom list list -> unit - ; storage_tail_datom_count : datom list list -> int - ; storage_tail_compaction_threshold : int ; transact : tx_meta:tx_meta -> db -> tx_op list -> tx_report } type reset_context = { store : ?storage:storage -> db -> unit ; datoms : db -> datom list + ; snapshot_db : db -> db } type context = { empty_db : ?schema:schema -> ?storage:storage -> unit -> db ; init_db : ?schema:schema -> ?storage:storage -> datom list -> db ; store : ?storage:storage -> db -> unit - ; store_tail : storage -> datom list list -> unit ; restore : storage -> db option - ; restore_tail_groups : storage -> datom list list - ; storage_tail_datom_count : datom list list -> int - ; storage_tail_compaction_threshold : int ; transact : tx_meta:tx_meta -> db -> tx_op list -> tx_report ; datoms : db -> datom list ; with_schema : db -> schema -> db @@ -64,13 +54,13 @@ let tx_meta_without_store_control tx_meta = | _ -> true) tx_meta -let make ?storage ?(storage_tail = []) db = +let make ?storage db = let db = match storage with | None -> db | Some _ -> { db with storage_ref = storage } in - { db; listeners = []; next_listener_id = 0; storage; storage_tail } + { db; listeners = []; next_listener_id = 0; storage } let create (context : creation_context) ?schema ?storage () = let db = context.empty_db ?schema ?storage () in @@ -122,15 +112,13 @@ let reset_schema (context : schema_context) conn schema = conn.db <- db; (match conn.storage with | None -> () - | Some storage -> - context.store ~storage db; - conn.storage_tail <- []); + | Some storage -> context.store ~storage db); db let restore (context : restore_context) storage = match context.restore storage with | None -> None - | Some db -> Some (make ~storage ~storage_tail:(context.restore_tail_groups storage) db) + | Some db -> Some (make ~storage db) let transact (context : transact_context) ?(tx_meta = []) conn tx_data = let skip_store = tx_meta_skips_store tx_meta in @@ -139,21 +127,12 @@ let transact (context : transact_context) ?(tx_meta = []) conn tx_data = if not skip_store then (match conn.storage with | None -> () - | Some storage -> - if report.tx_data <> [] then begin - let tail = conn.storage_tail @ [ report.tx_data ] in - if context.storage_tail_datom_count tail > context.storage_tail_compaction_threshold then begin - context.store ~storage report.db_after; - conn.storage_tail <- [] - end else begin - conn.storage_tail <- tail; - context.store_tail storage conn.storage_tail - end - end); + | Some storage -> context.store ~storage report.db_after); notify_listeners conn report; report let reset (context : reset_context) ?(tx_meta = []) conn db = + let db_before = context.snapshot_db conn.db in let db = match conn.storage with | None -> db @@ -163,12 +142,10 @@ let reset (context : reset_context) ?(tx_meta = []) conn db = List.map (fun datom -> { datom with added = false }) (context.datoms conn.db) @ context.datoms db in - let report = { db_before = conn.db; db_after = db; tx_data; tempids = []; tx_meta } in + let report = { db_before; db_after = db; tx_data; tempids = []; tx_meta } in conn.db <- db; (match conn.storage with | None -> () - | Some storage -> - context.store ~storage db; - conn.storage_tail <- []); + | Some storage -> context.store ~storage db); notify_listeners conn report; db diff --git a/impl/conn.mli b/impl/conn.mli index f7cda87..04a6201 100644 --- a/impl/conn.mli +++ b/impl/conn.mli @@ -13,33 +13,24 @@ type schema_context = ; with_schema : db -> schema -> db } -type restore_context = - { restore : storage -> db option - ; restore_tail_groups : storage -> datom list list - } +type restore_context = { restore : storage -> db option } type transact_context = { store : ?storage:storage -> db -> unit - ; store_tail : storage -> datom list list -> unit - ; storage_tail_datom_count : datom list list -> int - ; storage_tail_compaction_threshold : int ; transact : tx_meta:tx_meta -> db -> tx_op list -> tx_report } type reset_context = { store : ?storage:storage -> db -> unit ; datoms : db -> datom list + ; snapshot_db : db -> db } type context = { empty_db : ?schema:schema -> ?storage:storage -> unit -> db ; init_db : ?schema:schema -> ?storage:storage -> datom list -> db ; store : ?storage:storage -> db -> unit - ; store_tail : storage -> datom list list -> unit ; restore : storage -> db option - ; restore_tail_groups : storage -> datom list list - ; storage_tail_datom_count : datom list list -> int - ; storage_tail_compaction_threshold : int ; transact : tx_meta:tx_meta -> db -> tx_op list -> tx_report ; datoms : db -> datom list ; with_schema : db -> schema -> db diff --git a/impl/datascript.ml b/impl/datascript.ml index 4f7976c..8953437 100644 --- a/impl/datascript.ml +++ b/impl/datascript.ml @@ -29,7 +29,7 @@ module Serialize = Serialize module Storage = Storage module Util = Util module Upsert = Upsert -module PSet = Persistent_sorted_set +module Index = Index let validate_entity_id = Db_impl.validate_entity_id @@ -53,14 +53,19 @@ let normalize_datom_for_schema = Db_impl.normalize_datom_for_schema let refresh_db_indexes = Db_impl.refresh_indexes let refresh_db_indexes_with_added_datoms = Db_impl.refresh_indexes_with_added_datoms let refresh_db_indexes_with_tx_data = Db_impl.refresh_indexes_with_tx_data +let snapshot_db = Db_impl.snapshot_db let empty_db ?(schema = []) ?storage () = Db_impl.empty_db db_core_context ~schema ?storage () let empty db = Db_impl.empty db_core_context db +let warm_query_parser = ref (fun _db -> ()) + let init_db ?(schema = []) ?storage datoms = - Db_impl.init_db db_core_context ~schema ?storage datoms + let db = Db_impl.init_db db_core_context ~schema ?storage datoms in + !warm_query_parser db; + db let visible_datoms = Db_impl.visible_datoms @@ -77,7 +82,7 @@ let serialize_context : Serialize.context = { next_db_uid ; validate_schema ; normalize_datom_for_schema - ; refresh_db_indexes + ; with_datoms = Db_impl.with_datoms } let from_serializable snapshot = @@ -87,11 +92,6 @@ let store ?storage db = Storage.store ?storage db let memory_storage = Storage.memory_storage -let file_storage = Storage.file_storage -let store_tail = Storage.store_tail -let storage_tail_compaction_threshold = Storage.tail_compaction_threshold -let storage_tail_datom_count = Storage.tail_datom_count -let restore_tail_groups = Storage.restore_tail_groups let storage_addresses = Storage.storage_addresses let storage = Storage.storage let addresses = Storage.addresses @@ -245,15 +245,17 @@ let find_avet_exact db attr value = else if left == bound then -compare_prefix right left else Util.compare_datom Avet left right in - match - PSet.slice ~from_:bound ~to_:bound ~cmp db.avet_index - @ List.filter + match Index.find_first_slice ~from_:bound ~to_:bound ~cmp db.avet_index with + | Some datom when datom.a = attr && value_equal datom.v value -> Some datom + | _ -> ( + match + List.filter (fun datom -> datom.a = attr && value_equal datom.v value) (Option.value (Hashtbl.find_opt db.duplicate_avet_by_attr attr) ~default:[]) - |> List.sort (Util.compare_datom Avet) - with - | datom :: _ -> Some datom - | [] -> None + |> List.sort (Util.compare_datom Avet) + with + | datom :: _ -> Some datom + | [] -> None) let find_eavt_exact db entity_id attr value = let bound = datom ~e:entity_id ~a:attr ~v:value () in @@ -269,15 +271,17 @@ let find_eavt_exact db entity_id attr value = else if left == bound then -compare_prefix right left else Util.compare_datom Eavt left right in - match - PSet.slice ~from_:bound ~to_:bound ~cmp db.eavt_index - @ List.filter + match Index.find_first_slice ~from_:bound ~to_:bound ~cmp db.eavt_index with + | Some datom when datom.e = entity_id && datom.a = attr && value_equal datom.v value -> Some datom + | _ -> ( + match + List.filter (fun datom -> datom.e = entity_id && datom.a = attr && value_equal datom.v value) (Option.value (Hashtbl.find_opt db.duplicate_eavt_by_entity entity_id) ~default:[]) - |> List.sort (Util.compare_datom Eavt) - with - | datom :: _ -> Some datom - | [] -> None + |> List.sort (Util.compare_datom Eavt) + with + | datom :: _ -> Some datom + | [] -> None) let rec coerce_tuple_lookup_value_db db attr value = match schema_attr db attr, value with @@ -736,61 +740,13 @@ let db_with tx_ops db = let db_after, _, _ = apply_tx tx_ops db in db_after -let apply_tail_group db group = - List.iter - (fun datom -> - if datom.added && is_unique db datom.a then - match Db_access_impl.find_datom db Avet ~a:datom.a ~v:datom.v () with - | Some existing when existing.e <> datom.e -> - invalid_arg "tail group conflicts with an existing unique value" - | Some _ | None -> ()) - group; - let group = - List.fold_left - (fun tx_data datom -> - if datom.added && cardinality db datom.a = One then - let existing = - Db_access_impl.datoms db Eavt ~e:datom.e ~a:datom.a () - |> Seq.filter (fun existing -> not (value_equal existing.v datom.v)) - |> Seq.map (fun existing -> - { existing with tx = datom.tx; added = false }) - |> List.of_seq - in - List.rev_append existing (datom :: tx_data) - else - datom :: tx_data) - [] - group - |> List.rev - in - let max_eid = - List.fold_left - (fun max_eid datom -> - let max_eid = - if datom.e <= max_allocatable_entity_id then max max_eid datom.e - else max_eid - in - max_eid_in_value max_eid datom.v) - db.max_eid - group - in - let db = refresh_db_indexes_with_tx_data db group in - { db with max_eid } - -let storage_tail_context : Storage.tail_context = - { apply_group = apply_tail_group } - -let db_with_tail db tail = - Storage.db_with_tail storage_tail_context db tail - -let storage_restore_context : Storage.restore_context = - { next_db_uid; db_with_tail } +let storage_restore_context : Storage.restore_context = { next_db_uid } let restore storage = Storage.restore storage_restore_context storage let restore_conn storage = - let context : Conn.restore_context = { restore; restore_tail_groups } in + let context : Conn.restore_context = { restore } in Conn.restore context storage let tx_meta_skips_store tx_meta = @@ -800,36 +756,27 @@ let tx_meta_skips_store tx_meta = | _ -> false) tx_meta -let persist_transact_tail ~tx_meta db tx_data = - if tx_data <> [] && not (tx_meta_skips_store tx_meta) then +let persist_transact ~tx_meta db = + if not (tx_meta_skips_store tx_meta) then match db.storage_ref with | None -> () - | Some storage -> - let tail = restore_tail_groups storage @ [ tx_data ] in - if storage_tail_datom_count tail > storage_tail_compaction_threshold then - store ~storage db - else - store_tail storage tail + | Some storage -> store ~storage db let transact_report ?(tx_meta = []) db tx_ops = + let db_before = snapshot_db db in let db_after, tempids, tx_data = apply_tx tx_ops db in - { db_before = db; db_after; tx_data; tempids; tx_meta } + { db_before; db_after; tx_data; tempids; tx_meta } let transact ?(tx_meta = []) db tx_ops = let report = transact_report ~tx_meta db tx_ops in - persist_transact_tail ~tx_meta report.db_after report.tx_data; + persist_transact ~tx_meta report.db_after; report let with_tx ?tx_meta db tx_ops = transact ?tx_meta db tx_ops let transact_conn ?(tx_meta = []) conn tx_data = let context : Conn.transact_context = - { store - ; store_tail - ; storage_tail_datom_count - ; storage_tail_compaction_threshold - ; transact = (fun ~tx_meta db tx_data -> transact_report ~tx_meta db tx_data) - } + { store; transact = (fun ~tx_meta db tx_data -> transact_report ~tx_meta db tx_data) } in Conn.transact context ~tx_meta conn tx_data @@ -877,7 +824,10 @@ let squuid_time_millis = Db_impl.squuid_time_millis let reset_conn ?(tx_meta = []) conn db = let context : Conn.reset_context = - { store; datoms = (fun db -> datoms_list db Eavt ()) } + { store + ; datoms = (fun db -> datoms_list db Eavt ()) + ; snapshot_db + } in Conn.reset context ~tx_meta conn db @@ -1139,11 +1089,26 @@ let datoms_by_attr_value db attr value = | None -> false in if Option.is_none ident_entity_value && query_value_uses_avet value && query_attr_uses_avet db attr then - datoms_list db Avet ~a:attr ~v:value () + Db_access_impl.avet_datoms_by_value db attr value else datoms_list db Aevt ~a:attr () |> List.filter datom_value_matches +let entity_ids_by_attr_value db attr value = + match resolve_query_value_for_attr db attr value with + | None -> Some [] + | Some value -> + let value = + if is_tuple_attr db attr then + coerce_tuple_lookup_value_db db attr value + else + normalize_value value + in + if query_value_uses_avet value && query_attr_uses_avet db attr then + Db_access_impl.avet_entity_ids_by_attr_value db attr value + else + None + let pattern_value_needs_attr_resolution db attr value = is_tuple_attr db attr || @@ -1163,24 +1128,24 @@ let primary_attr_datoms db index attr = else if left == bound then -compare_prefix right left else Util.compare_datom index left right in - PSet.slice ~from_:bound ~to_:bound ~cmp index_set + Index.slice ~from_:bound ~to_:bound ~cmp index_set in match index with | Aevt -> (match Hashtbl.find_opt db.aevt_by_attr attr with - | Some datoms -> datoms + | Some datoms -> Array.to_list datoms | None -> let datoms = attr_prefix_datoms Aevt db.aevt_index in - Hashtbl.replace db.aevt_by_attr attr datoms; + Hashtbl.replace db.aevt_by_attr attr (Array.of_list datoms); datoms) | Avet -> (match Hashtbl.find_opt db.avet_by_attr attr with - | Some datoms -> datoms + | Some datoms -> Array.to_list datoms | None -> let datoms = attr_prefix_datoms Avet db.avet_index in - Hashtbl.replace db.avet_by_attr attr datoms; + Hashtbl.replace db.avet_by_attr attr (Array.of_list datoms); datoms) - | Eavt -> PSet.to_list db.eavt_index + | Eavt -> Index.to_list db.eavt_index let primary_attr_datoms_seq db index ?e ~a ?v ?tx () = let datoms = primary_attr_datoms db index a in @@ -1203,9 +1168,12 @@ let primary_attr_datoms_seq db index ?e ~a ?v ?tx () = | None -> true) let query_attr_datoms_seq db index ?e ~a ?v ?tx () = - match db.duplicate_datoms with - | [] -> datoms db index ?e ~a ?v ?tx () - | _ -> primary_attr_datoms_seq db index ?e ~a ?v ?tx () + let attr = a in + match db.duplicate_datoms, index, e, v, tx with + | [], Avet, None, Some value, None -> + List.to_seq (Db_access_impl.avet_datoms_by_value db attr value) + | [], _, _, _, _ -> datoms db index ?e ~a:attr ?v ?tx () + | _ -> primary_attr_datoms_seq db index ?e ~a:attr ?v ?tx () let pattern_datoms db e_term a_term v_term tx_term = let e = query_entity_id_term db e_term in @@ -1411,6 +1379,10 @@ module Query_where_impl = Query_where.Make (struct let is_ref_attr = is_ref_attr let cardinality_one db attr = cardinality db attr = One let normalize_value = normalize_value + let datoms_by_attr_value = datoms_by_attr_value + let entity_ids_by_attr_value = entity_ids_by_attr_value + let query_attr_uses_avet = query_attr_uses_avet + let query_value_uses_avet = query_value_uses_avet end) let eval_clauses = Query_where_impl.eval_clauses @@ -1448,7 +1420,22 @@ let parse_with = Parser_impl.parse_with let parse_query_return form = Parser_impl.parse_query_return parser_query_context form let parse_query_return_map form = Parser_impl.parse_query_return_map parser_query_context form let parse_query form = Parser_impl.parse_query parser_query_context form -let parse_query_string input = Parser_impl.parse_query_string parser_query_context input + +let query_string_cache : (string, query) Hashtbl.t = Hashtbl.create 32 + +let parse_query_string_uncached input = + Parser_impl.parse_query_string parser_query_context input + +let cached_query_string input = + match Hashtbl.find_opt query_string_cache input with + | Some query -> query + | None -> + let query = parse_query_string_uncached input in + Hashtbl.replace query_string_cache input query; + query + +let parse_query_string input = cached_query_string input + let parse_query_string_with_pull_context ?default_pull_db ?pull_db_for_source input = Parser_impl.parse_query_string_with_pull_context parser_query_context ?default_pull_db ?pull_db_for_source input let parse_query_return_string input = Parser_impl.parse_query_return_string parser_query_context input @@ -1723,9 +1710,24 @@ module Query = struct if duplicate_value_var then None else + match find_vars, value_var_attrs, constant_patterns with + | [ var ], [], [ (attr, value) ] when var = e_var -> ( + match entity_ids_by_attr_value db attr value with + | Some [] -> Some [] + | Some entity_ids -> + Some (List.map (fun entity_id -> [ Result_entity entity_id ]) entity_ids) + | None -> None) + | _ -> None + |> function + | Some rows -> Some rows + | None -> let constant_datoms = constant_patterns - |> List.map (fun (attr, value) -> attr, datoms_by_attr_value db attr value) + |> List.map (fun (attr, value) -> + match entity_ids_by_attr_value db attr value with + | Some entity_ids -> + attr, List.map (fun e -> datom ~e ~a:attr ~v:value ()) entity_ids + | None -> attr, datoms_by_attr_value db attr value) in if List.exists (fun (_, datoms) -> datoms = []) constant_datoms then Some [] @@ -1807,15 +1809,6 @@ module Query = struct | Some rows -> rows | None -> Query_impl.q query_context ?inputs db query - let query_string_cache : (string, query) Hashtbl.t = Hashtbl.create 32 - let cached_query_string input = - match Hashtbl.find_opt query_string_cache input with - | Some query -> query - | None -> - let query = parse_query_string input in - Hashtbl.replace query_string_cache input query; - query - let q_string ?inputs db input = if string_includes input "pull" then q ?inputs db (parse_query_string_with_pull_context ~default_pull_db:db input) @@ -3112,6 +3105,18 @@ end let q = Query.q let q_string = Query.q_string + +let () = + warm_query_parser := + (let warmed = ref false in + fun db -> + if not !warmed then ( + warmed := true; + ignore (read_edn "1"); + ignore (parse_query_string_uncached "[:find ?e :where [?e :name \"warmup\"]]"); + ignore (parse_query_string_uncached "[:find ?e :where [?e :age 1]]"); + ignore (q_string db "[:find ?e :where [?e :name \"warmup\"]]"))) + let q_with = Query.q_with let q_with_string = Query.q_with_string let q_sources = Query.q_sources diff --git a/impl/datascript.mli b/impl/datascript.mli index fbf1e4f..7fa533b 100644 --- a/impl/datascript.mli +++ b/impl/datascript.mli @@ -88,22 +88,17 @@ module Conn : sig ; with_schema : db -> schema -> db } - type restore_context = - { restore : storage -> db option - ; restore_tail_groups : storage -> datom list list - } + type restore_context = { restore : storage -> db option } type transact_context = { store : ?storage:storage -> db -> unit - ; store_tail : storage -> datom list list -> unit - ; storage_tail_datom_count : datom list list -> int - ; storage_tail_compaction_threshold : int ; transact : tx_meta:tx_meta -> db -> tx_op list -> tx_report } type reset_context = { store : ?storage:storage -> db -> unit ; datoms : db -> datom list + ; snapshot_db : db -> db } val create : creation_context -> ?schema:schema -> ?storage:storage -> unit -> t @@ -223,7 +218,7 @@ module Serialize : sig { next_db_uid : unit -> int ; validate_schema : schema -> schema ; normalize_datom_for_schema : schema -> datom -> datom - ; refresh_db_indexes : db -> db + ; with_datoms : db -> datom list -> db } val serializable : db -> serializable_db @@ -231,26 +226,11 @@ module Serialize : sig end module Storage : sig - type tail_context = - { apply_group : db -> datom list -> db - } - - type restore_context = - { next_db_uid : unit -> int - ; db_with_tail : db -> datom list list -> db - } + type restore_context = { next_db_uid : unit -> int } - val root_address : storage_address - val tail_address : storage_address val memory_storage : unit -> storage - val file_storage : string -> storage val store : ?storage:storage -> db -> unit - val store_tail : storage -> datom list list -> unit - val tail_compaction_threshold : int - val tail_datom_count : datom list list -> int val restore_root_snapshot : storage -> serializable_db option - val restore_tail_groups : storage -> datom list list - val db_with_tail : tail_context -> db -> datom list list -> db val restore : restore_context -> storage -> db option val storage_addresses : storage -> storage_address list val storage : db -> storage option @@ -404,11 +384,8 @@ val serializable : db -> serializable_db val from_serializable : serializable_db -> db val db_from_reader_string : string -> db val memory_storage : unit -> storage -val file_storage : string -> storage val store : ?storage:storage -> db -> unit -val store_tail : storage -> datom list list -> unit val restore : storage -> db option -val db_with_tail : db -> datom list list -> db val storage : db -> storage option val addresses : db list -> storage_address list val settings : db -> (attr * value) list diff --git a/impl/db.ml b/impl/db.ml index 87579d1..c24023a 100644 --- a/impl/db.ml +++ b/impl/db.ml @@ -1,6 +1,6 @@ open Datascript_types -module PSet = Persistent_sorted_set +module Index = Index let tx0 = 0x20000000 @@ -62,14 +62,7 @@ let normalize_datom_for_schema schema d = ignore schema; Util.normalize_datom_value d -let empty_index index = - PSet.empty_by ~cmp:(Util.compare_datom index) () - -let build_index index datoms = - let cmp = Util.compare_datom index in - let items = Array.of_list datoms in - Array.sort cmp items; - PSet.of_sorted_array_by ~cmp items +let empty_index index lmdb = Index.empty index lmdb let duplicate_datoms datoms = let datoms = List.sort (Util.compare_datom Eavt) datoms in @@ -83,6 +76,18 @@ let duplicate_datoms datoms = in loop None [] datoms +let primary_datoms index datoms = + let datoms = List.sort (Util.compare_datom index) datoms in + let rec loop previous primary = function + | [] -> List.rev primary + | datom :: rest -> + (match previous with + | Some previous when Util.compare_datom index previous datom = 0 -> + loop (Some datom) primary rest + | _ -> loop (Some datom) (datom :: primary) rest) + in + loop None [] datoms + let duplicate_eavt_by_entity duplicate_datoms = let table = Hashtbl.create 1024 in List.iter @@ -103,32 +108,64 @@ let duplicate_datoms_by_attr duplicate_datoms = Hashtbl.iter (fun attr datoms -> Hashtbl.replace table attr (List.rev datoms)) table; table -let build_avet_index schema datoms = - datoms - |> List.filter (fun d -> Schema.schema_attr_is_avet_accessible schema d.a) - |> build_index Avet +let invalidate_attr_tables db = + if Hashtbl.length db.aevt_by_attr = 0 && Hashtbl.length db.avet_by_attr = 0 then + db + else + { db with + aevt_by_attr = Hashtbl.create 0 + ; avet_by_attr = Hashtbl.create 0 + ; avet_entities_by_attr_value = Hashtbl.create 0 + } + +let lmdb_of_db db = + try Index.lmdb_of (Index.db_of db.eavt_index) + with Invalid_argument _ -> + let lmdb, _ = Index.create_lmdb db.storage_ref in + lmdb + +let group_sorted_datoms_by_attr datoms = + let table = Hashtbl.create 32 in + let rec flush attr group = function + | [] -> Hashtbl.replace table attr (Array.of_list (List.rev group)) + | datom :: rest when datom.a = attr -> + flush attr (datom :: group) rest + | datom :: rest -> + Hashtbl.replace table attr (Array.of_list (List.rev group)); + flush datom.a [ datom ] rest + in + (match datoms with + | [] -> () + | datom :: rest -> flush datom.a [ datom ] rest); + table -let datoms_by_attr datoms = - let table = Hashtbl.create 1024 in +let index_avet_entities_by_attr_value avet_sorted = + let table = Hashtbl.create 256 in List.iter (fun datom -> - let existing = Option.value (Hashtbl.find_opt table datom.a) ~default:[] in - Hashtbl.replace table datom.a (datom :: existing)) - datoms; - Hashtbl.iter (fun attr datoms -> Hashtbl.replace table attr (List.rev datoms)) table; + let key = (datom.a, datom.v) in + let existing = Option.value (Hashtbl.find_opt table key) ~default:[] in + Hashtbl.replace table key (datom.e :: existing)) + avet_sorted; + Hashtbl.iter (fun key entity_ids -> Hashtbl.replace table key (List.rev entity_ids)) table; table -let invalidate_attr_tables db = - if Hashtbl.length db.aevt_by_attr = 0 && Hashtbl.length db.avet_by_attr = 0 then - db - else - { db with aevt_by_attr = Hashtbl.create 0; avet_by_attr = Hashtbl.create 0 } +let datoms_of_avet_entities attr value entity_ids = + List.map (fun e -> { e; a = attr; v = value; tx = tx0; added = true }) entity_ids let set_indexes_from_datoms db datoms = - let eavt_index = build_index Eavt datoms in - let aevt_index = build_index Aevt datoms in - let avet_index = build_avet_index db.schema datoms in + let lmdb = lmdb_of_db db in let duplicate_datoms = duplicate_datoms datoms in + let eavt_datoms = primary_datoms Eavt datoms in + let aevt_sorted = List.sort (Util.compare_datom Aevt) eavt_datoms in + let avet_sorted = + eavt_datoms + |> List.filter (fun d -> Schema.schema_attr_is_avet_accessible db.schema d.a) + |> List.sort (Util.compare_datom Avet) + in + let eavt_index = Index.of_bulk Eavt eavt_datoms lmdb in + let aevt_index = Index.of_bulk Aevt aevt_sorted lmdb in + let avet_index = Index.of_bulk Avet avet_sorted lmdb in let duplicate_aevt_datoms = List.sort (Util.compare_datom Aevt) duplicate_datoms in let duplicate_avet_datoms = duplicate_datoms @@ -143,8 +180,9 @@ let set_indexes_from_datoms db datoms = eavt_index ; aevt_index ; avet_index - ; aevt_by_attr = datoms_by_attr (PSet.to_list aevt_index) - ; avet_by_attr = datoms_by_attr (PSet.to_list avet_index) + ; aevt_by_attr = group_sorted_datoms_by_attr aevt_sorted + ; avet_by_attr = group_sorted_datoms_by_attr avet_sorted + ; avet_entities_by_attr_value = index_avet_entities_by_attr_value avet_sorted ; duplicate_datoms ; duplicate_aevt_datoms ; duplicate_avet_datoms @@ -155,7 +193,7 @@ let set_indexes_from_datoms db datoms = } let eavt_datoms db = - PSet.to_list db.eavt_index @ db.duplicate_datoms |> List.sort (Util.compare_datom Eavt) + Index.to_list db.eavt_index @ db.duplicate_datoms |> List.sort (Util.compare_datom Eavt) let refresh_indexes db = set_indexes_from_datoms db (eavt_datoms db) @@ -163,7 +201,7 @@ let refresh_indexes db = let add_datoms_to_index include_datom datoms index_set = List.fold_left (fun index_set datom -> - if include_datom datom then PSet.add datom index_set else index_set) + if include_datom datom then Index.add datom index_set else index_set) index_set datoms @@ -206,17 +244,20 @@ let find_active_datom_by_fact db datom = Option.value (Hashtbl.find_opt db.duplicate_eavt_by_entity datom.e) ~default:[] |> List.filter (fun active -> active.a = datom.a && value_equal active.v datom.v) in - match PSet.slice ~from_:bound ~to_:bound ~cmp db.eavt_index @ duplicate_matches with - | [] -> None - | matches -> Some (matches |> List.sort (Util.compare_datom Eavt) |> List.hd) + match Index.find_first_slice ~from_:bound ~to_:bound ~cmp db.eavt_index with + | Some active when active.e = datom.e && active.a = datom.a && value_equal active.v datom.v -> Some active + | _ -> ( + match duplicate_matches with + | [] -> None + | matches -> Some (matches |> List.sort (Util.compare_datom Eavt) |> List.hd)) let add_datom_to_indexes db datom = { db with - eavt_index = PSet.add datom db.eavt_index - ; aevt_index = PSet.add datom db.aevt_index + eavt_index = Index.add datom db.eavt_index + ; aevt_index = Index.add datom db.aevt_index ; avet_index = if Schema.schema_attr_is_avet_accessible db.schema datom.a then - PSet.add datom db.avet_index + Index.add datom db.avet_index else db.avet_index ; max_datom_e = max db.max_datom_e datom.e @@ -233,27 +274,41 @@ let refresh_indexes_with_tx_data db tx_data = | None -> db | Some active -> { db with - eavt_index = PSet.remove active db.eavt_index - ; aevt_index = PSet.remove active db.aevt_index - ; avet_index = PSet.remove active db.avet_index + eavt_index = Index.remove active db.eavt_index + ; aevt_index = Index.remove active db.aevt_index + ; avet_index = Index.remove active db.avet_index }) db tx_data in invalidate_attr_tables db +let snapshot_db db = + { db with + eavt_index = Index.copy db.eavt_index + ; aevt_index = Index.copy db.aevt_index + ; avet_index = Index.copy db.avet_index + } + let with_datoms db datoms = set_indexes_from_datoms db datoms +let storage_ref_of ?storage auto_storage_ref = + match storage with + | Some attached_storage -> Some attached_storage + | None -> auto_storage_ref + let empty_db context ?(schema = []) ?storage () = let schema = Schema.validate_schema schema in + let lmdb, auto_storage_ref = Index.create_lmdb None in { db_uid = context.next_db_uid () ; schema - ; eavt_index = empty_index Eavt - ; aevt_index = empty_index Aevt - ; avet_index = empty_index Avet + ; eavt_index = empty_index Eavt lmdb + ; aevt_index = empty_index Aevt lmdb + ; avet_index = empty_index Avet lmdb ; aevt_by_attr = Hashtbl.create 0 ; avet_by_attr = Hashtbl.create 0 + ; avet_entities_by_attr_value = Hashtbl.create 0 ; duplicate_datoms = [] ; duplicate_aevt_datoms = [] ; duplicate_avet_datoms = [] @@ -264,7 +319,7 @@ let empty_db context ?(schema = []) ?storage () = ; max_datom_e = 0 ; max_tx = tx0 ; filter_pred = None - ; storage_ref = storage + ; storage_ref = storage_ref_of ?storage auto_storage_ref ; tx_fns = [] } @@ -277,13 +332,15 @@ let init_db context ?(schema = []) ?storage datoms = List.fold_left (fun max_eid d -> max_eid_in_value (max_eid_with_entity_id max_eid d.e) d.v) 0 datoms in let max_tx = List.fold_left (fun max_tx d -> max max_tx d.tx) tx0 datoms in + let lmdb, auto_storage_ref = Index.create_lmdb None in { db_uid = context.next_db_uid () ; schema - ; eavt_index = empty_index Eavt - ; aevt_index = empty_index Aevt - ; avet_index = empty_index Avet + ; eavt_index = empty_index Eavt lmdb + ; aevt_index = empty_index Aevt lmdb + ; avet_index = empty_index Avet lmdb ; aevt_by_attr = Hashtbl.create 0 ; avet_by_attr = Hashtbl.create 0 + ; avet_entities_by_attr_value = Hashtbl.create 0 ; duplicate_datoms = [] ; duplicate_aevt_datoms = [] ; duplicate_avet_datoms = [] @@ -294,7 +351,7 @@ let init_db context ?(schema = []) ?storage datoms = ; max_datom_e = 0 ; max_tx ; filter_pred = None - ; storage_ref = storage + ; storage_ref = storage_ref_of ?storage auto_storage_ref ; tx_fns = [] } |> fun db -> with_datoms db datoms @@ -390,32 +447,28 @@ let duplicate_attr_datoms db index attr = | Eavt -> duplicate_index_datoms db index let primary_attr_datoms db index attr = - let attr_prefix_datoms index index_set = - let bound = datom ~e:0 ~a:attr ~v:Nil () in - let compare_prefix left right = compare left.a right.a in - let cmp left right = - if right == bound then compare_prefix left right - else if left == bound then -compare_prefix right left - else Util.compare_datom index left right - in - PSet.slice ~from_:bound ~to_:bound ~cmp index_set + let attr_prefix_datoms _index index_set = + Index.fold_attr_prefix (fun acc datom -> datom :: acc) [] index_set attr |> List.rev + in + let attr_prefix_array index index_set = + Array.of_list (attr_prefix_datoms index index_set) in match index with | Aevt -> (match Hashtbl.find_opt db.aevt_by_attr attr with - | Some datoms -> datoms + | Some datoms -> Array.to_list datoms | None -> - let datoms = attr_prefix_datoms Aevt db.aevt_index in + let datoms = attr_prefix_array Aevt db.aevt_index in Hashtbl.replace db.aevt_by_attr attr datoms; - datoms) + Array.to_list datoms) | Avet -> (match Hashtbl.find_opt db.avet_by_attr attr with - | Some datoms -> datoms + | Some datoms -> Array.to_list datoms | None -> - let datoms = attr_prefix_datoms Avet db.avet_index in + let datoms = attr_prefix_array Avet db.avet_index in Hashtbl.replace db.avet_by_attr attr datoms; - datoms) - | Eavt -> PSet.to_list db.eavt_index + Array.to_list datoms) + | Eavt -> Index.to_list db.eavt_index let duplicate_prefix_datoms db index e a = match index, e, a with @@ -423,18 +476,8 @@ let duplicate_prefix_datoms db index e a = | (Aevt | Avet), _, Some attr -> duplicate_attr_datoms db index attr | _ -> duplicate_index_datoms db index -let exact_sorted_slice cmp bound datoms = - let rec drop_before = function - | datom :: rest when cmp datom bound < 0 -> drop_before rest - | datoms -> take_equal [] datoms - and take_equal acc = function - | datom :: rest when cmp datom bound = 0 -> take_equal (datom :: acc) rest - | _ -> List.rev acc - in - drop_before datoms - let raw_index_datoms_list db index = - merge_sorted_datoms index (stored_index db index |> PSet.to_list) (duplicate_index_datoms db index) + merge_sorted_datoms index (stored_index db index |> Index.to_list) (duplicate_index_datoms db index) let visible_index_datoms db index = let datoms = raw_index_datoms_list db index in @@ -444,14 +487,14 @@ let visible_index_datoms db index = let index_datoms_seq db index = match db.duplicate_datoms with - | [] -> stored_index db index |> PSet.seq |> PSet.to_seq + | [] -> stored_index db index |> Index.seq |> Index.to_seq | _ -> raw_index_datoms_list db index |> List.to_seq let reverse_index_datoms_seq db index = match db.duplicate_datoms with - | [] -> stored_index db index |> PSet.rslice_seq |> PSet.to_seq + | [] -> stored_index db index |> Index.rslice_seq |> Index.to_seq | _ -> - let indexed = stored_index db index |> PSet.rslice_seq |> PSet.to_seq in + let indexed = stored_index db index |> Index.rslice_seq |> Index.to_seq in let duplicates = duplicate_index_datoms db index |> List.rev |> List.to_seq in merge_sorted_datom_seqs (fun left right -> Util.compare_datom index right left) @@ -535,6 +578,61 @@ let compare_bound_fields context fields left right = function (compare_bound_e fields left right) (compare_bound_tx fields left right) +let array_attr_value_slice context index bound bound_fields arr = + let prefix left right = compare_bound_fields context bound_fields left right index in + let len = Array.length arr in + let rec lower lo hi = + if lo >= hi then lo + else + let mid = (lo + hi) / 2 in + if prefix arr.(mid) bound < 0 then lower (mid + 1) hi else lower lo mid + in + let start = lower 0 len in + let rec upper index = + if index >= len || prefix arr.(index) bound > 0 then index else upper (index + 1) + in + let stop = upper start in + if start >= stop then [] + else Array.sub arr start (stop - start) |> Array.to_list + +let array_attr_value_seq context index bound bound_fields arr = + let prefix left right = compare_bound_fields context bound_fields left right index in + let len = Array.length arr in + let rec lower lo hi = + if lo >= hi then lo + else + let mid = (lo + hi) / 2 in + if prefix arr.(mid) bound < 0 then lower (mid + 1) hi else lower lo mid + in + let start = lower 0 len in + let rec upper index = + if index >= len || prefix arr.(index) bound > 0 then index else upper (index + 1) + in + let stop = upper start in + let rec loop index () = + if index >= stop then Seq.Nil else Seq.Cons (arr.(index), loop (index + 1)) + in + loop start + +let array_exact_prefix_slice cmp bound arr = + let len = Array.length arr in + let rec lower lo hi = + if lo >= hi then lo + else + let mid = (lo + hi) / 2 in + if cmp arr.(mid) bound < 0 then lower (mid + 1) hi else lower lo mid + in + let start = lower 0 len in + let rec upper index = + if index >= len || cmp arr.(index) bound <> 0 then index else upper (index + 1) + in + let stop = upper start in + if start >= stop then [] + else Array.sub arr start (stop - start) |> Array.to_list + +let exact_sorted_slice cmp bound datoms = + array_exact_prefix_slice cmp bound (Array.of_list datoms) + let slice_cmp context index from_bound from_fields to_bound to_fields left right = if right == from_bound then compare_bound_fields context from_fields left right index @@ -625,6 +723,41 @@ let exact_prefix_bound index e a v tx = Some (bound_datom ~e ~a ~v ~tx (), fields ~e:true ~a:true ~v:true ~tx:true ()) | _ -> None) +let avet_entity_ids_by_attr_value context db attr value = + match Hashtbl.find_opt db.avet_entities_by_attr_value (attr, value) with + | Some entity_ids -> Some entity_ids + | None -> ( + match Hashtbl.find_opt db.avet_by_attr attr with + | Some datoms -> + let bound = bound_datom ~a:attr ~v:value () in + let bound_fields = fields ~a:true ~v:true () in + Some + (array_attr_value_slice context Avet bound bound_fields datoms + |> List.map (fun datom -> datom.e)) + | None -> None) + +let avet_datoms_by_value context db attr value = + let bound = bound_datom ~a:attr ~v:value () in + let bound_fields = fields ~a:true ~v:true () in + match Hashtbl.find_opt db.avet_entities_by_attr_value (attr, value) with + | Some entity_ids -> datoms_of_avet_entities attr value entity_ids + | None -> ( + match Hashtbl.find_opt db.avet_by_attr attr with + | Some datoms -> array_attr_value_slice context Avet bound bound_fields datoms + | None -> + let cmp = exact_prefix_slice_cmp context Avet bound bound_fields in + Index.slice_seq ~from_:bound ~to_:bound ~cmp (stored_index db Avet) + |> Index.seq_to_list) + +let avet_datoms_by_value_seq context db attr value = + let bound = bound_datom ~a:attr ~v:value () in + let bound_fields = fields ~a:true ~v:true () in + match Hashtbl.find_opt db.avet_by_attr attr with + | Some datoms -> array_attr_value_seq context Avet bound bound_fields datoms + | None -> + let cmp = exact_prefix_slice_cmp context Avet bound bound_fields in + Index.slice_seq ~from_:bound ~to_:bound ~cmp (stored_index db Avet) |> Index.to_seq + let exact_prefix_datoms context db index e a v tx = match exact_prefix_bound index e a v tx with | None -> None @@ -642,10 +775,13 @@ let exact_prefix_datoms context db index e a v tx = let duplicates = duplicate_prefix_datoms db index e a |> exact_sorted_slice cmp bound in Some (merge_sorted_datom_seqs (Util.compare_datom index) (List.to_seq indexed) (List.to_seq duplicates)) | _ -> - (match db.duplicate_datoms with - | [] -> Some (PSet.slice_seq ~from_:bound ~to_:bound ~cmp (stored_index db index) |> PSet.to_seq) + (match db.duplicate_datoms, index, e, a, v, tx with + | [], Avet, None, Some _, Some _, None -> + Some (avet_datoms_by_value_seq context db (Option.get a) (Option.get v)) + | [], _, _, _, _, _ -> + Some (Index.slice_seq ~from_:bound ~to_:bound ~cmp (stored_index db index) |> Index.to_seq) | _ -> - let indexed = PSet.slice_seq ~from_:bound ~to_:bound ~cmp (stored_index db index) |> PSet.to_seq in + let indexed = Index.slice_seq ~from_:bound ~to_:bound ~cmp (stored_index db index) |> Index.to_seq in let duplicates = duplicate_prefix_datoms db index e a |> exact_sorted_slice cmp bound in Some (merge_sorted_datom_seqs (Util.compare_datom index) indexed (List.to_seq duplicates))))) @@ -654,11 +790,20 @@ let exact_prefix_datoms_list context db index e a v tx = | None -> None | Some (bound, bound_fields) -> let cmp = exact_prefix_slice_cmp context index bound bound_fields in + let exact_attr_prefix = + match index, e, a, v, tx with + | Aevt, None, Some _, None, None -> true + | _ -> false + in (match db.duplicate_datoms with | [] -> Some - (PSet.slice_seq ~from_:bound ~to_:bound ~cmp (stored_index db index) - |> PSet.seq_to_list) + (match index, a, v, exact_attr_prefix with + | Avet, Some attr, Some value, false -> avet_datoms_by_value context db attr value + | (Aevt | Avet), Some attr, None, true -> primary_attr_datoms db index attr + | _ -> + Index.slice_seq ~from_:bound ~to_:bound ~cmp (stored_index db index) + |> Index.seq_to_list) | _ -> exact_prefix_datoms context db index e a v tx |> Option.map List.of_seq) @@ -674,7 +819,7 @@ let lower_prefix_datoms context db index e a v tx = primary_attr_datoms db index attr |> List.filter (fun datom -> cmp datom bound >= 0) |> List.to_seq - | _ -> PSet.slice_seq ~from_:bound ~cmp (stored_index db index) |> PSet.to_seq + | _ -> Index.slice_seq ~from_:bound ~cmp (stored_index db index) |> Index.to_seq in (match db.duplicate_datoms with | [] -> Some indexed @@ -694,7 +839,7 @@ let reverse_upper_prefix_datoms context db index e a v tx = |> List.filter (fun datom -> cmp datom bound <= 0) |> List.rev |> List.to_seq - | _ -> PSet.rslice_seq ~from_:bound ~cmp (stored_index db index) |> PSet.to_seq + | _ -> Index.rslice_seq ~from_:bound ~cmp (stored_index db index) |> Index.to_seq in (match db.duplicate_datoms with | [] -> Some indexed @@ -741,7 +886,7 @@ let avet_range_datoms context db attr start stop = match db.duplicate_datoms with | [] -> let cmp = slice_cmp context Avet from_bound from_fields to_bound to_fields in - PSet.slice_seq ~from_:from_bound ~to_:to_bound ~cmp db.avet_index |> PSet.to_seq + Index.slice_seq ~from_:from_bound ~to_:to_bound ~cmp db.avet_index |> Index.to_seq | _ -> primary_attr_datoms db Avet attr |> List.filter (fun datom -> lower_matches datom && upper_matches datom) @@ -786,6 +931,7 @@ let datoms context db index ?e ?a ?v ?tx () = let exact_attr_prefix = match index, e, a, v, tx with | Aevt, None, Some _, None, None -> exact + | Avet, None, Some _, Some _, None -> exact | _ -> false in let datoms = @@ -829,7 +975,6 @@ let fold_datoms f init context db index ?e ?a ?v ?tx () = match db.duplicate_datoms, exact_prefix_bound index e a prefix_v prefix_tx with | [], Some (bound, bound_fields) -> let cmp = exact_prefix_slice_cmp context index bound bound_fields in - let seq = PSet.slice_seq ~from_:bound ~to_:bound ~cmp (stored_index db index) in let fold = match exact_attr_prefix || (e, a, v, tx) = (None, None, None, None), db.filter_pred with | true, None -> f @@ -837,12 +982,17 @@ let fold_datoms f init context db index ?e ?a ?v ?tx () = | false, None -> fold_filter | false, Some _ -> fold_filter_and_pred in - PSet.fold_seq fold init seq + (match exact_attr_prefix, index, a with + | true, (Aevt | Avet), Some attr -> + List.fold_left fold init (primary_attr_datoms db index attr) + | _ -> + let seq = Index.slice_seq ~from_:bound ~to_:bound ~cmp (stored_index db index) in + Index.fold_seq fold init seq) | [], None when (e, a, v, tx) = (None, None, None, None) -> (match db.filter_pred with - | None -> PSet.fold f init (stored_index db index) + | None -> Index.fold f init (stored_index db index) | Some pred -> - PSet.fold (fun acc datom -> if pred datom then f acc datom else acc) init (stored_index db index)) + Index.fold (fun acc datom -> if pred datom then f acc datom else acc) init (stored_index db index)) | _ -> datoms context db index ?e ?a ?v ?tx () |> Seq.fold_left f init diff --git a/impl/db.mli b/impl/db.mli index fd8d812..7cace2e 100644 --- a/impl/db.mli +++ b/impl/db.mli @@ -18,6 +18,7 @@ val normalize_datom_for_schema : schema -> datom -> datom val refresh_indexes : db -> db val refresh_indexes_with_added_datoms : db -> datom list -> db val refresh_indexes_with_tx_data : db -> datom list -> db +val snapshot_db : db -> db val with_datoms : db -> datom list -> db val empty_db : core_context -> ?schema:schema -> ?storage:storage -> unit -> db val empty : core_context -> db -> db @@ -55,6 +56,9 @@ val fold_datoms : unit -> 'acc val datoms_list : index_context -> db -> index -> ?e:entity_id -> ?a:attr -> ?v:value -> ?tx:tx -> unit -> datom list +val avet_entity_ids_by_attr_value : index_context -> db -> attr -> value -> entity_id list option +val avet_datoms_by_value : index_context -> db -> attr -> value -> datom list +val avet_datoms_by_value_seq : index_context -> db -> attr -> value -> datom Seq.t val datoms_ref : index_context -> db -> index -> ?e:entity_ref -> ?a:attr -> ?v:value -> ?tx:tx -> unit -> datom Seq.t val find_datom : index_context -> db -> index -> ?e:entity_id -> ?a:attr -> ?v:value -> ?tx:tx -> unit -> datom option val find_datom_ref : index_context -> db -> index -> ?e:entity_ref -> ?a:attr -> ?v:value -> ?tx:tx -> unit -> datom option diff --git a/impl/db_access.ml b/impl/db_access.ml index c055800..18defd9 100644 --- a/impl/db_access.ml +++ b/impl/db_access.ml @@ -104,6 +104,15 @@ end) = struct let datoms_list db index ?e ?a ?v ?tx () = Db.datoms_list db_index_context db index ?e ?a ?v ?tx () + + let avet_datoms_by_value db attr value = + Db.avet_datoms_by_value db_index_context db attr value + + let avet_datoms_by_value_seq db attr value = + Db.avet_datoms_by_value_seq db_index_context db attr value + + let avet_entity_ids_by_attr_value db attr value = + Db.avet_entity_ids_by_attr_value db_index_context db attr value let datoms_ref db index ?e ?a ?v ?tx () = Db.datoms_ref db_index_context db index ?e ?a ?v ?tx () diff --git a/impl/dune b/impl/dune index e0bf38a..e1e6971 100644 --- a/impl/dune +++ b/impl/dune @@ -1,6 +1,7 @@ (library (name datascript) (public_name datascript_ocaml) - (virtual_modules platform) + (virtual_modules platform index storage) (modes native byte melange) - (libraries datascript_types persistent_sorted_set_ocaml)) + (modules (:standard \ storage_pss storage_lmdb_impl)) + (libraries datascript_types)) diff --git a/impl/index.mli b/impl/index.mli new file mode 100644 index 0000000..2b6ff6f --- /dev/null +++ b/impl/index.mli @@ -0,0 +1,38 @@ +open Datascript_types + +type t = index_set +type 'a seq +type lmdb + +val create_lmdb : storage option -> lmdb * storage option +val lmdb_of : lmdb -> lmdb +val db_of : t -> lmdb +val lmdb_for_storage : storage -> lmdb +val sync_indexes_to_storage : t -> t -> t -> storage -> unit +val load_indexes_from_storage : storage -> lmdb -> unit + +val empty : index -> lmdb -> t +val of_sorted_list : index -> datom list -> lmdb -> t +val of_sorted_lists : (index * datom list) list -> lmdb -> unit +val of_eavt_datoms : avet:(string -> bool) -> datom list -> lmdb -> unit +val of_bulk : index -> datom list -> lmdb -> t +val add : datom -> t -> t +val remove : datom -> t -> t +val lookup : t -> datom -> datom option +val to_list : t -> datom list +val fold : ('acc -> datom -> 'acc) -> 'acc -> t -> 'acc +val fold_slice : + ('acc -> datom -> 'acc) -> 'acc -> ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> 'acc +val find_first_slice : + ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom option +val fold_attr_prefix : ('acc -> datom -> 'acc) -> 'acc -> t -> string -> 'acc +val slice : ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom list +val slice_seq : ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom seq +val rslice_seq : ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom seq +val seq : t -> datom seq +val seq_to_list : datom seq -> datom list +val fold_seq : ('acc -> datom -> 'acc) -> 'acc -> datom seq -> 'acc +val to_seq : datom seq -> datom Seq.t +val seek : datom -> datom seq -> datom seq +val flush : t -> t +val copy : t -> t diff --git a/impl/platform.mli b/impl/platform.mli index a639eeb..1d7071a 100644 --- a/impl/platform.mli +++ b/impl/platform.mli @@ -1,13 +1,8 @@ type regex -open Datascript_types - (** Return the current wall-clock time as seconds since the Unix epoch. *) val now_seconds : unit -> float -(** Create a file-backed storage instance rooted at the given path. *) -val file_storage : string -> storage - (** Compile a platform-specific regular expression from a pattern string. *) val compile_regex : string -> regex diff --git a/impl/platform/jsoo/dune b/impl/platform/jsoo/dune index 3909a7a..efd469d 100644 --- a/impl/platform/jsoo/dune +++ b/impl/platform/jsoo/dune @@ -3,4 +3,8 @@ (public_name datascript-ocaml-jsoo) (implements datascript) (modes byte) - (libraries js_of_ocaml persistent_sorted_set_ocaml.native)) + (libraries + js_of_ocaml + lmdb_db_native + lmdb_index_native + storage_lmdb_native)) diff --git a/impl/platform/jsoo/index.ml b/impl/platform/jsoo/index.ml new file mode 100644 index 0000000..953e2cb --- /dev/null +++ b/impl/platform/jsoo/index.ml @@ -0,0 +1,59 @@ +open Datascript_types + +(* Native LMDB indexes use identity coercions because [index_set] stays abstract in + [Datascript_types] while this module owns the concrete LMDB representation. *) +external inject : Datascript_lmdb_index.t -> index_set = "%identity" +external project : index_set -> Datascript_lmdb_index.t = "%identity" + +type t = index_set +type 'a seq = 'a Datascript_lmdb_index.seq +type lmdb = Datascript_lmdb_db.t + +let create_lmdb storage = + match storage with + | Some storage -> (Datascript_storage_lmdb.lmdb storage, Some storage) + | None -> (Datascript_lmdb_db.create_temp (), None) + +let lmdb_of lmdb = lmdb +let db_of t = Datascript_lmdb_index.db_of (project t) + +let lmdb_for_storage storage = Datascript_storage_lmdb.lmdb storage + +let sync_indexes_to_storage eavt aevt avet target_storage = + let target = Datascript_storage_lmdb.lmdb target_storage in + Datascript_lmdb_index.sync_merged_to_lmdb (project eavt) target; + Datascript_lmdb_index.sync_merged_to_lmdb (project aevt) target; + Datascript_lmdb_index.sync_merged_to_lmdb (project avet) target + +let load_indexes_from_storage storage target_lmdb = + let source = Datascript_storage_lmdb.lmdb storage in + if source != target_lmdb then Datascript_storage_lmdb.sync_indexes source target_lmdb + +let empty index lmdb = Datascript_lmdb_index.empty index lmdb |> inject +let of_sorted_list index datoms lmdb = Datascript_lmdb_index.of_sorted_list index datoms lmdb |> inject +let of_sorted_lists index_datoms lmdb = Datascript_lmdb_index.of_sorted_lists index_datoms lmdb +let of_eavt_datoms ~avet datoms lmdb = Datascript_lmdb_index.of_eavt_datoms ~avet datoms lmdb +let of_bulk index datoms lmdb = Datascript_lmdb_index.of_bulk index datoms lmdb |> inject + +let add datom t = Datascript_lmdb_index.add datom (project t) |> inject +let remove datom t = Datascript_lmdb_index.remove datom (project t) |> inject +let lookup t datom = Datascript_lmdb_index.lookup (project t) datom +let to_list t = Datascript_lmdb_index.to_list (project t) +let fold f init t = Datascript_lmdb_index.fold f init (project t) +let fold_slice f init ?from_ ?to_ ?cmp t = + Datascript_lmdb_index.fold_slice f init ?from_ ?to_ ?cmp (project t) +let find_first_slice ?from_ ?to_ ?cmp t = + Datascript_lmdb_index.find_first_slice ?from_ ?to_ ?cmp (project t) + +let fold_attr_prefix f init t attr = + Datascript_lmdb_index.fold_attr_prefix f init (project t) attr +let slice ?from_ ?to_ ?cmp t = Datascript_lmdb_index.slice ?from_ ?to_ ?cmp (project t) +let slice_seq ?from_ ?to_ ?cmp t = Datascript_lmdb_index.slice_seq ?from_ ?to_ ?cmp (project t) +let rslice_seq ?from_ ?to_ ?cmp t = Datascript_lmdb_index.rslice_seq ?from_ ?to_ ?cmp (project t) +let seq t = Datascript_lmdb_index.seq (project t) +let seq_to_list = Datascript_lmdb_index.seq_to_list +let fold_seq = Datascript_lmdb_index.fold_seq +let to_seq = Datascript_lmdb_index.to_seq +let seek = Datascript_lmdb_index.seek +let flush t = Datascript_lmdb_index.flush (project t) |> inject +let copy t = Datascript_lmdb_index.copy (project t) |> inject diff --git a/impl/platform/jsoo/platform.ml b/impl/platform/jsoo/platform.ml index 0642250..d9843b5 100644 --- a/impl/platform/jsoo/platform.ml +++ b/impl/platform/jsoo/platform.ml @@ -6,9 +6,6 @@ let now_seconds () = let now_ms = Js.Unsafe.meth_call Js.date "now" [||] in Js.to_float now_ms /. 1000.0 -let file_storage _dir = - invalid_arg "file_storage is not supported on js_of_ocaml" - let compile_regex = Regexp.regexp let replace_regex ~first_only regex value replacement = diff --git a/impl/platform/jsoo/storage.ml b/impl/platform/jsoo/storage.ml new file mode 100644 index 0000000..92340fc --- /dev/null +++ b/impl/platform/jsoo/storage.ml @@ -0,0 +1,104 @@ +open Datascript_types + +module Index = Index + +type restore_context = { next_db_uid : unit -> int } + +let memory_storage = Datascript_storage_lmdb.memory_storage + +let store ?storage db = + match storage, db.storage_ref with + | Some target_storage, _ | None, Some target_storage -> + Index.sync_indexes_to_storage db.eavt_index db.aevt_index db.avet_index target_storage; + Datascript_storage_lmdb.store_db target_storage db + | None, None -> invalid_arg "db has no attached storage" + +let restore_root_snapshot storage = + let schema, max_eid, max_tx, duplicate_datoms = + Datascript_storage_lmdb.restore_meta (Datascript_storage_lmdb.lmdb storage) + in + let lmdb, _ = Index.create_lmdb None in + Index.load_indexes_from_storage storage lmdb; + Some + { serializable_schema = schema + ; serializable_datoms = Index.to_list (Index.empty Eavt lmdb) @ duplicate_datoms + ; serializable_max_eid = max_eid + ; serializable_max_tx = max_tx + } + +let restore context storage = + let schema, max_eid, max_tx, duplicate_datoms = + Datascript_storage_lmdb.restore_meta (Datascript_storage_lmdb.lmdb storage) + in + let schema = Schema.validate_schema schema in + let lmdb, _ = Index.create_lmdb None in + Index.load_indexes_from_storage storage lmdb; + let duplicate_eavt_by_entity = + let table = Hashtbl.create 1024 in + List.iter + (fun datom -> + let existing = Option.value (Hashtbl.find_opt table datom.e) ~default:[] in + Hashtbl.replace table datom.e (datom :: existing)) + duplicate_datoms; + Hashtbl.iter (fun entity_id datoms -> Hashtbl.replace table entity_id (List.rev datoms)) table; + table + in + let duplicate_datoms_by_attr duplicate_datoms = + let table = Hashtbl.create 1024 in + List.iter + (fun datom -> + let existing = Option.value (Hashtbl.find_opt table datom.a) ~default:[] in + Hashtbl.replace table datom.a (datom :: existing)) + duplicate_datoms; + Hashtbl.iter (fun attr datoms -> Hashtbl.replace table attr (List.rev datoms)) table; + table + in + let duplicate_aevt_datoms = List.sort (Util.compare_datom Aevt) duplicate_datoms in + let duplicate_avet_datoms = + duplicate_datoms + |> List.filter (fun datom -> Schema.schema_attr_is_avet_accessible schema datom.a) + |> List.sort (Util.compare_datom Avet) + in + Some + { db_uid = context.next_db_uid () + ; schema + ; eavt_index = Index.empty Eavt lmdb + ; aevt_index = Index.empty Aevt lmdb + ; avet_index = Index.empty Avet lmdb + ; aevt_by_attr = Hashtbl.create 0 + ; avet_by_attr = Hashtbl.create 0 + ; avet_entities_by_attr_value = Hashtbl.create 0 + ; duplicate_datoms + ; duplicate_aevt_datoms + ; duplicate_avet_datoms + ; duplicate_eavt_by_entity + ; duplicate_aevt_by_attr = duplicate_datoms_by_attr duplicate_aevt_datoms + ; duplicate_avet_by_attr = duplicate_datoms_by_attr duplicate_avet_datoms + ; max_eid + ; max_datom_e = max_eid + ; max_tx + ; filter_pred = None + ; storage_ref = Some storage + ; tx_fns = [] + } + +let storage_addresses storage = storage.storage_list_addresses () +let storage (db : db) = db.storage_ref + +let storage_root_addresses storage = storage.storage_list_addresses () + +let addresses dbs = + dbs + |> List.concat_map (fun db -> + match db.storage_ref with + | None -> [] + | Some storage -> storage_root_addresses storage) + |> List.sort_uniq compare + +let settings (_db : db) = + [ "branching-factor", Int 32 + ; "ref-type", Keyword "weak" + ; "storage", Bool (Option.is_some _db.storage_ref) + ] + +let collect_garbage _storage = () diff --git a/impl/platform/melange/dune b/impl/platform/melange/dune index 9a91be2..917d3df 100644 --- a/impl/platform/melange/dune +++ b/impl/platform/melange/dune @@ -3,6 +3,10 @@ (public_name datascript-ocaml-melange) (implements datascript) (modes melange) - (libraries melange.js persistent_sorted_set_ocaml.melange) + (libraries + melange.js + lmdb_db_melange + lmdb_index_melange + storage_lmdb_melange) (preprocess (pps melange.ppx))) diff --git a/impl/platform/melange/index.ml b/impl/platform/melange/index.ml new file mode 100644 index 0000000..953e2cb --- /dev/null +++ b/impl/platform/melange/index.ml @@ -0,0 +1,59 @@ +open Datascript_types + +(* Native LMDB indexes use identity coercions because [index_set] stays abstract in + [Datascript_types] while this module owns the concrete LMDB representation. *) +external inject : Datascript_lmdb_index.t -> index_set = "%identity" +external project : index_set -> Datascript_lmdb_index.t = "%identity" + +type t = index_set +type 'a seq = 'a Datascript_lmdb_index.seq +type lmdb = Datascript_lmdb_db.t + +let create_lmdb storage = + match storage with + | Some storage -> (Datascript_storage_lmdb.lmdb storage, Some storage) + | None -> (Datascript_lmdb_db.create_temp (), None) + +let lmdb_of lmdb = lmdb +let db_of t = Datascript_lmdb_index.db_of (project t) + +let lmdb_for_storage storage = Datascript_storage_lmdb.lmdb storage + +let sync_indexes_to_storage eavt aevt avet target_storage = + let target = Datascript_storage_lmdb.lmdb target_storage in + Datascript_lmdb_index.sync_merged_to_lmdb (project eavt) target; + Datascript_lmdb_index.sync_merged_to_lmdb (project aevt) target; + Datascript_lmdb_index.sync_merged_to_lmdb (project avet) target + +let load_indexes_from_storage storage target_lmdb = + let source = Datascript_storage_lmdb.lmdb storage in + if source != target_lmdb then Datascript_storage_lmdb.sync_indexes source target_lmdb + +let empty index lmdb = Datascript_lmdb_index.empty index lmdb |> inject +let of_sorted_list index datoms lmdb = Datascript_lmdb_index.of_sorted_list index datoms lmdb |> inject +let of_sorted_lists index_datoms lmdb = Datascript_lmdb_index.of_sorted_lists index_datoms lmdb +let of_eavt_datoms ~avet datoms lmdb = Datascript_lmdb_index.of_eavt_datoms ~avet datoms lmdb +let of_bulk index datoms lmdb = Datascript_lmdb_index.of_bulk index datoms lmdb |> inject + +let add datom t = Datascript_lmdb_index.add datom (project t) |> inject +let remove datom t = Datascript_lmdb_index.remove datom (project t) |> inject +let lookup t datom = Datascript_lmdb_index.lookup (project t) datom +let to_list t = Datascript_lmdb_index.to_list (project t) +let fold f init t = Datascript_lmdb_index.fold f init (project t) +let fold_slice f init ?from_ ?to_ ?cmp t = + Datascript_lmdb_index.fold_slice f init ?from_ ?to_ ?cmp (project t) +let find_first_slice ?from_ ?to_ ?cmp t = + Datascript_lmdb_index.find_first_slice ?from_ ?to_ ?cmp (project t) + +let fold_attr_prefix f init t attr = + Datascript_lmdb_index.fold_attr_prefix f init (project t) attr +let slice ?from_ ?to_ ?cmp t = Datascript_lmdb_index.slice ?from_ ?to_ ?cmp (project t) +let slice_seq ?from_ ?to_ ?cmp t = Datascript_lmdb_index.slice_seq ?from_ ?to_ ?cmp (project t) +let rslice_seq ?from_ ?to_ ?cmp t = Datascript_lmdb_index.rslice_seq ?from_ ?to_ ?cmp (project t) +let seq t = Datascript_lmdb_index.seq (project t) +let seq_to_list = Datascript_lmdb_index.seq_to_list +let fold_seq = Datascript_lmdb_index.fold_seq +let to_seq = Datascript_lmdb_index.to_seq +let seek = Datascript_lmdb_index.seek +let flush t = Datascript_lmdb_index.flush (project t) |> inject +let copy t = Datascript_lmdb_index.copy (project t) |> inject diff --git a/impl/platform/melange/platform.ml b/impl/platform/melange/platform.ml index 5d9e848..78aee1c 100644 --- a/impl/platform/melange/platform.ml +++ b/impl/platform/melange/platform.ml @@ -6,9 +6,6 @@ external set_last_index : Js.Re.t -> int -> unit = "lastIndex" [@@mel.set] let now_seconds () = date_now () /. 1000.0 -let file_storage _dir = - invalid_arg "file_storage is not supported on Melange" - let compile_regex pattern = pattern let regexp ?(global = false) pattern = diff --git a/impl/platform/melange/storage.ml b/impl/platform/melange/storage.ml new file mode 100644 index 0000000..92340fc --- /dev/null +++ b/impl/platform/melange/storage.ml @@ -0,0 +1,104 @@ +open Datascript_types + +module Index = Index + +type restore_context = { next_db_uid : unit -> int } + +let memory_storage = Datascript_storage_lmdb.memory_storage + +let store ?storage db = + match storage, db.storage_ref with + | Some target_storage, _ | None, Some target_storage -> + Index.sync_indexes_to_storage db.eavt_index db.aevt_index db.avet_index target_storage; + Datascript_storage_lmdb.store_db target_storage db + | None, None -> invalid_arg "db has no attached storage" + +let restore_root_snapshot storage = + let schema, max_eid, max_tx, duplicate_datoms = + Datascript_storage_lmdb.restore_meta (Datascript_storage_lmdb.lmdb storage) + in + let lmdb, _ = Index.create_lmdb None in + Index.load_indexes_from_storage storage lmdb; + Some + { serializable_schema = schema + ; serializable_datoms = Index.to_list (Index.empty Eavt lmdb) @ duplicate_datoms + ; serializable_max_eid = max_eid + ; serializable_max_tx = max_tx + } + +let restore context storage = + let schema, max_eid, max_tx, duplicate_datoms = + Datascript_storage_lmdb.restore_meta (Datascript_storage_lmdb.lmdb storage) + in + let schema = Schema.validate_schema schema in + let lmdb, _ = Index.create_lmdb None in + Index.load_indexes_from_storage storage lmdb; + let duplicate_eavt_by_entity = + let table = Hashtbl.create 1024 in + List.iter + (fun datom -> + let existing = Option.value (Hashtbl.find_opt table datom.e) ~default:[] in + Hashtbl.replace table datom.e (datom :: existing)) + duplicate_datoms; + Hashtbl.iter (fun entity_id datoms -> Hashtbl.replace table entity_id (List.rev datoms)) table; + table + in + let duplicate_datoms_by_attr duplicate_datoms = + let table = Hashtbl.create 1024 in + List.iter + (fun datom -> + let existing = Option.value (Hashtbl.find_opt table datom.a) ~default:[] in + Hashtbl.replace table datom.a (datom :: existing)) + duplicate_datoms; + Hashtbl.iter (fun attr datoms -> Hashtbl.replace table attr (List.rev datoms)) table; + table + in + let duplicate_aevt_datoms = List.sort (Util.compare_datom Aevt) duplicate_datoms in + let duplicate_avet_datoms = + duplicate_datoms + |> List.filter (fun datom -> Schema.schema_attr_is_avet_accessible schema datom.a) + |> List.sort (Util.compare_datom Avet) + in + Some + { db_uid = context.next_db_uid () + ; schema + ; eavt_index = Index.empty Eavt lmdb + ; aevt_index = Index.empty Aevt lmdb + ; avet_index = Index.empty Avet lmdb + ; aevt_by_attr = Hashtbl.create 0 + ; avet_by_attr = Hashtbl.create 0 + ; avet_entities_by_attr_value = Hashtbl.create 0 + ; duplicate_datoms + ; duplicate_aevt_datoms + ; duplicate_avet_datoms + ; duplicate_eavt_by_entity + ; duplicate_aevt_by_attr = duplicate_datoms_by_attr duplicate_aevt_datoms + ; duplicate_avet_by_attr = duplicate_datoms_by_attr duplicate_avet_datoms + ; max_eid + ; max_datom_e = max_eid + ; max_tx + ; filter_pred = None + ; storage_ref = Some storage + ; tx_fns = [] + } + +let storage_addresses storage = storage.storage_list_addresses () +let storage (db : db) = db.storage_ref + +let storage_root_addresses storage = storage.storage_list_addresses () + +let addresses dbs = + dbs + |> List.concat_map (fun db -> + match db.storage_ref with + | None -> [] + | Some storage -> storage_root_addresses storage) + |> List.sort_uniq compare + +let settings (_db : db) = + [ "branching-factor", Int 32 + ; "ref-type", Keyword "weak" + ; "storage", Bool (Option.is_some _db.storage_ref) + ] + +let collect_garbage _storage = () diff --git a/impl/platform/native/dune b/impl/platform/native/dune index e6f1550..ea3584c 100644 --- a/impl/platform/native/dune +++ b/impl/platform/native/dune @@ -3,4 +3,9 @@ (public_name datascript-ocaml-native) (implements datascript) (modes native byte) - (libraries str unix persistent_sorted_set_ocaml.native)) + (libraries + str + unix + lmdb_db_native + lmdb_index_native + storage_lmdb_native)) diff --git a/impl/platform/native/index.ml b/impl/platform/native/index.ml new file mode 100644 index 0000000..fb4d676 --- /dev/null +++ b/impl/platform/native/index.ml @@ -0,0 +1,58 @@ +open Datascript_types + +(* Native LMDB indexes use identity coercions because [index_set] stays abstract in + [Datascript_types] while this module owns the concrete LMDB representation. *) +external inject : Datascript_lmdb_index.t -> index_set = "%identity" +external project : index_set -> Datascript_lmdb_index.t = "%identity" + +type t = index_set +type 'a seq = 'a Datascript_lmdb_index.seq +type lmdb = Datascript_lmdb_db.t + +let create_lmdb storage = + match storage with + | Some storage -> (Datascript_storage_lmdb.lmdb storage, Some storage) + | None -> (Datascript_lmdb_db.create_temp (), None) + +let lmdb_of lmdb = lmdb +let db_of t = Datascript_lmdb_index.db_of (project t) + +let lmdb_for_storage storage = Datascript_storage_lmdb.lmdb storage + +let sync_indexes_to_storage eavt aevt avet target_storage = + let target = Datascript_storage_lmdb.lmdb target_storage in + Datascript_lmdb_index.sync_merged_to_lmdb (project eavt) target; + Datascript_lmdb_index.sync_merged_to_lmdb (project aevt) target; + Datascript_lmdb_index.sync_merged_to_lmdb (project avet) target + +let load_indexes_from_storage storage target_lmdb = + let source = Datascript_storage_lmdb.lmdb storage in + if source != target_lmdb then Datascript_storage_lmdb.sync_indexes source target_lmdb + +let empty index lmdb = Datascript_lmdb_index.empty index lmdb |> inject +let of_sorted_list index datoms lmdb = Datascript_lmdb_index.of_sorted_list index datoms lmdb |> inject +let of_sorted_lists index_datoms lmdb = Datascript_lmdb_index.of_sorted_lists index_datoms lmdb +let of_eavt_datoms ~avet datoms lmdb = Datascript_lmdb_index.of_eavt_datoms ~avet datoms lmdb +let of_bulk index datoms lmdb = Datascript_lmdb_index.of_bulk index datoms lmdb |> inject + +let add datom t = Datascript_lmdb_index.add datom (project t) |> inject +let remove datom t = Datascript_lmdb_index.remove datom (project t) |> inject +let lookup t datom = Datascript_lmdb_index.lookup (project t) datom +let to_list t = Datascript_lmdb_index.to_list (project t) +let fold f init t = Datascript_lmdb_index.fold f init (project t) +let fold_slice f init ?from_ ?to_ ?cmp t = + Datascript_lmdb_index.fold_slice f init ?from_ ?to_ ?cmp (project t) +let find_first_slice ?from_ ?to_ ?cmp t = + Datascript_lmdb_index.find_first_slice ?from_ ?to_ ?cmp (project t) +let fold_attr_prefix f init t attr = + Datascript_lmdb_index.fold_attr_prefix f init (project t) attr +let slice ?from_ ?to_ ?cmp t = Datascript_lmdb_index.slice ?from_ ?to_ ?cmp (project t) +let slice_seq ?from_ ?to_ ?cmp t = Datascript_lmdb_index.slice_seq ?from_ ?to_ ?cmp (project t) +let rslice_seq ?from_ ?to_ ?cmp t = Datascript_lmdb_index.rslice_seq ?from_ ?to_ ?cmp (project t) +let seq t = Datascript_lmdb_index.seq (project t) +let seq_to_list = Datascript_lmdb_index.seq_to_list +let fold_seq = Datascript_lmdb_index.fold_seq +let to_seq = Datascript_lmdb_index.to_seq +let seek = Datascript_lmdb_index.seek +let flush t = Datascript_lmdb_index.flush (project t) |> inject +let copy t = Datascript_lmdb_index.copy (project t) |> inject diff --git a/impl/platform/native/platform.ml b/impl/platform/native/platform.ml index b620154..1e126c5 100644 --- a/impl/platform/native/platform.ml +++ b/impl/platform/native/platform.ml @@ -1,88 +1,7 @@ type regex = Str.regexp -open Datascript_types - let now_seconds = Unix.gettimeofday -let ensure_storage_dir dir = - if Sys.file_exists dir then begin - if not (Sys.is_directory dir) then - invalid_arg ("storage path is not a directory: " ^ dir) - end - else Sys.mkdir dir 0o755 - -let hex_digit value = - Char.chr (if value < 10 then Char.code '0' + value else Char.code 'a' + value - 10) - -let hex_value = function - | '0' .. '9' as ch -> Char.code ch - Char.code '0' - | 'a' .. 'f' as ch -> Char.code ch - Char.code 'a' + 10 - | 'A' .. 'F' as ch -> Char.code ch - Char.code 'A' + 10 - | ch -> invalid_arg ("invalid storage address hex digit: " ^ String.make 1 ch) - -let encode_storage_address address = - String.init - (String.length address * 2) - (fun index -> - let code = Char.code address.[index / 2] in - if index mod 2 = 0 then hex_digit (code lsr 4) else hex_digit (code land 0x0f)) - -let decode_storage_address encoded = - if String.length encoded mod 2 <> 0 then - invalid_arg ("invalid storage address filename: " ^ encoded); - String.init - (String.length encoded / 2) - (fun index -> - let high = hex_value encoded.[index * 2] in - let low = hex_value encoded.[index * 2 + 1] in - Char.chr ((high lsl 4) lor low)) - -let storage_payload_path dir address = - Filename.concat dir (encode_storage_address address ^ ".bin") - -let file_storage dir = - ensure_storage_dir dir; - let write_payload address payload = - let channel = open_out_bin (storage_payload_path dir address) in - Fun.protect - ~finally:(fun () -> close_out_noerr channel) - (fun () -> Marshal.to_channel channel payload []) - in - let read_payload address = - let path = storage_payload_path dir address in - if not (Sys.file_exists path) then None - else - let channel = open_in_bin path in - Fun.protect - ~finally:(fun () -> close_in_noerr channel) - (fun () -> Some (Marshal.from_channel channel : storage_payload)) - in - let list_addresses () = - Sys.readdir dir - |> Array.to_list - |> List.filter_map (fun filename -> - if Filename.extension filename = ".bin" then - let base = Filename.remove_extension filename in - Some (decode_storage_address base) - else - None) - |> List.sort_uniq compare - in - let delete addresses = - List.iter - (fun address -> - let path = storage_payload_path dir address in - if Sys.file_exists path then Sys.remove path) - addresses - in - { storage_store = - (fun entries -> - List.iter (fun (address, payload) -> write_payload address payload) entries) - ; storage_restore = read_payload - ; storage_list_addresses = list_addresses - ; storage_delete = delete - } - let str_pattern_of_pattern pattern = let buffer = Buffer.create (String.length pattern) in let add_escaped = function diff --git a/impl/platform/native/storage.ml b/impl/platform/native/storage.ml new file mode 100644 index 0000000..92340fc --- /dev/null +++ b/impl/platform/native/storage.ml @@ -0,0 +1,104 @@ +open Datascript_types + +module Index = Index + +type restore_context = { next_db_uid : unit -> int } + +let memory_storage = Datascript_storage_lmdb.memory_storage + +let store ?storage db = + match storage, db.storage_ref with + | Some target_storage, _ | None, Some target_storage -> + Index.sync_indexes_to_storage db.eavt_index db.aevt_index db.avet_index target_storage; + Datascript_storage_lmdb.store_db target_storage db + | None, None -> invalid_arg "db has no attached storage" + +let restore_root_snapshot storage = + let schema, max_eid, max_tx, duplicate_datoms = + Datascript_storage_lmdb.restore_meta (Datascript_storage_lmdb.lmdb storage) + in + let lmdb, _ = Index.create_lmdb None in + Index.load_indexes_from_storage storage lmdb; + Some + { serializable_schema = schema + ; serializable_datoms = Index.to_list (Index.empty Eavt lmdb) @ duplicate_datoms + ; serializable_max_eid = max_eid + ; serializable_max_tx = max_tx + } + +let restore context storage = + let schema, max_eid, max_tx, duplicate_datoms = + Datascript_storage_lmdb.restore_meta (Datascript_storage_lmdb.lmdb storage) + in + let schema = Schema.validate_schema schema in + let lmdb, _ = Index.create_lmdb None in + Index.load_indexes_from_storage storage lmdb; + let duplicate_eavt_by_entity = + let table = Hashtbl.create 1024 in + List.iter + (fun datom -> + let existing = Option.value (Hashtbl.find_opt table datom.e) ~default:[] in + Hashtbl.replace table datom.e (datom :: existing)) + duplicate_datoms; + Hashtbl.iter (fun entity_id datoms -> Hashtbl.replace table entity_id (List.rev datoms)) table; + table + in + let duplicate_datoms_by_attr duplicate_datoms = + let table = Hashtbl.create 1024 in + List.iter + (fun datom -> + let existing = Option.value (Hashtbl.find_opt table datom.a) ~default:[] in + Hashtbl.replace table datom.a (datom :: existing)) + duplicate_datoms; + Hashtbl.iter (fun attr datoms -> Hashtbl.replace table attr (List.rev datoms)) table; + table + in + let duplicate_aevt_datoms = List.sort (Util.compare_datom Aevt) duplicate_datoms in + let duplicate_avet_datoms = + duplicate_datoms + |> List.filter (fun datom -> Schema.schema_attr_is_avet_accessible schema datom.a) + |> List.sort (Util.compare_datom Avet) + in + Some + { db_uid = context.next_db_uid () + ; schema + ; eavt_index = Index.empty Eavt lmdb + ; aevt_index = Index.empty Aevt lmdb + ; avet_index = Index.empty Avet lmdb + ; aevt_by_attr = Hashtbl.create 0 + ; avet_by_attr = Hashtbl.create 0 + ; avet_entities_by_attr_value = Hashtbl.create 0 + ; duplicate_datoms + ; duplicate_aevt_datoms + ; duplicate_avet_datoms + ; duplicate_eavt_by_entity + ; duplicate_aevt_by_attr = duplicate_datoms_by_attr duplicate_aevt_datoms + ; duplicate_avet_by_attr = duplicate_datoms_by_attr duplicate_avet_datoms + ; max_eid + ; max_datom_e = max_eid + ; max_tx + ; filter_pred = None + ; storage_ref = Some storage + ; tx_fns = [] + } + +let storage_addresses storage = storage.storage_list_addresses () +let storage (db : db) = db.storage_ref + +let storage_root_addresses storage = storage.storage_list_addresses () + +let addresses dbs = + dbs + |> List.concat_map (fun db -> + match db.storage_ref with + | None -> [] + | Some storage -> storage_root_addresses storage) + |> List.sort_uniq compare + +let settings (_db : db) = + [ "branching-factor", Int 32 + ; "ref-type", Keyword "weak" + ; "storage", Bool (Option.is_some _db.storage_ref) + ] + +let collect_garbage _storage = () diff --git a/impl/query_where.ml b/impl/query_where.ml index bb7d0bb..addc3d0 100644 --- a/impl/query_where.ml +++ b/impl/query_where.ml @@ -25,6 +25,10 @@ module Make (Context : sig val is_ref_attr : db -> attr -> bool val cardinality_one : db -> attr -> bool val normalize_value : value -> value + val datoms_by_attr_value : db -> attr -> value -> datom list + val entity_ids_by_attr_value : db -> attr -> value -> entity_id list option + val query_attr_uses_avet : db -> attr -> bool + val query_value_uses_avet : value -> bool end) = struct open Context @@ -1034,15 +1038,27 @@ end) = struct not (query_evaluator_context.is_reverse_ref attr) in let datoms_matching attr value = - let datoms = source_context.pattern_datoms source_db (QVar e_var) (QAttr attr) (QValue value) None in - if direct_attr attr then - List.of_seq datoms + if + direct_attr attr && query_value_uses_avet value + && query_attr_uses_avet source_db attr + then + datoms_by_attr_value source_db attr value else - datoms - |> Seq.filter (fun datom -> - Option.is_some - (source_context.match_data_pattern source_db [] (QVar e_var) (QAttr attr) (QValue value) datom)) - |> List.of_seq + let datoms = source_context.pattern_datoms source_db (QVar e_var) (QAttr attr) (QValue value) None in + if direct_attr attr then + List.of_seq datoms + else + datoms + |> Seq.filter (fun datom -> + Option.is_some + (source_context.match_data_pattern source_db [] (QVar e_var) (QAttr attr) (QValue value) datom)) + |> List.of_seq + in + let avet_entity_ids attr value = + if direct_attr attr && query_value_uses_avet value && query_attr_uses_avet source_db attr then + entity_ids_by_attr_value source_db attr value + else + None in let constant_datoms = constant_patterns @@ -1054,10 +1070,43 @@ end) = struct |> unique_vars in let lookup_vars = relation_lookup_vars source_db [ QVar e_var; QWildcard; QWildcard ] in - if List.exists (fun (_, _, datoms) -> Lazy.force datoms = []) constant_datoms then + if + List.exists + (fun (attr, value, datoms) -> + match avet_entity_ids attr value with + | Some [] -> true + | Some _ -> false + | None -> Lazy.force datoms = []) + constant_datoms + then Some { attrs; rows = []; lookup_vars; unique_rows = true } else + let avet_single_entity_rows = + match constant_patterns, value_var_patterns, required_patterns, excluded_patterns, relation_comparisons with + | [ (attr, value) ], [], [], [], [] -> ( + match avet_entity_ids attr value with + | Some entity_ids -> Some (List.map (fun entity_id -> [ Result_entity entity_id ]) entity_ids) + | None -> None) + | _ -> None + in + if Option.is_some avet_single_entity_rows then + Some + { attrs + ; rows = Option.get avet_single_entity_rows + ; lookup_vars + ; unique_rows = true + } + else let constant_sets = + let set_from_entity_ids entity_ids = + let entities = Bytes.make (source_db.max_datom_e + 1) '\000' in + List.iter + (fun entity_id -> + if entity_id >= 0 && entity_id < Bytes.length entities then + Bytes.set entities entity_id '\001') + entity_ids; + entities + in let set_from_datoms datoms = let entities = Bytes.make (source_db.max_datom_e + 1) '\000' in List.iter @@ -1068,7 +1117,15 @@ end) = struct entities in constant_datoms - |> List.map (fun (_, _, datoms) -> set_from_datoms (Lazy.force datoms)) + |> List.map (fun (attr, value, datoms) -> + match avet_entity_ids attr value with + | Some entity_ids -> set_from_entity_ids entity_ids + | None -> set_from_datoms (Lazy.force datoms)) + in + let constant_count (attr, value, datoms) = + match avet_entity_ids attr value with + | Some entity_ids -> List.length entity_ids + | None -> List.length (Lazy.force datoms) in let candidate_entities () = match constant_datoms with @@ -1081,10 +1138,12 @@ end) = struct | [], [] -> []) | datoms_by_constant -> datoms_by_constant - |> List.sort (fun (_, _, left) (_, _, right) -> - compare (List.length (Lazy.force left)) (List.length (Lazy.force right))) + |> List.sort (fun left right -> compare (constant_count left) (constant_count right)) |> function - | (_, _, datoms) :: _ -> List.map (fun datom -> datom.e) (Lazy.force datoms) + | (attr, value, datoms) :: _ -> ( + match avet_entity_ids attr value with + | Some entity_ids -> entity_ids + | None -> List.map (fun datom -> datom.e) (Lazy.force datoms)) | [] -> [] in let has_pattern entity_id attr value_term = @@ -1432,7 +1491,7 @@ end) = struct binding_row attrs binding) |> List.of_seq in - let rows = + let compute_default_rows () = match value_var_patterns with | (scan_value_var, scan_attr) :: remaining_value_vars when direct_attr scan_attr @@ -1463,6 +1522,14 @@ end) = struct in bindings |> List.filter_map (binding_row attrs)) in + let rows = + match constant_patterns, value_var_patterns, required_patterns, excluded_patterns, relation_comparisons with + | [ (attr, value) ], [], [], [], [] when value_var_patterns = [] -> ( + match avet_entity_ids attr value with + | Some entity_ids -> List.map (fun entity_id -> [ Result_entity entity_id ]) entity_ids + | None -> compute_default_rows ()) + | _ -> compute_default_rows () + in let unique_rows = source_db.duplicate_datoms = [] && List.mem e_var attrs diff --git a/impl/serialize.ml b/impl/serialize.ml index 06dd4fc..e039a09 100644 --- a/impl/serialize.ml +++ b/impl/serialize.ml @@ -1,95 +1,46 @@ open Datascript_types -module PSet = Persistent_sorted_set +module Index = Index +module Schema = Schema type context = { next_db_uid : unit -> int ; validate_schema : schema -> schema ; normalize_datom_for_schema : schema -> datom -> datom - ; refresh_db_indexes : db -> db + ; with_datoms : db -> datom list -> db } let serializable db = { serializable_schema = db.schema ; serializable_datoms = - PSet.to_list db.eavt_index @ db.duplicate_datoms |> List.sort (Util.compare_datom Eavt) + Index.to_list db.eavt_index @ db.duplicate_datoms |> List.sort (Datascript_types.Compare.compare_datom Eavt) ; serializable_max_eid = db.max_eid ; serializable_max_tx = db.max_tx } -let empty_index index = - PSet.empty_by ~cmp:(Util.compare_datom index) () - -let index_from_datoms index datoms = - let cmp = Util.compare_datom index in - let items = Array.of_list datoms in - Array.sort cmp items; - PSet.of_sorted_array_by ~cmp items - -let duplicate_datoms datoms = - let datoms = List.sort (Util.compare_datom Eavt) datoms in - let rec loop previous duplicates = function - | [] -> List.rev duplicates - | datom :: rest -> - (match previous with - | Some previous when Util.compare_datom Eavt previous datom = 0 -> - loop (Some datom) (datom :: duplicates) rest - | _ -> loop (Some datom) duplicates rest) - in - loop None [] datoms - -let duplicate_aevt_datoms duplicate_datoms = - List.sort (Util.compare_datom Aevt) duplicate_datoms - -let duplicate_avet_datoms schema duplicate_datoms = - duplicate_datoms - |> List.filter (fun datom -> Schema.schema_attr_is_avet_accessible schema datom.a) - |> List.sort (Util.compare_datom Avet) - -let duplicate_eavt_by_entity duplicate_datoms = - let table = Hashtbl.create 1024 in - List.iter - (fun datom -> - let existing = Option.value (Hashtbl.find_opt table datom.e) ~default:[] in - Hashtbl.replace table datom.e (datom :: existing)) - duplicate_datoms; - Hashtbl.iter (fun entity_id datoms -> Hashtbl.replace table entity_id (List.rev datoms)) table; - table - -let duplicate_datoms_by_attr duplicate_datoms = - let table = Hashtbl.create 1024 in - List.iter - (fun datom -> - let existing = Option.value (Hashtbl.find_opt table datom.a) ~default:[] in - Hashtbl.replace table datom.a (datom :: existing)) - duplicate_datoms; - Hashtbl.iter (fun attr datoms -> Hashtbl.replace table attr (List.rev datoms)) table; - table - let from_serializable context snapshot = let schema = context.validate_schema snapshot.serializable_schema in let datoms = List.map (context.normalize_datom_for_schema schema) snapshot.serializable_datoms in - let duplicate_datoms = duplicate_datoms datoms in - let duplicate_aevt_datoms = duplicate_aevt_datoms duplicate_datoms in - let duplicate_avet_datoms = duplicate_avet_datoms schema duplicate_datoms in + let lmdb, storage_ref = Index.create_lmdb None in { db_uid = context.next_db_uid () ; schema - ; eavt_index = index_from_datoms Eavt datoms - ; aevt_index = empty_index Aevt - ; avet_index = empty_index Avet + ; eavt_index = Index.empty Eavt lmdb + ; aevt_index = Index.empty Aevt lmdb + ; avet_index = Index.empty Avet lmdb ; aevt_by_attr = Hashtbl.create 0 ; avet_by_attr = Hashtbl.create 0 - ; duplicate_datoms - ; duplicate_aevt_datoms - ; duplicate_avet_datoms - ; duplicate_eavt_by_entity = duplicate_eavt_by_entity duplicate_datoms - ; duplicate_aevt_by_attr = duplicate_datoms_by_attr duplicate_aevt_datoms - ; duplicate_avet_by_attr = duplicate_datoms_by_attr duplicate_avet_datoms + ; avet_entities_by_attr_value = Hashtbl.create 0 + ; duplicate_datoms = [] + ; duplicate_aevt_datoms = [] + ; duplicate_avet_datoms = [] + ; duplicate_eavt_by_entity = Hashtbl.create 0 + ; duplicate_aevt_by_attr = Hashtbl.create 0 + ; duplicate_avet_by_attr = Hashtbl.create 0 ; max_eid = snapshot.serializable_max_eid ; max_datom_e = 0 ; max_tx = snapshot.serializable_max_tx ; filter_pred = None - ; storage_ref = None + ; storage_ref ; tx_fns = [] } - |> context.refresh_db_indexes + |> fun db -> context.with_datoms db datoms diff --git a/impl/serialize.mli b/impl/serialize.mli index c071cbb..3dac898 100644 --- a/impl/serialize.mli +++ b/impl/serialize.mli @@ -4,7 +4,7 @@ type context = { next_db_uid : unit -> int ; validate_schema : schema -> schema ; normalize_datom_for_schema : schema -> datom -> datom - ; refresh_db_indexes : db -> db + ; with_datoms : db -> datom list -> db } val serializable : db -> serializable_db diff --git a/impl/storage.mli b/impl/storage.mli index f9020f7..0bc203b 100644 --- a/impl/storage.mli +++ b/impl/storage.mli @@ -1,25 +1,10 @@ open Datascript_types -type tail_context = - { apply_group : db -> datom list -> db - } +type restore_context = { next_db_uid : unit -> int } -type restore_context = - { next_db_uid : unit -> int - ; db_with_tail : db -> datom list list -> db - } - -val root_address : storage_address -val tail_address : storage_address val memory_storage : unit -> storage -val file_storage : string -> storage val store : ?storage:storage -> db -> unit -val store_tail : storage -> datom list list -> unit -val tail_compaction_threshold : int -val tail_datom_count : datom list list -> int val restore_root_snapshot : storage -> serializable_db option -val restore_tail_groups : storage -> datom list list -val db_with_tail : tail_context -> db -> datom list list -> db val restore : restore_context -> storage -> db option val storage_addresses : storage -> storage_address list val storage : db -> storage option diff --git a/impl/storage_lmdb_impl.ml b/impl/storage_lmdb_impl.ml new file mode 100644 index 0000000..b65fcfb --- /dev/null +++ b/impl/storage_lmdb_impl.ml @@ -0,0 +1,123 @@ +open Datascript_types + +module Index = Index + +type tail_context = + { apply_group : db -> datom list -> db + } + +type restore_context = { next_db_uid : unit -> int } + +let memory_storage = Datascript_storage_lmdb.memory_storage +let file_storage = Datascript_storage_lmdb.file_storage + +let store ?storage db = + match storage, db.storage_ref with + | Some storage, _ | None, Some storage -> + let lmdb = Datascript_storage_lmdb.lmdb storage in + Datascript_storage_lmdb.store_meta lmdb db + | None, None -> invalid_arg "db has no attached storage" + +let store_tail _storage _tail = () + +let tail_compaction_threshold = 0 +let tail_datom_count _tail = 0 + +let restore_tail_groups _storage = [] + +let restore_root_snapshot storage = + let lmdb = Datascript_storage_lmdb.lmdb storage in + let schema, max_eid, max_tx, duplicate_datoms = + Datascript_storage_lmdb.restore_meta lmdb + in + Some + { serializable_schema = schema + ; serializable_datoms = Index.to_list (Index.empty Eavt lmdb) @ duplicate_datoms + ; serializable_max_eid = max_eid + ; serializable_max_tx = max_tx + } + +let db_with_tail _context db tail = + List.fold_left + (fun db group -> + match group with + | [] -> db + | _ -> Db.refresh_indexes_with_tx_data db group) + db + tail + +let restore context storage = + let lmdb = Datascript_storage_lmdb.lmdb storage in + let schema, max_eid, max_tx, duplicate_datoms = + Datascript_storage_lmdb.restore_meta lmdb + in + let schema = Schema.validate_schema schema in + let duplicate_eavt_by_entity = + let table = Hashtbl.create 1024 in + List.iter + (fun datom -> + let existing = Option.value (Hashtbl.find_opt table datom.e) ~default:[] in + Hashtbl.replace table datom.e (datom :: existing)) + duplicate_datoms; + Hashtbl.iter (fun entity_id datoms -> Hashtbl.replace table entity_id (List.rev datoms)) table; + table + in + let duplicate_datoms_by_attr duplicate_datoms = + let table = Hashtbl.create 1024 in + List.iter + (fun datom -> + let existing = Option.value (Hashtbl.find_opt table datom.a) ~default:[] in + Hashtbl.replace table datom.a (datom :: existing)) + duplicate_datoms; + Hashtbl.iter (fun attr datoms -> Hashtbl.replace table attr (List.rev datoms)) table; + table + in + let duplicate_aevt_datoms = List.sort (Util.compare_datom Aevt) duplicate_datoms in + let duplicate_avet_datoms = + duplicate_datoms + |> List.filter (fun datom -> Schema.schema_attr_is_avet_accessible schema datom.a) + |> List.sort (Util.compare_datom Avet) + in + Some + { db_uid = context.next_db_uid () + ; schema + ; eavt_index = Index.empty Eavt lmdb + ; aevt_index = Index.empty Aevt lmdb + ; avet_index = Index.empty Avet lmdb + ; aevt_by_attr = Hashtbl.create 0 + ; avet_by_attr = Hashtbl.create 0 + ; avet_entities_by_attr_value = Hashtbl.create 0 + ; duplicate_datoms + ; duplicate_aevt_datoms + ; duplicate_avet_datoms + ; duplicate_eavt_by_entity + ; duplicate_aevt_by_attr = duplicate_datoms_by_attr duplicate_aevt_datoms + ; duplicate_avet_by_attr = duplicate_datoms_by_attr duplicate_avet_datoms + ; max_eid + ; max_datom_e = max_eid + ; max_tx + ; filter_pred = None + ; storage_ref = Some storage + ; tx_fns = [] + } + +let storage_addresses storage = storage.storage_list_addresses () +let storage (db : db) = db.storage_ref + +let storage_root_addresses storage = storage.storage_list_addresses () + +let addresses dbs = + dbs + |> List.concat_map (fun db -> + match db.storage_ref with + | None -> [] + | Some storage -> storage_root_addresses storage) + |> List.sort_uniq compare + +let settings (db : db) = + [ "branching-factor", Int 32 + ; "ref-type", Keyword "weak" + ; "storage", Bool (Option.is_some db.storage_ref) + ] + +let collect_garbage _storage = () diff --git a/impl/storage.ml b/impl/storage_pss.ml similarity index 99% rename from impl/storage.ml rename to impl/storage_pss.ml index ade23e5..cc8819c 100644 --- a/impl/storage.ml +++ b/impl/storage_pss.ml @@ -253,6 +253,7 @@ let restore context storage = ; avet_index ; aevt_by_attr = Hashtbl.create 0 ; avet_by_attr = Hashtbl.create 0 + ; avet_entities_by_attr_value = Hashtbl.create 0 ; duplicate_datoms ; duplicate_aevt_datoms ; duplicate_avet_datoms diff --git a/impl/util.ml b/impl/util.ml index b0497da..3776793 100644 --- a/impl/util.ml +++ b/impl/util.ml @@ -53,306 +53,18 @@ and value_equal left right = | Ref_to left, Ref_to right -> entity_ref_equal left right | _ -> false -let split_keyword keyword = - match String.index_opt keyword '/' with - | None -> "", keyword - | Some index -> - let namespace = String.sub keyword 0 index in - let name = String.sub keyword (index + 1) (String.length keyword - index - 1) in - namespace, name - -let rec compare_list_items_with compare_item left right = - match left, right with - | [], [] -> 0 - | left :: left_rest, right :: right_rest -> - let comparison = compare_item left right in - if comparison <> 0 then comparison else compare_list_items_with compare_item left_rest right_rest - | [], _ | _, [] -> 0 - -let compare_list_with compare_item left right = - let length_comparison = compare (List.length left) (List.length right) in - if length_comparison <> 0 then length_comparison - else compare_list_items_with compare_item left right - -let compare_option_with compare_item left right = - match left, right with - | None, None -> 0 - | None, Some _ -> -1 - | Some _, None -> 1 - | Some left, Some right -> compare_item left right - -let i32 value = Int32.of_int value -let i32_to_int value = Int32.to_int value -let i32_add left right = Int32.add left right -let i32_mul left right = Int32.mul left right -let i32_xor left right = Int32.logxor left right -let i32_shift_left value bits = Int32.shift_left value bits -let i32_shift_right value bits = Int32.shift_right value bits -let i32_shift_right_logical value bits = Int32.shift_right_logical value bits - -let i32_rotate_left value bits = - Int32.logor (Int32.shift_left value bits) (Int32.shift_right_logical value (32 - bits)) - -let murmur3_mix_k1 value = - value - |> fun value -> i32_mul value (i32 (-862048943)) - |> fun value -> i32_rotate_left value 15 - |> fun value -> i32_mul value (i32 461845907) - -let murmur3_mix_h1 hash value = - i32_xor hash value - |> fun hash -> i32_rotate_left hash 13 - |> fun hash -> i32_add (i32_mul hash (i32 5)) (i32 (-430675100)) - -let murmur3_fmix hash length = - i32_xor hash (i32 length) - |> fun hash -> i32_xor hash (i32_shift_right_logical hash 16) - |> fun hash -> i32_mul hash (i32 (-2048144789)) - |> fun hash -> i32_xor hash (i32_shift_right_logical hash 13) - |> fun hash -> i32_mul hash (i32 (-1028477387)) - |> fun hash -> i32_xor hash (i32_shift_right_logical hash 16) - -let murmur3_hash_int value = - if value = 0 then 0 - else - value - |> i32 - |> murmur3_mix_k1 - |> murmur3_mix_h1 Int32.zero - |> fun hash -> murmur3_fmix hash 4 - |> i32_to_int - -let murmur3_hash_long value = - if value = Int64.zero then 0 - else - let low = Int64.to_int value |> i32 in - let high = Int64.shift_right_logical value 32 |> Int64.to_int |> i32 in - Int32.zero - |> fun hash -> murmur3_mix_h1 hash (murmur3_mix_k1 low) - |> fun hash -> murmur3_mix_h1 hash (murmur3_mix_k1 high) - |> fun hash -> murmur3_fmix hash 8 - |> i32_to_int - -let murmur3_hash_unencoded_chars text = - let hash = ref Int32.zero in - let index = ref 1 in - let length = String.length text in - while !index < length do - let code = - Char.code text.[!index - 1] lor (Char.code text.[!index] lsl 16) - in - hash := murmur3_mix_h1 !hash (murmur3_mix_k1 (i32 code)); - index := !index + 2 - done; - if length land 1 = 1 then - hash := i32_xor !hash (murmur3_mix_k1 (i32 (Char.code text.[length - 1]))); - murmur3_fmix !hash (2 * length) |> i32_to_int - -let java_string_hash text = - let hash = ref Int32.zero in - String.iter - (fun ch -> hash := i32_add (i32_mul !hash (i32 31)) (i32 (Char.code ch))) - text; - i32_to_int !hash - -let hex_value = function - | '0' .. '9' as ch -> Char.code ch - Char.code '0' - | 'a' .. 'f' as ch -> 10 + Char.code ch - Char.code 'a' - | 'A' .. 'F' as ch -> 10 + Char.code ch - Char.code 'A' - | _ -> invalid_arg "invalid UUID hex digit" - -let uuid_halves uuid = - let digits = - uuid - |> String.to_seq - |> Seq.filter (( <> ) '-') - |> List.of_seq - in - if List.length digits <> 32 then invalid_arg ("invalid UUID: " ^ uuid); - let take_hex count digits = - let rec loop acc remaining rest = - if remaining = 0 then acc, rest - else - match rest with - | [] -> invalid_arg ("invalid UUID: " ^ uuid) - | ch :: rest -> - loop - (Int64.logor (Int64.shift_left acc 4) (Int64.of_int (hex_value ch))) - (remaining - 1) - rest - in - loop Int64.zero count digits - in - let most, rest = take_hex 16 digits in - let least, _ = take_hex 16 rest in - most, least - -let int64_low_i32 value = - Int64.logand value 0xffffffffL |> Int64.to_int |> i32 - -let int64_high_i32 value = - Int64.shift_right_logical value 32 |> int64_low_i32 - -let java_uuid_hash uuid = - let most, least = uuid_halves uuid in - i32_xor - (i32_xor (int64_high_i32 most) (int64_low_i32 most)) - (i32_xor (int64_high_i32 least) (int64_low_i32 least)) - |> i32_to_int - -let clojure_hash_combine seed hash = - i32_xor - (i32 seed) - (i32_add - (i32_add (i32 hash) (i32 (-1640531527))) - (i32_add (i32_shift_left (i32 seed) 6) (i32_shift_right (i32 seed) 2))) - |> i32_to_int - -let clojure_symbol_hash symbol = - let namespace, name = split_keyword symbol in - let namespace_hash = if namespace = "" then 0 else java_string_hash namespace in - clojure_hash_combine (murmur3_hash_unencoded_chars name) namespace_hash - -let clojure_keyword_hash name = - i32_add (i32 (clojure_symbol_hash name)) (i32 (-1640531527)) |> i32_to_int - -let murmur3_mix_coll_hash hash count = - hash - |> i32 - |> murmur3_mix_k1 - |> murmur3_mix_h1 Int32.zero - |> fun hash -> murmur3_fmix hash count - |> i32_to_int - -let murmur3_hash_ordered hashes = - let count, hash = - List.fold_left - (fun (count, hash) value_hash -> - count + 1, i32_add (i32_mul (i32 31) hash) (i32 value_hash)) - (0, i32 1) - hashes - in - murmur3_mix_coll_hash (i32_to_int hash) count - -let murmur3_hash_unordered hashes = - let count, hash = - List.fold_left - (fun (count, hash) value_hash -> count + 1, i32_add hash (i32 value_hash)) - (0, Int32.zero) - hashes - in - murmur3_mix_coll_hash (i32_to_int hash) count - -let rec clojure_hasheq = function - | Nil -> 0 - | Bool true -> 1231 - | Bool false -> 1237 - | Int value -> murmur3_hash_long (Int64.of_int value) - | Float value -> Hashtbl.hash value - | String value -> murmur3_hash_int (java_string_hash value) - | Symbol value -> clojure_symbol_hash value - | Keyword value -> clojure_keyword_hash value - | List values | Vector values -> murmur3_hash_ordered (List.map clojure_hasheq values) - | Set values -> murmur3_hash_unordered (List.map clojure_hasheq values) - | Map entries -> - entries - |> List.map (fun (key, value) -> murmur3_hash_ordered [ clojure_hasheq key; clojure_hasheq value ]) - |> murmur3_hash_unordered - | Tuple values -> - values - |> List.map (function None -> 0 | Some value -> clojure_hasheq value) - |> murmur3_hash_ordered - | Ref value -> murmur3_hash_long (Int64.of_int value) - | Uuid value -> java_uuid_hash value - | Instant value -> murmur3_hash_long (Int64.of_int value) - | Regex value -> Hashtbl.hash value - | TxRef -> Hashtbl.hash TxRef - | Ref_to value -> Hashtbl.hash (Ref_to value) - -let value_type_rank = function - | Nil -> 0 - | Keyword _ -> 1 - | Symbol _ -> 2 - | Map _ -> 3 - | Set _ -> 4 - | List _ -> 5 - | Vector _ -> 6 - | Tuple _ -> 7 - | Bool _ -> 8 - | Int _ | Float _ | Ref _ -> 9 - | String _ -> 10 - | Regex _ -> 11 - | Instant _ -> 12 - | Uuid _ -> 13 - | TxRef -> 14 - | Ref_to _ -> 15 - -let rec compare_value left right = - match left, right with - | Int left, Int right -> compare left right - | Float left, Float right -> compare left right - | Int left, Float right -> compare (float_of_int left) right - | Float left, Int right -> compare left (float_of_int right) - | Ref left, Ref right -> compare left right - | Int left, Ref right -> compare left right - | Ref left, Int right -> compare left right - | Float left, Ref right -> compare left (float_of_int right) - | Ref left, Float right -> compare (float_of_int left) right - | String left, String right -> compare left right - | Symbol left, Symbol right -> compare (split_keyword left) (split_keyword right) - | Bool left, Bool right -> compare left right - | Uuid left, Uuid right -> compare left right - | Instant left, Instant right -> compare left right - | Regex left, Regex right -> compare left right - | Nil, Nil -> 0 - | Keyword left, Keyword right -> compare (split_keyword left) (split_keyword right) - | List left, List right -> compare_list_with compare_value left right - | Vector left, Vector right -> compare_list_with compare_value left right - | List left, Tuple right -> - compare_list_with (compare_option_with compare_value) (List.map (fun value -> Some value) left) right - | Set _, Set _ -> compare (clojure_hasheq left) (clojure_hasheq right) - | Map _, Map _ -> compare (clojure_hasheq left) (clojure_hasheq right) - | Tuple left, Tuple right -> compare_list_with (compare_option_with compare_value) left right - | Tuple left, List right -> - compare_list_with (compare_option_with compare_value) left (List.map (fun value -> Some value) right) - | _ -> - let rank_comparison = compare (value_type_rank left) (value_type_rank right) in - if rank_comparison <> 0 then rank_comparison else compare left right - -and compare_map_entry (left_key, left_value) (right_key, right_value) = - let comparison = compare_value left_key right_key in - if comparison <> 0 then comparison else compare_value left_value right_value +let compare_list_with = Datascript_types.Compare.compare_list_with +let compare_option_with = Datascript_types.Compare.compare_option_with +let split_keyword = Datascript_types.Compare.split_keyword +let compare_value = Datascript_types.Compare.compare_value +let compare_datom = Datascript_types.Compare.compare_datom +let compare_map_entry = Datascript_types.Compare.compare_map_entry let first_nonzero comparisons = List.find_opt (( <> ) 0) comparisons |> Option.value ~default:0 -let first_nonzero4 first second third fourth = - if first <> 0 then first - else if second <> 0 then second - else if third <> 0 then third - else fourth - -let compare_datom index left right = - match index with - | Eavt -> - first_nonzero4 - (compare left.e right.e) - (compare left.a right.a) - (compare_value left.v right.v) - (compare left.tx right.tx) - | Aevt -> - first_nonzero4 - (compare left.a right.a) - (compare left.e right.e) - (compare_value left.v right.v) - (compare left.tx right.tx) - | Avet -> - first_nonzero4 - (compare left.a right.a) - (compare_value left.v right.v) - (compare left.e right.e) - (compare left.tx right.tx) +let first_nonzero4 = Datascript_types.Compare.first_nonzero4 let rec normalize_value = function | List values -> List (List.map normalize_value values) diff --git a/impl/util.mli b/impl/util.mli index b09ff8b..04bc8a4 100644 --- a/impl/util.mli +++ b/impl/util.mli @@ -8,6 +8,7 @@ val compare_list_with : ('a -> 'a -> int) -> 'a list -> 'a list -> int val compare_option_with : ('a -> 'a -> int) -> 'a option -> 'a option -> int val compare_value : value -> value -> int val first_nonzero : int list -> int +val first_nonzero4 : int -> int -> int -> int -> int val compare_datom : index -> datom -> datom -> int val normalize_value : value -> value val normalize_datom_value : datom -> datom diff --git a/lmdb/datascript_lmdb.ml b/lmdb/datascript_lmdb.ml new file mode 100644 index 0000000..300cfe3 --- /dev/null +++ b/lmdb/datascript_lmdb.ml @@ -0,0 +1,89 @@ +module Ds = Datascript +open Lmdb + +type session = + { path : string + ; env : Env.t + ; map : (string, string, [ `Uni ]) Map.t + ; mutable closed : bool + } + +let kvs_map_name = "kvs" +let default_map_size = 1024 * 1024 * 1024 + +let lock_path path = path ^ "-lock" + +let remove_files path = + if Sys.file_exists path then Sys.remove path; + let lock = lock_path path in + if Sys.file_exists lock then Sys.remove lock + +let ensure_open session = + if session.closed then invalid_arg "LMDB session is closed" + +let open_env db_path = + Env.(create Rw ~flags:Flags.no_subdir ~map_size:default_map_size ~max_maps:8 db_path) + +let open_map env = + try Map.open_existing Nodup ~key:Conv.string ~value:Conv.string ~name:kvs_map_name env + with Not_found -> + Map.create Nodup ~key:Conv.string ~value:Conv.string ~name:kvs_map_name env + +let open_session db_path = + remove_files db_path; + let env = open_env db_path in + let map = open_map env in + { path = db_path; env; map; closed = false } + +let close session = + if not session.closed then ( + Map.close session.map; + Env.sync session.env; + Env.close session.env; + session.closed <- true) + +let encode_payload payload = Datascript_sqlite_codec.encode_storage_payload payload + +let decode_payload content = Datascript_sqlite_codec.decode_storage_payload content + +let storage session : Ds.storage = + { storage_store = + (fun entries -> + ensure_open session; + ignore + (Txn.go Rw session.env (fun txn -> + List.iter + (fun (address, payload) -> + Map.set ~txn session.map address (encode_payload payload)) + entries; + None))) + ; storage_restore = + (fun address -> + ensure_open session; + (try Some (Map.get session.map address |> decode_payload) + with Not_found -> None)) + ; storage_list_addresses = + (fun () -> + ensure_open session; + let addresses = ref [] in + let next = Map.to_dispenser session.map in + let rec loop () = + match next () with + | None -> () + | Some (address, _) -> + addresses := address :: !addresses; + loop () + in + loop (); + List.rev !addresses) + ; storage_delete = + (fun addresses -> + ensure_open session; + ignore + (Txn.go Rw session.env (fun txn -> + List.iter + (fun address -> + try Map.remove ~txn session.map address with Not_found -> ()) + addresses; + None))) + } diff --git a/lmdb/datascript_lmdb_codec.ml b/lmdb/datascript_lmdb_codec.ml new file mode 100644 index 0000000..dfc7b12 --- /dev/null +++ b/lmdb/datascript_lmdb_codec.ml @@ -0,0 +1,310 @@ +open Datascript_types + +let value_payload_cache = Hashtbl.create 256 + +let int32_be value = + let value = Int32.of_int value in + String.init 4 (fun index -> + let shift = (3 - index) * 8 in + Char.chr (Int32.to_int (Int32.shift_right_logical value shift) land 0xff)) + +let int32_of_be bytes = + if String.length bytes <> 4 then invalid_arg "invalid int32 key segment"; + let byte index = Char.code bytes.[index] in + Int32.of_int + ((byte 0 lsl 24) lor (byte 1 lsl 16) lor (byte 2 lsl 8) lor byte 3) + |> Int32.to_int + +let int64_be_int64 value = + String.init 8 (fun index -> + let shift = (7 - index) * 8 in + Char.chr (Int64.to_int (Int64.shift_right_logical value shift) land 0xff)) + +let int64_of_be bytes = + if String.length bytes <> 8 then invalid_arg "invalid int64 key segment"; + let byte index = Char.code bytes.[index] in + List.fold_left + (fun acc index -> Int64.logor (Int64.shift_left acc 8) (Int64.of_int (byte index))) + 0L + [ 0; 1; 2; 3; 4; 5; 6; 7 ] + +let append_bytes buffer chunk = Buffer.add_string buffer chunk + +let append_int32 buffer value = append_bytes buffer (int32_be value) + +let append_int64 buffer value = append_bytes buffer (int64_be_int64 value) + +let float_sort_bits value = + let bits = Int64.bits_of_float value in + if Int64.compare bits 0L < 0 then Int64.logxor bits 0x7fffffffffffffffL else bits + +let append_string buffer text = + Buffer.add_string buffer text; + Buffer.add_char buffer '\000' + +let append_byte buffer value = Buffer.add_char buffer (Char.chr value) + +let read_int32 key offset = + if offset + 4 > String.length key then invalid_arg "truncated int32"; + int32_of_be (String.sub key offset 4), offset + 4 + +let read_string key offset = + let len = String.length key in + if offset >= len then invalid_arg "truncated string"; + let rec find_end index = + if index >= len then invalid_arg "unterminated string" + else if key.[index] = '\000' then index + else find_end (index + 1) + in + let end_offset = find_end offset in + String.sub key offset (end_offset - offset), end_offset + 1 + +let read_byte key offset = + if offset >= String.length key then invalid_arg "truncated byte"; + Char.code key.[offset], offset + 1 + +let encode_keyword_like tag text = + let namespace, name = Datascript_types.Compare.split_keyword text in + let buffer = Buffer.create (String.length text + 16) in + append_byte buffer tag; + append_string buffer namespace; + append_string buffer name; + Buffer.contents buffer + +let encode_tagged_hash tag value = + let buffer = Buffer.create 8 in + append_byte buffer tag; + append_int32 buffer (Datascript_types.Compare.clojure_hasheq value); + Buffer.contents buffer + +let rec encode_value_key = function + | Nil -> "\000" + | Keyword value -> encode_keyword_like 1 value + | Symbol value -> encode_keyword_like 2 value + | Map _ as value -> encode_tagged_hash 3 value + | Set _ as value -> encode_tagged_hash 4 value + | List values -> + let buffer = Buffer.create 64 in + append_byte buffer 5; + append_int32 buffer (List.length values); + List.iter (fun value -> append_bytes buffer (encode_value_key value)) values; + Buffer.contents buffer + | Vector values -> + let buffer = Buffer.create 64 in + append_byte buffer 6; + append_int32 buffer (List.length values); + List.iter (fun value -> append_bytes buffer (encode_value_key value)) values; + Buffer.contents buffer + | Tuple values -> + let buffer = Buffer.create 64 in + append_byte buffer 7; + append_int32 buffer (List.length values); + List.iter + (function + | None -> append_byte buffer 0 + | Some value -> + append_byte buffer 1; + append_bytes buffer (encode_value_key value)) + values; + Buffer.contents buffer + | Bool false -> "\008\000" + | Bool true -> "\008\001" + | Int value -> + let buffer = Buffer.create 16 in + append_byte buffer 9; + append_int64 buffer (float_sort_bits (float_of_int value)); + Buffer.contents buffer + | Float value -> + let buffer = Buffer.create 16 in + append_byte buffer 9; + append_int64 buffer (float_sort_bits value); + Buffer.contents buffer + | Ref value -> + let buffer = Buffer.create 16 in + append_byte buffer 9; + append_int64 buffer (float_sort_bits (float_of_int value)); + Buffer.contents buffer + | String value -> + let buffer = Buffer.create (String.length value + 8) in + append_byte buffer 10; + append_string buffer value; + Buffer.contents buffer + | Regex value -> + let buffer = Buffer.create (String.length value + 8) in + append_byte buffer 11; + append_string buffer value; + Buffer.contents buffer + | Instant value -> + let buffer = Buffer.create 16 in + append_byte buffer 12; + append_int32 buffer value; + Buffer.contents buffer + | Uuid value -> + let buffer = Buffer.create (String.length value + 8) in + append_byte buffer 13; + append_string buffer value; + Buffer.contents buffer + | TxRef -> "\014" + | Ref_to value -> + let buffer = Buffer.create 32 in + append_byte buffer 15; + append_int32 buffer (Hashtbl.hash value); + Buffer.contents buffer + +let rec decode_value_key bytes offset = + let tag, offset = read_byte bytes offset in + match tag with + | 0 -> Nil, offset + | 1 -> + let namespace, offset = read_string bytes offset in + let name, offset = read_string bytes offset in + (if namespace = "" then Keyword name else Keyword (namespace ^ "/" ^ name)), offset + | 2 -> + let namespace, offset = read_string bytes offset in + let name, offset = read_string bytes offset in + (if namespace = "" then Symbol name else Symbol (namespace ^ "/" ^ name)), offset + | 3 | 4 as tag -> + let _, offset = read_int32 bytes offset in + (if tag = 3 then Map [] else Set []), offset + | 5 | 6 as tag -> + let count, offset = read_int32 bytes offset in + if count < 0 then invalid_arg "invalid list length"; + let rec loop remaining offset acc = + if remaining = 0 then + (if tag = 5 then List (List.rev acc) else Vector (List.rev acc)), offset + else + let value, offset = decode_value_key bytes offset in + loop (remaining - 1) offset (value :: acc) + in + loop count offset [] + | 7 -> + let count, offset = read_int32 bytes offset in + if count < 0 then invalid_arg "invalid tuple length"; + let rec loop remaining offset acc = + if remaining = 0 then Tuple (List.rev acc), offset + else + let marker, offset = read_byte bytes offset in + let value, offset = + match marker with + | 0 -> None, offset + | 1 -> + let value, offset = decode_value_key bytes offset in + Some value, offset + | _ -> invalid_arg "invalid tuple slot marker" + in + loop (remaining - 1) offset (value :: acc) + in + loop count offset [] + | 8 -> + let value, offset = read_byte bytes offset in + (match value with 0 -> Bool false | 1 -> Bool true | _ -> invalid_arg "invalid bool key"), offset + | 9 -> + let bits, offset = + if offset + 8 > String.length bytes then invalid_arg "truncated numeric key" + else int64_of_be (String.sub bytes offset 8), offset + 8 + in + let raw = + if Int64.compare bits 0L < 0 then Int64.logxor bits 0x7fffffffffffffffL else bits + in + Float (Int64.float_of_bits raw), offset + | 10 -> + let value, offset = read_string bytes offset in + String value, offset + | 11 -> + let value, offset = read_string bytes offset in + Regex value, offset + | 12 -> + let value, offset = read_int32 bytes offset in + Instant value, offset + | 13 -> + let value, offset = read_string bytes offset in + Uuid value, offset + | 14 -> TxRef, offset + | 15 -> Ref_to (Entity_id 0), offset + 4 + | _ -> invalid_arg "invalid value key tag" + +let encode_index_attr_value_prefix index attr value = + let buffer = Buffer.create 64 in + (match index with + | Avet -> + append_string buffer attr; + append_bytes buffer (encode_value_key value) + | Aevt -> + append_string buffer attr; + append_int32 buffer 0; + append_bytes buffer (encode_value_key value) + | Eavt -> + append_int32 buffer 0; + append_string buffer attr; + append_bytes buffer (encode_value_key value)); + Buffer.contents buffer + +let encode_datom_key index datom = + let buffer = Buffer.create 64 in + (match index with + | Eavt -> + append_int32 buffer datom.e; + append_string buffer datom.a; + append_bytes buffer (encode_value_key datom.v); + append_int32 buffer datom.tx + | Aevt -> + append_string buffer datom.a; + append_int32 buffer datom.e; + append_bytes buffer (encode_value_key datom.v); + append_int32 buffer datom.tx + | Avet -> + append_string buffer datom.a; + append_bytes buffer (encode_value_key datom.v); + append_int32 buffer datom.e; + append_int32 buffer datom.tx); + Buffer.contents buffer + +let decode_datom_key index bytes = + let e, a, v, tx = + match index with + | Eavt -> + let e, offset = read_int32 bytes 0 in + let a, offset = read_string bytes offset in + let v, offset = decode_value_key bytes offset in + let tx, offset = read_int32 bytes offset in + if offset <> String.length bytes then invalid_arg "trailing eavt key bytes"; + e, a, v, tx + | Aevt -> + let a, offset = read_string bytes 0 in + let e, offset = read_int32 bytes offset in + let v, offset = decode_value_key bytes offset in + let tx, offset = read_int32 bytes offset in + if offset <> String.length bytes then invalid_arg "trailing aevt key bytes"; + e, a, v, tx + | Avet -> + let a, offset = read_string bytes 0 in + let v, offset = decode_value_key bytes offset in + let e, offset = read_int32 bytes offset in + let tx, offset = read_int32 bytes offset in + if offset <> String.length bytes then invalid_arg "trailing avet key bytes"; + e, a, v, tx + in + { e; a; v; tx; added = true } + +let encode_datom_value datom = + let cache_key = (datom.added, datom.v) in + match Hashtbl.find_opt value_payload_cache cache_key with + | Some encoded -> encoded + | None -> + let encoded = Marshal.to_string cache_key [] in + Hashtbl.add value_payload_cache cache_key encoded; + encoded + +let decode_datom_value bytes = + let added, v = Marshal.from_string bytes 0 in + { e = 0; a = ""; v; tx = 0; added } + +let compare_encoded_keys index left right = + Datascript_types.Compare.compare_datom index + (decode_datom_key index left) + (decode_datom_key index right) + +let encode_schema schema = Marshal.to_string schema [] +let decode_schema bytes = Marshal.from_string bytes 0 +let encode_datoms datoms = Marshal.to_string datoms [] +let decode_datoms bytes = Marshal.from_string bytes 0 diff --git a/lmdb/datascript_lmdb_codec.mli b/lmdb/datascript_lmdb_codec.mli new file mode 100644 index 0000000..39fb371 --- /dev/null +++ b/lmdb/datascript_lmdb_codec.mli @@ -0,0 +1,14 @@ +open Datascript_types + +val encode_datom_key : index -> datom -> string +val encode_index_attr_value_prefix : index -> string -> value -> string +val decode_datom_key : index -> string -> datom +val encode_datom_value : datom -> string +val decode_datom_value : string -> datom + +val compare_encoded_keys : index -> string -> string -> int + +val encode_schema : schema -> string +val decode_schema : string -> schema +val encode_datoms : datom list -> string +val decode_datoms : string -> datom list diff --git a/lmdb/datascript_lmdb_db.mli b/lmdb/datascript_lmdb_db.mli new file mode 100644 index 0000000..d53a813 --- /dev/null +++ b/lmdb/datascript_lmdb_db.mli @@ -0,0 +1,16 @@ +open Datascript_types + +type t + +val create_temp : unit -> t +val open_path : string -> t +val close : t -> unit +val sync : t -> unit +val remove_path : string -> unit + +val meta_get : t -> string -> string option +val meta_set : t -> string -> string -> unit + +val fold_index : index -> t -> (string -> string -> unit) -> unit +val put_index : index -> t -> string -> string -> unit +val remove_index : index -> t -> string -> unit diff --git a/lmdb/datascript_lmdb_db_melange.ml b/lmdb/datascript_lmdb_db_melange.ml new file mode 100644 index 0000000..459980c --- /dev/null +++ b/lmdb/datascript_lmdb_db_melange.ml @@ -0,0 +1,88 @@ +open Datascript_types + +type js = Js.t + +external open_root : string -> js = "open" + [@@mel.module "./datascript_lmdb_node.js"] + +external open_subdb : js -> string -> js = "openDB" + [@@mel.module "./datascript_lmdb_node.js"] + +external js_get : js -> string -> string Js.nullable = "get" + [@@mel.module "./datascript_lmdb_node.js"] + +external js_put : js -> string -> string -> unit = "put" + [@@mel.module "./datascript_lmdb_node.js"] + +external js_remove : js -> string -> unit = "remove" + [@@mel.module "./datascript_lmdb_node.js"] + +external js_sync : js -> unit = "sync" + [@@mel.module "./datascript_lmdb_node.js"] + +external js_close : js -> unit = "close" + [@@mel.module "./datascript_lmdb_node.js"] + +external js_range : js -> (string * string) array = "range" + [@@mel.module "./datascript_lmdb_node.js"] + +external temp_path : unit -> string = "tempPath" + [@@mel.module "./datascript_lmdb_node.js"] + +let remove_path _path = () + +let open_db path = + let root = open_root path in + { Datascript_lmdb_db.path; env = root; eavt = open_subdb root "ds/eavt" + ; aevt = open_subdb root "ds/aevt"; avet = open_subdb root "ds/avet" + ; meta = open_subdb root "ds/meta"; closed = false + } + +let create_temp () = open_db (temp_path ()) + +let open_path path = open_db path + +let ensure_open db = + if db.closed then invalid_arg "LMDB database is closed" + +let close db = + if not db.closed then ( + js_close db.env; + db.closed <- true) + +let sync db = + ensure_open db; + js_sync db.env + +let map_for_index index db = + match index with + | Eavt -> db.eavt + | Aevt -> db.aevt + | Avet -> db.avet + +let meta_get db key = + ensure_open db; + match Js.Nullable.toOption (js_get db.meta key) with + | None -> None + | Some value -> Some value + +let meta_set db key value = + ensure_open db; + js_put db.meta key value + +let with_write db f = + ensure_open db; + f () + +let fold_index index db f = + ensure_open db; + let map = map_for_index index db in + Array.iter (fun (key, value) -> f key value) (js_range map) + +let put_index index db key value = + ensure_open db; + js_put (map_for_index index db) key value + +let remove_index index db key = + ensure_open db; + js_remove (map_for_index index db) key diff --git a/lmdb/datascript_lmdb_index.ml b/lmdb/datascript_lmdb_index.ml new file mode 100644 index 0000000..35cd033 --- /dev/null +++ b/lmdb/datascript_lmdb_index.ml @@ -0,0 +1,106 @@ +open Datascript_types + +type t = { db : Datascript_lmdb_db.t; which : index } + +type 'a seq = { cmp : datom -> datom -> int; datoms : datom list; offset : int } + +let db_of t = t.db +let make index db = { db; which = index } +let cmp_for index = Datascript_types.Compare.compare_datom index + +let decode_entry index key value = + let datom = Datascript_lmdb_codec.decode_datom_key index key in + let payload = Datascript_lmdb_codec.decode_datom_value value in + { datom with added = payload.added; v = payload.v } + +let put_datom t datom = + let key = Datascript_lmdb_codec.encode_datom_key t.which datom in + let value = Datascript_lmdb_codec.encode_datom_value datom in + Datascript_lmdb_db.put_index t.which t.db key value + +let remove_datom t datom = + let key = Datascript_lmdb_codec.encode_datom_key t.which datom in + Datascript_lmdb_db.remove_index t.which t.db key + +let empty index db = make index db + +let of_sorted_list index datoms db = + let t = empty index db in + List.iter (put_datom t) datoms; + t + +let add datom t = + put_datom t datom; + t + +let remove datom t = + remove_datom t datom; + t + +let collect_datoms t = + let datoms = ref [] in + Datascript_lmdb_db.fold_index t.which t.db (fun key value -> + datoms := decode_entry t.which key value :: !datoms); + List.rev !datoms + +let to_list t = collect_datoms t +let fold f init t = List.fold_left f init (to_list t) + +let in_range cmp lower upper datom = + let above_lower = + match lower with + | None -> true + | Some lower -> cmp datom lower >= 0 + in + let below_upper = + match upper with + | None -> true + | Some upper -> cmp datom upper <= 0 + in + above_lower && below_upper + +let make_seq ?(cmp = cmp_for Eavt) ?from_ ?to_ datoms = + let datoms = List.filter (in_range cmp from_ to_) datoms in + { cmp; datoms; offset = 0 } + +let to_seq ({ datoms; offset } as seq) = + let rec loop index () = + if index >= List.length datoms then Seq.Nil + else Seq.Cons (List.nth datoms index, loop (index + 1)) + in + loop seq.offset + +let seq t = make_seq ~cmp:(cmp_for t.which) (to_list t) + +let slice_seq ?from_ ?to_ ?cmp t = + let cmp = Option.value ~default:(cmp_for t.which) cmp in + make_seq ~cmp ?from_ ?to_ (to_list t) + +let rslice_seq ?from_ ?to_ ?cmp t = + let cmp = Option.value ~default:(cmp_for t.which) cmp in + let datoms = + to_list t + |> List.filter (fun datom -> + match from_ with + | None -> true + | Some bound -> cmp datom bound <= 0) + |> List.filter (fun datom -> + match to_ with + | None -> true + | Some bound -> cmp datom bound >= 0) + |> List.rev + in + make_seq ~cmp datoms + +let slice ?from_ ?to_ ?cmp t = slice_seq ?from_ ?to_ ?cmp t |> seq_to_list + +let seq_to_list seq = to_seq seq |> List.of_seq +let fold_seq f init seq = List.fold_left f init (seq_to_list seq) + +let seek bound seq = + let rec count index = + if index >= List.length seq.datoms then index + else if seq.cmp (List.nth seq.datoms index) bound >= 0 then index + else count (index + 1) + in + { seq with offset = count 0 } diff --git a/lmdb/datascript_lmdb_index.mli b/lmdb/datascript_lmdb_index.mli new file mode 100644 index 0000000..9fe6431 --- /dev/null +++ b/lmdb/datascript_lmdb_index.mli @@ -0,0 +1,30 @@ +open Datascript_types + +type t +type 'a seq + +val db_of : t -> Datascript_lmdb_db.t + +val empty : index -> Datascript_lmdb_db.t -> t +val of_sorted_list : index -> datom list -> Datascript_lmdb_db.t -> t + +val add : datom -> t -> t +val remove : datom -> t -> t + +val to_list : t -> datom list +val fold : ('acc -> datom -> 'acc) -> 'acc -> t -> 'acc + +val slice : + ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom list + +val slice_seq : + ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom seq + +val rslice_seq : + ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom seq + +val seq : t -> datom seq +val seq_to_list : datom seq -> datom list +val fold_seq : ('acc -> datom -> 'acc) -> 'acc -> datom seq -> 'acc +val to_seq : datom seq -> datom Seq.t +val seek : datom -> datom seq -> datom seq diff --git a/lmdb/datascript_lmdb_node.js b/lmdb/datascript_lmdb_node.js new file mode 100644 index 0000000..f144c4f --- /dev/null +++ b/lmdb/datascript_lmdb_node.js @@ -0,0 +1,51 @@ +// Node LMDB bindings for Melange. Requires the `lmdb` npm package. +const { open } = require("lmdb"); +const fs = require("fs"); +const os = require("os"); +const path = require("path"); + +function removePath(dbPath) { + if (fs.existsSync(dbPath)) fs.rmSync(dbPath, { recursive: true, force: true }); +} + +exports.open = function (dbPath) { + removePath(dbPath); + return open({ path: dbPath, compression: false }); +}; + +exports.openDB = function (root, name) { + return root.openDB(name, {}); +}; + +exports.get = function (db, key) { + return db.get(key); +}; + +exports.put = function (db, key, value) { + db.put(key, value); +}; + +exports.remove = function (db, key) { + db.remove(key); +}; + +exports.sync = function (_root) {}; + +exports.close = function (root) { + root.close(); +}; + +exports.range = function (db) { + const entries = []; + for (const { key, value } of db.getRange()) { + entries.push([key, value]); + } + return entries; +}; + +exports.tempPath = function () { + return path.join( + os.tmpdir(), + "datascript_lmdb_" + Date.now() + "_" + Math.random().toString(16).slice(2) + ); +}; diff --git a/lmdb/datascript_storage_lmdb.ml b/lmdb/datascript_storage_lmdb.ml new file mode 100644 index 0000000..405958e --- /dev/null +++ b/lmdb/datascript_storage_lmdb.ml @@ -0,0 +1,80 @@ +open Datascript_types + +type t = Datascript_lmdb_db.t + +let registry : (storage, t) Hashtbl.t = Hashtbl.create 16 + +let lmdb storage = + match Hashtbl.find_opt registry storage with + | Some lmdb -> lmdb + | None -> invalid_arg "storage is not LMDB-backed" + +let register storage lmdb = Hashtbl.replace registry storage lmdb + +let create_temp () = Datascript_lmdb_db.create_temp () +let open_path path = Datascript_lmdb_db.open_path path +let close = Datascript_lmdb_db.close +let sync = Datascript_lmdb_db.sync + +let meta_get = Datascript_lmdb_db.meta_get +let meta_set = Datascript_lmdb_db.meta_set + +let meta_schema_key = "schema" +let meta_max_eid_key = "max_eid" +let meta_max_tx_key = "max_tx" +let meta_duplicates_key = "duplicate_datoms" + +let wrap lmdb = + let storage = + { storage_store = + (fun _entries -> sync lmdb) + ; storage_restore = + (fun address -> + if String.equal address "lmdb" then Some Storage_session else None) + ; storage_list_addresses = (fun () -> [ "lmdb" ]) + ; storage_delete = (fun _addresses -> ()) + } + in + register storage lmdb; + storage + +let memory_storage () = wrap (create_temp ()) + +let encode_int value = + Datascript_lmdb_codec.encode_datoms + [ { e = value; a = ""; v = Nil; tx = 0; added = true } ] + +let decode_int bytes = + match Datascript_lmdb_codec.decode_datoms bytes with + | { e; _ } :: _ -> e + | [] -> 0 + +let store_meta lmdb db = + meta_set lmdb meta_schema_key (Datascript_lmdb_codec.encode_schema db.schema); + meta_set lmdb meta_max_eid_key (encode_int db.max_eid); + meta_set lmdb meta_max_tx_key (encode_int db.max_tx); + meta_set lmdb meta_duplicates_key (Datascript_lmdb_codec.encode_datoms db.duplicate_datoms); + sync lmdb + +let restore_meta lmdb = + let schema = + match meta_get lmdb meta_schema_key with + | None -> [] + | Some bytes -> Datascript_lmdb_codec.decode_schema bytes + in + let max_eid = + match meta_get lmdb meta_max_eid_key with + | None -> 0 + | Some bytes -> decode_int bytes + in + let max_tx = + match meta_get lmdb meta_max_tx_key with + | None -> 0x20000000 + | Some bytes -> decode_int bytes + in + let duplicate_datoms = + match meta_get lmdb meta_duplicates_key with + | None -> [] + | Some bytes -> Datascript_lmdb_codec.decode_datoms bytes + in + schema, max_eid, max_tx, duplicate_datoms diff --git a/lmdb/dune b/lmdb/dune new file mode 100644 index 0000000..38bda56 --- /dev/null +++ b/lmdb/dune @@ -0,0 +1,23 @@ +(library + (name datascript_lmdb_codec) + (public_name datascript-ocaml-native.lmdb-codec) + (wrapped false) + (modes native melange) + (modules datascript_lmdb_codec) + (libraries datascript_types)) + +(library + (name datascript_lmdb) + (public_name datascript-ocaml-native.lmdb) + (wrapped false) + (modes native) + (modules datascript_lmdb) + (libraries + datascript-ocaml-native + datascript_sqlite + lmdb_db_native + storage_lmdb_native + lmdb)) + +(subdir native) +(subdir melange) diff --git a/lmdb/melange/datascript_lmdb_codec.ml b/lmdb/melange/datascript_lmdb_codec.ml new file mode 100644 index 0000000..dfc7b12 --- /dev/null +++ b/lmdb/melange/datascript_lmdb_codec.ml @@ -0,0 +1,310 @@ +open Datascript_types + +let value_payload_cache = Hashtbl.create 256 + +let int32_be value = + let value = Int32.of_int value in + String.init 4 (fun index -> + let shift = (3 - index) * 8 in + Char.chr (Int32.to_int (Int32.shift_right_logical value shift) land 0xff)) + +let int32_of_be bytes = + if String.length bytes <> 4 then invalid_arg "invalid int32 key segment"; + let byte index = Char.code bytes.[index] in + Int32.of_int + ((byte 0 lsl 24) lor (byte 1 lsl 16) lor (byte 2 lsl 8) lor byte 3) + |> Int32.to_int + +let int64_be_int64 value = + String.init 8 (fun index -> + let shift = (7 - index) * 8 in + Char.chr (Int64.to_int (Int64.shift_right_logical value shift) land 0xff)) + +let int64_of_be bytes = + if String.length bytes <> 8 then invalid_arg "invalid int64 key segment"; + let byte index = Char.code bytes.[index] in + List.fold_left + (fun acc index -> Int64.logor (Int64.shift_left acc 8) (Int64.of_int (byte index))) + 0L + [ 0; 1; 2; 3; 4; 5; 6; 7 ] + +let append_bytes buffer chunk = Buffer.add_string buffer chunk + +let append_int32 buffer value = append_bytes buffer (int32_be value) + +let append_int64 buffer value = append_bytes buffer (int64_be_int64 value) + +let float_sort_bits value = + let bits = Int64.bits_of_float value in + if Int64.compare bits 0L < 0 then Int64.logxor bits 0x7fffffffffffffffL else bits + +let append_string buffer text = + Buffer.add_string buffer text; + Buffer.add_char buffer '\000' + +let append_byte buffer value = Buffer.add_char buffer (Char.chr value) + +let read_int32 key offset = + if offset + 4 > String.length key then invalid_arg "truncated int32"; + int32_of_be (String.sub key offset 4), offset + 4 + +let read_string key offset = + let len = String.length key in + if offset >= len then invalid_arg "truncated string"; + let rec find_end index = + if index >= len then invalid_arg "unterminated string" + else if key.[index] = '\000' then index + else find_end (index + 1) + in + let end_offset = find_end offset in + String.sub key offset (end_offset - offset), end_offset + 1 + +let read_byte key offset = + if offset >= String.length key then invalid_arg "truncated byte"; + Char.code key.[offset], offset + 1 + +let encode_keyword_like tag text = + let namespace, name = Datascript_types.Compare.split_keyword text in + let buffer = Buffer.create (String.length text + 16) in + append_byte buffer tag; + append_string buffer namespace; + append_string buffer name; + Buffer.contents buffer + +let encode_tagged_hash tag value = + let buffer = Buffer.create 8 in + append_byte buffer tag; + append_int32 buffer (Datascript_types.Compare.clojure_hasheq value); + Buffer.contents buffer + +let rec encode_value_key = function + | Nil -> "\000" + | Keyword value -> encode_keyword_like 1 value + | Symbol value -> encode_keyword_like 2 value + | Map _ as value -> encode_tagged_hash 3 value + | Set _ as value -> encode_tagged_hash 4 value + | List values -> + let buffer = Buffer.create 64 in + append_byte buffer 5; + append_int32 buffer (List.length values); + List.iter (fun value -> append_bytes buffer (encode_value_key value)) values; + Buffer.contents buffer + | Vector values -> + let buffer = Buffer.create 64 in + append_byte buffer 6; + append_int32 buffer (List.length values); + List.iter (fun value -> append_bytes buffer (encode_value_key value)) values; + Buffer.contents buffer + | Tuple values -> + let buffer = Buffer.create 64 in + append_byte buffer 7; + append_int32 buffer (List.length values); + List.iter + (function + | None -> append_byte buffer 0 + | Some value -> + append_byte buffer 1; + append_bytes buffer (encode_value_key value)) + values; + Buffer.contents buffer + | Bool false -> "\008\000" + | Bool true -> "\008\001" + | Int value -> + let buffer = Buffer.create 16 in + append_byte buffer 9; + append_int64 buffer (float_sort_bits (float_of_int value)); + Buffer.contents buffer + | Float value -> + let buffer = Buffer.create 16 in + append_byte buffer 9; + append_int64 buffer (float_sort_bits value); + Buffer.contents buffer + | Ref value -> + let buffer = Buffer.create 16 in + append_byte buffer 9; + append_int64 buffer (float_sort_bits (float_of_int value)); + Buffer.contents buffer + | String value -> + let buffer = Buffer.create (String.length value + 8) in + append_byte buffer 10; + append_string buffer value; + Buffer.contents buffer + | Regex value -> + let buffer = Buffer.create (String.length value + 8) in + append_byte buffer 11; + append_string buffer value; + Buffer.contents buffer + | Instant value -> + let buffer = Buffer.create 16 in + append_byte buffer 12; + append_int32 buffer value; + Buffer.contents buffer + | Uuid value -> + let buffer = Buffer.create (String.length value + 8) in + append_byte buffer 13; + append_string buffer value; + Buffer.contents buffer + | TxRef -> "\014" + | Ref_to value -> + let buffer = Buffer.create 32 in + append_byte buffer 15; + append_int32 buffer (Hashtbl.hash value); + Buffer.contents buffer + +let rec decode_value_key bytes offset = + let tag, offset = read_byte bytes offset in + match tag with + | 0 -> Nil, offset + | 1 -> + let namespace, offset = read_string bytes offset in + let name, offset = read_string bytes offset in + (if namespace = "" then Keyword name else Keyword (namespace ^ "/" ^ name)), offset + | 2 -> + let namespace, offset = read_string bytes offset in + let name, offset = read_string bytes offset in + (if namespace = "" then Symbol name else Symbol (namespace ^ "/" ^ name)), offset + | 3 | 4 as tag -> + let _, offset = read_int32 bytes offset in + (if tag = 3 then Map [] else Set []), offset + | 5 | 6 as tag -> + let count, offset = read_int32 bytes offset in + if count < 0 then invalid_arg "invalid list length"; + let rec loop remaining offset acc = + if remaining = 0 then + (if tag = 5 then List (List.rev acc) else Vector (List.rev acc)), offset + else + let value, offset = decode_value_key bytes offset in + loop (remaining - 1) offset (value :: acc) + in + loop count offset [] + | 7 -> + let count, offset = read_int32 bytes offset in + if count < 0 then invalid_arg "invalid tuple length"; + let rec loop remaining offset acc = + if remaining = 0 then Tuple (List.rev acc), offset + else + let marker, offset = read_byte bytes offset in + let value, offset = + match marker with + | 0 -> None, offset + | 1 -> + let value, offset = decode_value_key bytes offset in + Some value, offset + | _ -> invalid_arg "invalid tuple slot marker" + in + loop (remaining - 1) offset (value :: acc) + in + loop count offset [] + | 8 -> + let value, offset = read_byte bytes offset in + (match value with 0 -> Bool false | 1 -> Bool true | _ -> invalid_arg "invalid bool key"), offset + | 9 -> + let bits, offset = + if offset + 8 > String.length bytes then invalid_arg "truncated numeric key" + else int64_of_be (String.sub bytes offset 8), offset + 8 + in + let raw = + if Int64.compare bits 0L < 0 then Int64.logxor bits 0x7fffffffffffffffL else bits + in + Float (Int64.float_of_bits raw), offset + | 10 -> + let value, offset = read_string bytes offset in + String value, offset + | 11 -> + let value, offset = read_string bytes offset in + Regex value, offset + | 12 -> + let value, offset = read_int32 bytes offset in + Instant value, offset + | 13 -> + let value, offset = read_string bytes offset in + Uuid value, offset + | 14 -> TxRef, offset + | 15 -> Ref_to (Entity_id 0), offset + 4 + | _ -> invalid_arg "invalid value key tag" + +let encode_index_attr_value_prefix index attr value = + let buffer = Buffer.create 64 in + (match index with + | Avet -> + append_string buffer attr; + append_bytes buffer (encode_value_key value) + | Aevt -> + append_string buffer attr; + append_int32 buffer 0; + append_bytes buffer (encode_value_key value) + | Eavt -> + append_int32 buffer 0; + append_string buffer attr; + append_bytes buffer (encode_value_key value)); + Buffer.contents buffer + +let encode_datom_key index datom = + let buffer = Buffer.create 64 in + (match index with + | Eavt -> + append_int32 buffer datom.e; + append_string buffer datom.a; + append_bytes buffer (encode_value_key datom.v); + append_int32 buffer datom.tx + | Aevt -> + append_string buffer datom.a; + append_int32 buffer datom.e; + append_bytes buffer (encode_value_key datom.v); + append_int32 buffer datom.tx + | Avet -> + append_string buffer datom.a; + append_bytes buffer (encode_value_key datom.v); + append_int32 buffer datom.e; + append_int32 buffer datom.tx); + Buffer.contents buffer + +let decode_datom_key index bytes = + let e, a, v, tx = + match index with + | Eavt -> + let e, offset = read_int32 bytes 0 in + let a, offset = read_string bytes offset in + let v, offset = decode_value_key bytes offset in + let tx, offset = read_int32 bytes offset in + if offset <> String.length bytes then invalid_arg "trailing eavt key bytes"; + e, a, v, tx + | Aevt -> + let a, offset = read_string bytes 0 in + let e, offset = read_int32 bytes offset in + let v, offset = decode_value_key bytes offset in + let tx, offset = read_int32 bytes offset in + if offset <> String.length bytes then invalid_arg "trailing aevt key bytes"; + e, a, v, tx + | Avet -> + let a, offset = read_string bytes 0 in + let v, offset = decode_value_key bytes offset in + let e, offset = read_int32 bytes offset in + let tx, offset = read_int32 bytes offset in + if offset <> String.length bytes then invalid_arg "trailing avet key bytes"; + e, a, v, tx + in + { e; a; v; tx; added = true } + +let encode_datom_value datom = + let cache_key = (datom.added, datom.v) in + match Hashtbl.find_opt value_payload_cache cache_key with + | Some encoded -> encoded + | None -> + let encoded = Marshal.to_string cache_key [] in + Hashtbl.add value_payload_cache cache_key encoded; + encoded + +let decode_datom_value bytes = + let added, v = Marshal.from_string bytes 0 in + { e = 0; a = ""; v; tx = 0; added } + +let compare_encoded_keys index left right = + Datascript_types.Compare.compare_datom index + (decode_datom_key index left) + (decode_datom_key index right) + +let encode_schema schema = Marshal.to_string schema [] +let decode_schema bytes = Marshal.from_string bytes 0 +let encode_datoms datoms = Marshal.to_string datoms [] +let decode_datoms bytes = Marshal.from_string bytes 0 diff --git a/lmdb/melange/datascript_lmdb_db.ml b/lmdb/melange/datascript_lmdb_db.ml new file mode 100644 index 0000000..1937eaf --- /dev/null +++ b/lmdb/melange/datascript_lmdb_db.ml @@ -0,0 +1,204 @@ +open Datascript_types +open Lmdb + +type t = + { path : string + ; env : Env.t + ; eavt : (string, string, [ `Uni ]) Map.t + ; aevt : (string, string, [ `Uni ]) Map.t + ; avet : (string, string, [ `Uni ]) Map.t + ; meta : (string, string, [ `Uni ]) Map.t + ; mutable closed : bool + } + +let default_map_size = 1024 * 1024 * 1024 +let lock_path path = path ^ "-lock" + +let remove_path path = + if Sys.file_exists path then Sys.remove path; + let lock = lock_path path in + if Sys.file_exists lock then Sys.remove lock + +let open_env db_path = + Env.(create Rw ~flags:Flags.no_subdir ~map_size:default_map_size ~max_maps:8 db_path) + +let open_named_map env name = + try Map.open_existing Nodup ~key:Conv.string ~value:Conv.string ~name env + with Not_found -> Map.create Nodup ~key:Conv.string ~value:Conv.string ~name env + +let open_db path = + remove_path path; + let env = open_env path in + { path; env; eavt = open_named_map env "ds/eavt"; aevt = open_named_map env "ds/aevt" + ; avet = open_named_map env "ds/avet"; meta = open_named_map env "ds/meta"; closed = false + } + +let open_path path = open_db path + +let ensure_open db = + if db.closed then invalid_arg ("LMDB database is closed: " ^ db.path) + +let close db = + if not db.closed then ( + Map.close db.eavt; + Map.close db.aevt; + Map.close db.avet; + Map.close db.meta; + Env.sync db.env; + Env.close db.env; + db.closed <- true) + +let temps_created = ref 0 + +let create_temp () = + let db = + open_db + (Filename.temp_file + ~temp_dir:(Filename.get_temp_dir_name ()) + "datascript_lmdb" + ".mdb") + in + Gc.finalise + (fun lmdb -> + if not lmdb.closed then close lmdb) + db; + incr temps_created; + if !temps_created mod 64 = 0 then Gc.full_major (); + db + +let sync db = + ensure_open db; + Env.sync db.env + +let map_for_index index db = + match index with + | Eavt -> db.eavt + | Aevt -> db.aevt + | Avet -> db.avet + +let meta_get db key = + ensure_open db; + try Some (Map.get db.meta key) with Not_found -> None + +let meta_set db key value = + ensure_open db; + ignore + (Txn.go Rw db.env (fun txn -> + Map.set ~txn db.meta key value; + ())) + +let with_write_txn db f = + ensure_open db; + ignore + (Txn.go Rw db.env (fun txn -> + f txn; + ())) + +let put_index_txn index txn db key value = + Map.set ~txn (map_for_index index db) key value + +let remove_index_txn index txn db key = + try Map.remove ~txn (map_for_index index db) key with Not_found -> () + +let put_index index db key value = + with_write_txn db (fun txn -> put_index_txn index txn db key value) + +let remove_index index db key = + with_write_txn db (fun txn -> remove_index_txn index txn db key) + +let get_index index db key = + ensure_open db; + try Some (Map.get (map_for_index index db) key) with Not_found -> None + +let fold_index index db f = + ensure_open db; + let map = map_for_index index db in + let next = Map.to_dispenser map in + let rec loop () = + match next () with + | None -> () + | Some (key, value) -> + f key value; + loop () + in + loop () + +let fold_index_prefix index db prefix f = + ensure_open db; + let map = map_for_index index db in + let prefix_len = String.length prefix in + (try + Cursor.go Ro map (fun cursor -> + (try ignore (Cursor.seek_range cursor prefix) with Not_found -> raise Exit); + let rec loop () = + let key, value = + try Cursor.current cursor + with Not_found -> raise Exit + in + if String.length key < prefix_len || String.sub key 0 prefix_len <> prefix then raise Exit; + f key value; + try + ignore (Cursor.next cursor); + loop () + with Not_found -> raise Exit + in + loop ()) + with Exit -> ()) + +let fold_index_range index db ?from_key ?to_key f = + ensure_open db; + let map = map_for_index index db in + (try + Cursor.go Ro map (fun cursor -> + (match from_key with + | None -> ( + try ignore (Cursor.first cursor) with Not_found -> raise Exit) + | Some key -> ( + try ignore (Cursor.seek_range cursor key) with Not_found -> raise Exit)); + let rec loop () = + let key, value = + try Cursor.current cursor + with Not_found -> raise Exit + in + (match to_key with + | Some bound when String.compare key bound > 0 -> raise Exit + | _ -> ()); + f key value; + try + ignore (Cursor.next cursor); + loop () + with Not_found -> raise Exit + in + loop ()) + with Exit -> ()) + +let fold_index_range_until index db ?from_key ?stop f = + ensure_open db; + let map = map_for_index index db in + (try + Cursor.go Ro map (fun cursor -> + (match from_key with + | None -> ( + try ignore (Cursor.first cursor) with Not_found -> raise Exit) + | Some key -> ( + try ignore (Cursor.seek_range cursor key) with Not_found -> raise Exit)); + let rec loop () = + let key, value = + try Cursor.current cursor + with Not_found -> raise Exit + in + (match stop with + | Some stop when stop key value -> raise Exit + | _ -> ()); + f key value; + try + ignore (Cursor.next cursor); + loop () + with Not_found -> raise Exit + in + loop ()) + with Exit -> ()) + +let copy_index_txn index txn from_db to_db = + fold_index index from_db (fun key value -> + put_index_txn index txn to_db key value) diff --git a/lmdb/melange/datascript_lmdb_db.mli b/lmdb/melange/datascript_lmdb_db.mli new file mode 100644 index 0000000..d53a813 --- /dev/null +++ b/lmdb/melange/datascript_lmdb_db.mli @@ -0,0 +1,16 @@ +open Datascript_types + +type t + +val create_temp : unit -> t +val open_path : string -> t +val close : t -> unit +val sync : t -> unit +val remove_path : string -> unit + +val meta_get : t -> string -> string option +val meta_set : t -> string -> string -> unit + +val fold_index : index -> t -> (string -> string -> unit) -> unit +val put_index : index -> t -> string -> string -> unit +val remove_index : index -> t -> string -> unit diff --git a/lmdb/melange/datascript_lmdb_index.ml b/lmdb/melange/datascript_lmdb_index.ml new file mode 100644 index 0000000..5b7558e --- /dev/null +++ b/lmdb/melange/datascript_lmdb_index.ml @@ -0,0 +1,282 @@ +open Datascript_types + +type t = + { db : Datascript_lmdb_db.t + ; which : index + ; additions : datom list + ; removals : datom list + } + +type 'a seq = { cmp : datom -> datom -> int; datoms : datom list; offset : int } + +let db_of t = t.db +let make index db = { db; which = index; additions = []; removals = [] } +let cmp_for index = Datascript_types.Compare.compare_datom index +let overlay_empty t = t.additions = [] && t.removals = [] + +let datom_key t datom = Datascript_lmdb_codec.encode_datom_key t.which datom + +let decode_entry index key value = + let datom = Datascript_lmdb_codec.decode_datom_key index key in + let payload = Datascript_lmdb_codec.decode_datom_value value in + { datom with added = payload.added; v = payload.v } + +let put_datom_txn txn t datom = + let key = datom_key t datom in + let value = Datascript_lmdb_codec.encode_datom_value datom in + Datascript_lmdb_db.put_index_txn t.which txn t.db key value + +let remove_datom_txn txn t datom = + let key = datom_key t datom in + Datascript_lmdb_db.remove_index_txn t.which txn t.db key + +let empty index db = make index db + +let of_sorted_list index datoms db = + let t = empty index db in + if datoms = [] then t + else ( + Datascript_lmdb_db.with_write_txn db (fun txn -> + List.iter (put_datom_txn txn t) datoms); + t) + +let of_sorted_lists index_datoms db = + Datascript_lmdb_db.with_write_txn db (fun txn -> + List.iter + (fun (index, datoms) -> + let t = make index db in + List.iter (put_datom_txn txn t) datoms) + index_datoms) + +let of_eavt_datoms ~avet eavt_datoms db = + if eavt_datoms = [] then () + else ( + let eavt = make Eavt db in + let aevt = make Aevt db in + let avet_index = make Avet db in + Datascript_lmdb_db.with_write_txn db (fun txn -> + List.iter + (fun datom -> + put_datom_txn txn eavt datom; + put_datom_txn txn aevt datom; + if avet datom.a then put_datom_txn txn avet_index datom) + eavt_datoms)) + +let of_bulk index datoms db = { db; which = index; additions = datoms; removals = [] } + +let additions_only t = t.additions <> [] && t.removals = [] + +let add datom t = + let key = datom_key t datom in + let additions = datom :: List.filter (fun d -> datom_key t d <> key) t.additions in + let removals = List.filter (fun d -> datom_key t d <> key) t.removals in + { t with additions; removals } + +let remove datom t = + let key = datom_key t datom in + let additions = List.filter (fun d -> datom_key t d <> key) t.additions in + let already_removed = List.exists (fun d -> datom_key t d = key) t.removals in + let removals = + if already_removed || List.exists (fun d -> datom_key t d = key) t.additions then t.removals + else datom :: t.removals + in + { t with additions; removals } + +let overlay_tables t = + let removed = Hashtbl.create (List.length t.removals) in + List.iter (fun datom -> Hashtbl.add removed (datom_key t datom) ()) t.removals; + let added = Hashtbl.create (List.length t.additions) in + List.iter (fun datom -> Hashtbl.replace added (datom_key t datom) datom) t.additions; + removed, added + +let stored_visible key removed added = + not (Hashtbl.mem removed key || Hashtbl.mem added key) + +let in_range cmp lower upper datom = + let above_lower = + match lower with + | None -> true + | Some lower -> cmp datom lower >= 0 + in + let below_upper = + match upper with + | None -> true + | Some upper -> cmp datom upper <= 0 + in + above_lower && below_upper + +let bound_key t = function + | None -> None + | Some datom -> Some (datom_key t datom) + +exception Stop_search + +let fold_stored t f acc = + if overlay_empty t then + let acc = ref acc in + Datascript_lmdb_db.fold_index t.which t.db (fun key value -> + acc := f !acc (decode_entry t.which key value)); + !acc + else + let removed, added = overlay_tables t in + let acc = ref acc in + Datascript_lmdb_db.fold_index t.which t.db (fun key value -> + if stored_visible key removed added then + acc := f !acc (decode_entry t.which key value)); + !acc + +let fold_stored_prefix t attr f acc = + let prefix = attr ^ "\000" in + if overlay_empty t then + let acc = ref acc in + Datascript_lmdb_db.fold_index_prefix t.which t.db prefix (fun key value -> + acc := f !acc (decode_entry t.which key value)); + !acc + else + let removed, added = overlay_tables t in + let acc = ref acc in + Datascript_lmdb_db.fold_index_prefix t.which t.db prefix (fun key value -> + if stored_visible key removed added then + acc := f !acc (decode_entry t.which key value)); + !acc + +let fold_stored_attr_value_prefix t attr value f acc = + let prefix = Datascript_lmdb_codec.encode_index_attr_value_prefix t.which attr value in + if overlay_empty t then + let acc = ref acc in + Datascript_lmdb_db.fold_index_prefix t.which t.db prefix (fun key value -> + acc := f !acc (decode_entry t.which key value)); + !acc + else + let removed, added = overlay_tables t in + let acc = ref acc in + Datascript_lmdb_db.fold_index_prefix t.which t.db prefix (fun key value -> + if stored_visible key removed added then + acc := f !acc (decode_entry t.which key value)); + !acc + +let fold_stored_bounded t ?from_ ?to_ cmp f acc = + match bound_key t from_ with + | None -> fold_stored t f acc + | Some from_key -> + let removed, added = + if overlay_empty t then (Hashtbl.create 0, Hashtbl.create 0) else overlay_tables t + in + let acc = ref acc in + Datascript_lmdb_db.fold_index_range_until t.which t.db ~from_key + ~stop:(fun key value -> + if not (stored_visible key removed added) then false + else + match to_ with + | Some bound -> + let datom = decode_entry t.which key value in + cmp datom bound > 0 + | None -> false) + (fun key value -> + if stored_visible key removed added then + let datom = decode_entry t.which key value in + if in_range cmp from_ to_ datom then acc := f !acc datom); + !acc + +let fold_stored_bounded t ?from_ ?to_ cmp f acc = + let cmp = Option.value ~default:(cmp_for t.which) cmp in + let apply acc datom = if in_range cmp from_ to_ datom then f acc datom else acc in + if not (overlay_empty t) then + collect_datoms t + |> List.filter (fun datom -> in_range cmp from_ to_ datom) + |> List.fold_left f init + else + let acc = + match from_, to_ with + | Some bound, Some bound' when bound == bound' && bound.a <> "" && bound.e = 0 && bound.v = Nil + && (t.which = Aevt || t.which = Avet) -> + fold_stored_prefix t bound.a apply init + | Some bound, Some bound' when bound == bound' && bound.a <> "" && bound.v <> Nil && bound.e = 0 -> + fold_stored_attr_value_prefix t bound.a bound.v apply init + | _ -> fold_stored_bounded t ?from_ ?to_ cmp apply init + in + acc + +let find_first_slice ?from_ ?to_ ?cmp t = + let cmp = Option.value ~default:(cmp_for t.which) cmp in + let found = ref None in + let consider datom = + if !found = None && in_range cmp from_ to_ datom then ( + found := Some datom; + raise Stop_search) + in + (try + if not (overlay_empty t) then + collect_datoms t |> List.iter consider + else + match from_, to_ with + | Some bound, Some bound' when bound == bound' && bound.a <> "" && bound.e = 0 && bound.v = Nil + && (t.which = Aevt || t.which = Avet) -> + fold_stored_prefix t bound.a (fun () datom -> consider datom) () + | Some bound, Some bound' when bound == bound' && bound.a <> "" && bound.v <> Nil && bound.e = 0 -> + fold_stored_attr_value_prefix t bound.a bound.v (fun () datom -> consider datom) () + | _ -> fold_stored_bounded t ?from_ ?to_ cmp (fun () datom -> consider datom) () + with Stop_search -> ()); + !found + +let fold_attr_prefix f init t attr = + let apply acc datom = if datom.a = attr then f acc datom else acc in + if not (overlay_empty t) then + collect_datoms t + |> List.filter (fun datom -> datom.a = attr) + |> List.fold_left f init + else + fold_stored_prefix t attr apply init + +let materialize_range t ?from_ ?to_ cmp = + fold_slice (fun acc datom -> datom :: acc) [] ?from_ ?to_ ~cmp t |> List.rev + +let make_seq cmp datoms = { cmp; datoms; offset = 0 } + +let to_seq ({ cmp = _; datoms; offset = start }) = + let rec loop index () = + if index >= List.length datoms then Seq.Nil + else Seq.Cons (List.nth datoms index, loop (index + 1)) + in + loop start + +let seq t = make_seq (cmp_for t.which) (to_list t) + +let slice_seq ?from_ ?to_ ?cmp t = + let cmp = Option.value ~default:(cmp_for t.which) cmp in + make_seq cmp (materialize_range t ?from_ ?to_ cmp) + +let rslice_seq ?from_ ?to_ ?cmp t = + let cmp = Option.value ~default:(cmp_for t.which) cmp in + let datoms = + to_list t + |> List.filter (fun datom -> + match from_ with + | None -> true + | Some bound -> cmp datom bound <= 0) + |> List.filter (fun datom -> + match to_ with + | None -> true + | Some bound -> cmp datom bound >= 0) + |> List.rev + in + make_seq cmp datoms + +let seq_to_list seq = to_seq seq |> List.of_seq + +let fold_seq f init { cmp = _; datoms; offset } = + let rec loop index acc = + if index >= List.length datoms then acc + else loop (index + 1) (f acc (List.nth datoms index)) + in + loop offset init + +let slice ?from_ ?to_ ?cmp t = slice_seq ?from_ ?to_ ?cmp t |> seq_to_list + +let seek bound seq = + let rec count index = + if index >= List.length seq.datoms then index + else if seq.cmp (List.nth seq.datoms index) bound >= 0 then index + else count (index + 1) + in + { seq with offset = count 0 } diff --git a/lmdb/melange/datascript_lmdb_index.mli b/lmdb/melange/datascript_lmdb_index.mli new file mode 100644 index 0000000..cc85977 --- /dev/null +++ b/lmdb/melange/datascript_lmdb_index.mli @@ -0,0 +1,32 @@ +open Datascript_types + +type t +type 'a seq + +val db_of : t -> Datascript_lmdb_db.t +val empty : index -> Datascript_lmdb_db.t -> t +val of_sorted_list : index -> datom list -> Datascript_lmdb_db.t -> t +val of_sorted_lists : (index * datom list) list -> Datascript_lmdb_db.t -> unit +val of_eavt_datoms : avet:(string -> bool) -> datom list -> Datascript_lmdb_db.t -> unit +val of_bulk : index -> datom list -> Datascript_lmdb_db.t -> t +val add : datom -> t -> t +val remove : datom -> t -> t +val flush : t -> t +val copy : t -> t +val sync_merged_to_lmdb : t -> Datascript_lmdb_db.t -> unit +val lookup : t -> datom -> datom option +val to_list : t -> datom list +val fold : ('acc -> datom -> 'acc) -> 'acc -> t -> 'acc +val fold_slice : + ('acc -> datom -> 'acc) -> 'acc -> ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> 'acc +val find_first_slice : + ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom option +val fold_attr_prefix : ('acc -> datom -> 'acc) -> 'acc -> t -> string -> 'acc +val slice : ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom list +val slice_seq : ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom seq +val rslice_seq : ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom seq +val seq : t -> datom seq +val seq_to_list : datom seq -> datom list +val fold_seq : ('acc -> datom -> 'acc) -> 'acc -> datom seq -> 'acc +val to_seq : datom seq -> datom Seq.t +val seek : datom -> datom seq -> datom seq diff --git a/lmdb/melange/datascript_lmdb_node.js b/lmdb/melange/datascript_lmdb_node.js new file mode 100644 index 0000000..f144c4f --- /dev/null +++ b/lmdb/melange/datascript_lmdb_node.js @@ -0,0 +1,51 @@ +// Node LMDB bindings for Melange. Requires the `lmdb` npm package. +const { open } = require("lmdb"); +const fs = require("fs"); +const os = require("os"); +const path = require("path"); + +function removePath(dbPath) { + if (fs.existsSync(dbPath)) fs.rmSync(dbPath, { recursive: true, force: true }); +} + +exports.open = function (dbPath) { + removePath(dbPath); + return open({ path: dbPath, compression: false }); +}; + +exports.openDB = function (root, name) { + return root.openDB(name, {}); +}; + +exports.get = function (db, key) { + return db.get(key); +}; + +exports.put = function (db, key, value) { + db.put(key, value); +}; + +exports.remove = function (db, key) { + db.remove(key); +}; + +exports.sync = function (_root) {}; + +exports.close = function (root) { + root.close(); +}; + +exports.range = function (db) { + const entries = []; + for (const { key, value } of db.getRange()) { + entries.push([key, value]); + } + return entries; +}; + +exports.tempPath = function () { + return path.join( + os.tmpdir(), + "datascript_lmdb_" + Date.now() + "_" + Math.random().toString(16).slice(2) + ); +}; diff --git a/lmdb/melange/datascript_storage_lmdb.ml b/lmdb/melange/datascript_storage_lmdb.ml new file mode 100644 index 0000000..27eb03a --- /dev/null +++ b/lmdb/melange/datascript_storage_lmdb.ml @@ -0,0 +1,101 @@ +open Datascript_types + +type t = Datascript_lmdb_db.t + +module Storage_registry = struct + type t = storage + + let equal left right = left == right + + let hash storage = Hashtbl.hash (Obj.repr storage) +end + +module Registry = Hashtbl.Make (Storage_registry) + +let registry = Registry.create 16 + +let lmdb storage = + match Registry.find_opt registry storage with + | Some lmdb -> lmdb + | None -> invalid_arg "storage is not LMDB-backed" + +let register storage lmdb = Registry.replace registry storage lmdb + +let create_temp () = Datascript_lmdb_db.create_temp () +let open_path path = Datascript_lmdb_db.open_path path +let close = Datascript_lmdb_db.close +let sync = Datascript_lmdb_db.sync + +let meta_get = Datascript_lmdb_db.meta_get +let meta_set = Datascript_lmdb_db.meta_set + +let meta_schema_key = "schema" +let meta_max_eid_key = "max_eid" +let meta_max_tx_key = "max_tx" +let meta_duplicates_key = "duplicate_datoms" + +let wrap lmdb = + let storage = + { storage_store = + (fun _entries -> sync lmdb) + ; storage_restore = + (fun address -> + if String.equal address "lmdb" then Some Storage_session else None) + ; storage_list_addresses = (fun () -> [ "lmdb" ]) + ; storage_delete = (fun _addresses -> ()) + } + in + register storage lmdb; + storage + +let memory_storage () = wrap (create_temp ()) + +let encode_int value = + Datascript_lmdb_codec.encode_datoms + [ { e = value; a = ""; v = Nil; tx = 0; added = true } ] + +let decode_int bytes = + match Datascript_lmdb_codec.decode_datoms bytes with + | { e; _ } :: _ -> e + | [] -> 0 + +let store_meta lmdb db = + meta_set lmdb meta_schema_key (Datascript_lmdb_codec.encode_schema db.schema); + meta_set lmdb meta_max_eid_key (encode_int db.max_eid); + meta_set lmdb meta_max_tx_key (encode_int db.max_tx); + meta_set lmdb meta_duplicates_key (Datascript_lmdb_codec.encode_datoms db.duplicate_datoms); + sync lmdb + +let restore_meta lmdb = + let schema = + match meta_get lmdb meta_schema_key with + | None -> [] + | Some bytes -> Datascript_lmdb_codec.decode_schema bytes + in + let max_eid = + match meta_get lmdb meta_max_eid_key with + | None -> 0 + | Some bytes -> decode_int bytes + in + let max_tx = + match meta_get lmdb meta_max_tx_key with + | None -> 0x20000000 + | Some bytes -> decode_int bytes + in + let duplicate_datoms = + match meta_get lmdb meta_duplicates_key with + | None -> [] + | Some bytes -> Datascript_lmdb_codec.decode_datoms bytes + in + schema, max_eid, max_tx, duplicate_datoms + +let sync_indexes from_lmdb to_lmdb = + if from_lmdb != to_lmdb then + Datascript_lmdb_db.with_write_txn to_lmdb (fun txn -> + List.iter + (fun index -> + Datascript_lmdb_db.copy_index_txn index txn from_lmdb to_lmdb) + [ Eavt; Aevt; Avet ]) + +let store_db storage db = + store_meta (lmdb storage) db diff --git a/lmdb/melange/dune b/lmdb/melange/dune new file mode 100644 index 0000000..dadefd3 --- /dev/null +++ b/lmdb/melange/dune @@ -0,0 +1,25 @@ +(include_subdirs no) + +(library + (name lmdb_db_melange) + (public_name datascript-ocaml-melange.lmdb-db) + (wrapped false) + (modes melange) + (modules datascript_lmdb_db) + (libraries datascript_lmdb_codec melange.js)) + +(library + (name lmdb_index_melange) + (public_name datascript-ocaml-melange.lmdb-index) + (wrapped false) + (modes melange) + (modules datascript_lmdb_index) + (libraries datascript_lmdb_codec lmdb_db_melange)) + +(library + (name storage_lmdb_melange) + (public_name datascript-ocaml-melange.storage-lmdb) + (wrapped false) + (modes melange) + (modules datascript_storage_lmdb) + (libraries datascript_lmdb_codec lmdb_db_melange datascript_types)) diff --git a/lmdb/native/datascript_lmdb_db.ml b/lmdb/native/datascript_lmdb_db.ml new file mode 100644 index 0000000..1937eaf --- /dev/null +++ b/lmdb/native/datascript_lmdb_db.ml @@ -0,0 +1,204 @@ +open Datascript_types +open Lmdb + +type t = + { path : string + ; env : Env.t + ; eavt : (string, string, [ `Uni ]) Map.t + ; aevt : (string, string, [ `Uni ]) Map.t + ; avet : (string, string, [ `Uni ]) Map.t + ; meta : (string, string, [ `Uni ]) Map.t + ; mutable closed : bool + } + +let default_map_size = 1024 * 1024 * 1024 +let lock_path path = path ^ "-lock" + +let remove_path path = + if Sys.file_exists path then Sys.remove path; + let lock = lock_path path in + if Sys.file_exists lock then Sys.remove lock + +let open_env db_path = + Env.(create Rw ~flags:Flags.no_subdir ~map_size:default_map_size ~max_maps:8 db_path) + +let open_named_map env name = + try Map.open_existing Nodup ~key:Conv.string ~value:Conv.string ~name env + with Not_found -> Map.create Nodup ~key:Conv.string ~value:Conv.string ~name env + +let open_db path = + remove_path path; + let env = open_env path in + { path; env; eavt = open_named_map env "ds/eavt"; aevt = open_named_map env "ds/aevt" + ; avet = open_named_map env "ds/avet"; meta = open_named_map env "ds/meta"; closed = false + } + +let open_path path = open_db path + +let ensure_open db = + if db.closed then invalid_arg ("LMDB database is closed: " ^ db.path) + +let close db = + if not db.closed then ( + Map.close db.eavt; + Map.close db.aevt; + Map.close db.avet; + Map.close db.meta; + Env.sync db.env; + Env.close db.env; + db.closed <- true) + +let temps_created = ref 0 + +let create_temp () = + let db = + open_db + (Filename.temp_file + ~temp_dir:(Filename.get_temp_dir_name ()) + "datascript_lmdb" + ".mdb") + in + Gc.finalise + (fun lmdb -> + if not lmdb.closed then close lmdb) + db; + incr temps_created; + if !temps_created mod 64 = 0 then Gc.full_major (); + db + +let sync db = + ensure_open db; + Env.sync db.env + +let map_for_index index db = + match index with + | Eavt -> db.eavt + | Aevt -> db.aevt + | Avet -> db.avet + +let meta_get db key = + ensure_open db; + try Some (Map.get db.meta key) with Not_found -> None + +let meta_set db key value = + ensure_open db; + ignore + (Txn.go Rw db.env (fun txn -> + Map.set ~txn db.meta key value; + ())) + +let with_write_txn db f = + ensure_open db; + ignore + (Txn.go Rw db.env (fun txn -> + f txn; + ())) + +let put_index_txn index txn db key value = + Map.set ~txn (map_for_index index db) key value + +let remove_index_txn index txn db key = + try Map.remove ~txn (map_for_index index db) key with Not_found -> () + +let put_index index db key value = + with_write_txn db (fun txn -> put_index_txn index txn db key value) + +let remove_index index db key = + with_write_txn db (fun txn -> remove_index_txn index txn db key) + +let get_index index db key = + ensure_open db; + try Some (Map.get (map_for_index index db) key) with Not_found -> None + +let fold_index index db f = + ensure_open db; + let map = map_for_index index db in + let next = Map.to_dispenser map in + let rec loop () = + match next () with + | None -> () + | Some (key, value) -> + f key value; + loop () + in + loop () + +let fold_index_prefix index db prefix f = + ensure_open db; + let map = map_for_index index db in + let prefix_len = String.length prefix in + (try + Cursor.go Ro map (fun cursor -> + (try ignore (Cursor.seek_range cursor prefix) with Not_found -> raise Exit); + let rec loop () = + let key, value = + try Cursor.current cursor + with Not_found -> raise Exit + in + if String.length key < prefix_len || String.sub key 0 prefix_len <> prefix then raise Exit; + f key value; + try + ignore (Cursor.next cursor); + loop () + with Not_found -> raise Exit + in + loop ()) + with Exit -> ()) + +let fold_index_range index db ?from_key ?to_key f = + ensure_open db; + let map = map_for_index index db in + (try + Cursor.go Ro map (fun cursor -> + (match from_key with + | None -> ( + try ignore (Cursor.first cursor) with Not_found -> raise Exit) + | Some key -> ( + try ignore (Cursor.seek_range cursor key) with Not_found -> raise Exit)); + let rec loop () = + let key, value = + try Cursor.current cursor + with Not_found -> raise Exit + in + (match to_key with + | Some bound when String.compare key bound > 0 -> raise Exit + | _ -> ()); + f key value; + try + ignore (Cursor.next cursor); + loop () + with Not_found -> raise Exit + in + loop ()) + with Exit -> ()) + +let fold_index_range_until index db ?from_key ?stop f = + ensure_open db; + let map = map_for_index index db in + (try + Cursor.go Ro map (fun cursor -> + (match from_key with + | None -> ( + try ignore (Cursor.first cursor) with Not_found -> raise Exit) + | Some key -> ( + try ignore (Cursor.seek_range cursor key) with Not_found -> raise Exit)); + let rec loop () = + let key, value = + try Cursor.current cursor + with Not_found -> raise Exit + in + (match stop with + | Some stop when stop key value -> raise Exit + | _ -> ()); + f key value; + try + ignore (Cursor.next cursor); + loop () + with Not_found -> raise Exit + in + loop ()) + with Exit -> ()) + +let copy_index_txn index txn from_db to_db = + fold_index index from_db (fun key value -> + put_index_txn index txn to_db key value) diff --git a/lmdb/native/datascript_lmdb_db.mli b/lmdb/native/datascript_lmdb_db.mli new file mode 100644 index 0000000..262c841 --- /dev/null +++ b/lmdb/native/datascript_lmdb_db.mli @@ -0,0 +1,32 @@ +open Datascript_types + +type t + +val create_temp : unit -> t +val open_path : string -> t +val close : t -> unit +val sync : t -> unit +val remove_path : string -> unit + +val meta_get : t -> string -> string option +val meta_set : t -> string -> string -> unit + +val with_write_txn : t -> ([ `Read | `Write ] Lmdb.Txn.t -> unit) -> unit +val put_index_txn : index -> [ `Read | `Write ] Lmdb.Txn.t -> t -> string -> string -> unit +val remove_index_txn : index -> [ `Read | `Write ] Lmdb.Txn.t -> t -> string -> unit +val copy_index_txn : index -> [ `Read | `Write ] Lmdb.Txn.t -> t -> t -> unit + +val get_index : index -> t -> string -> string option +val fold_index : index -> t -> (string -> string -> unit) -> unit +val fold_index_range : + index -> t -> ?from_key:string -> ?to_key:string -> (string -> string -> unit) -> unit +val fold_index_range_until : + index -> + t -> + ?from_key:string -> + ?stop:(string -> string -> bool) -> + (string -> string -> unit) -> + unit +val fold_index_prefix : index -> t -> string -> (string -> string -> unit) -> unit +val put_index : index -> t -> string -> string -> unit +val remove_index : index -> t -> string -> unit diff --git a/lmdb/native/datascript_lmdb_index.ml b/lmdb/native/datascript_lmdb_index.ml new file mode 100644 index 0000000..fda02f8 --- /dev/null +++ b/lmdb/native/datascript_lmdb_index.ml @@ -0,0 +1,511 @@ +open Datascript_types + +type t = + { db : Datascript_lmdb_db.t + ; which : index + ; additions : datom list + ; additions_arr : datom array option + ; removals : datom list + ; bulk : bool + } + +type 'a seq = { cmp : datom -> datom -> int; datoms : datom list; offset : int } + +exception Stop_search + +let db_of t = t.db +let make index db = { db; which = index; additions = []; additions_arr = None; removals = []; bulk = false } +let cmp_for index = Datascript_types.Compare.compare_datom index +let overlay_empty t = t.additions = [] && t.removals = [] + +let datom_key t datom = Datascript_lmdb_codec.encode_datom_key t.which datom + +let decode_entry index key value = + let datom = Datascript_lmdb_codec.decode_datom_key index key in + let payload = Datascript_lmdb_codec.decode_datom_value value in + { datom with added = payload.added; v = payload.v } + +let put_datom_txn txn t datom = + let key = datom_key t datom in + let value = Datascript_lmdb_codec.encode_datom_value datom in + Datascript_lmdb_db.put_index_txn t.which txn t.db key value + +let remove_datom_txn txn t datom = + let key = datom_key t datom in + Datascript_lmdb_db.remove_index_txn t.which txn t.db key + +let empty index db = make index db + +let of_sorted_list index datoms db = + let t = empty index db in + if datoms = [] then t + else ( + Datascript_lmdb_db.with_write_txn db (fun txn -> + List.iter (put_datom_txn txn t) datoms); + t) + +let of_sorted_lists index_datoms db = + Datascript_lmdb_db.with_write_txn db (fun txn -> + List.iter + (fun (index, datoms) -> + let t = make index db in + List.iter (put_datom_txn txn t) datoms) + index_datoms) + +let of_eavt_datoms ~avet eavt_datoms db = + if eavt_datoms = [] then () + else ( + let eavt = make Eavt db in + let aevt = make Aevt db in + let avet_index = make Avet db in + Datascript_lmdb_db.with_write_txn db (fun txn -> + List.iter + (fun datom -> + put_datom_txn txn eavt datom; + put_datom_txn txn aevt datom; + if avet datom.a then put_datom_txn txn avet_index datom) + eavt_datoms)) + +let of_bulk index datoms db = + { db; which = index; additions = []; additions_arr = Some (Array.of_list datoms); removals = []; bulk = true } + +let array_find_first cmp bound arr = + let len = Array.length arr in + let rec lower lo hi = + if lo >= hi then lo + else + let mid = (lo + hi) / 2 in + if cmp arr.(mid) bound < 0 then lower (mid + 1) hi else lower lo mid + in + let index = lower 0 len in + if index < len && cmp arr.(index) bound = 0 then Some arr.(index) else None + +let array_lower_bound cmp bound arr = + let len = Array.length arr in + let rec lower lo hi = + if lo >= hi then lo + else + let mid = (lo + hi) / 2 in + if cmp arr.(mid) bound < 0 then lower (mid + 1) hi else lower lo mid + in + lower 0 len + +let sorted_bulk_array t = + match t.additions_arr with + | Some arr -> arr + | None -> + let arr = Array.of_list t.additions in + Array.sort (cmp_for t.which) arr; + arr + +let bulk_datoms t = + let base = Array.to_list (sorted_bulk_array t) in + match t.additions with + | [] -> base + | overlay -> List.merge (cmp_for t.which) (List.sort (cmp_for t.which) overlay) base + +let in_range cmp lower upper datom = + let above_lower = + match lower with + | None -> true + | Some lower -> cmp datom lower >= 0 + in + let below_upper = + match upper with + | None -> true + | Some upper -> cmp datom upper <= 0 + in + above_lower && below_upper + +let array_fold_in_range cmp from_ to_ arr f init = + let len = Array.length arr in + let start = + match from_ with + | None -> 0 + | Some bound -> array_lower_bound cmp bound arr + in + let rec loop index acc = + if index >= len then acc + else + let datom = arr.(index) in + if not (in_range cmp from_ to_ datom) then acc + else loop (index + 1) (f acc datom) + in + loop start init + +let array_materialize_range cmp from_ to_ arr = + array_fold_in_range cmp from_ to_ arr (fun acc datom -> datom :: acc) [] |> List.rev + +let array_fold_attr_prefix f init attr arr index = + let cmp = cmp_for index in + let bound = { e = 0; a = attr; v = Nil; tx = 0; added = true } in + let start = array_lower_bound cmp bound arr in + let len = Array.length arr in + let rec loop i acc = + if i >= len then acc + else + let datom = arr.(i) in + if datom.a <> attr then acc else loop (i + 1) (f acc datom) + in + loop start init + +let values_equal left right = + match left, right with + | String left, String right + | Symbol left, Symbol right + | Keyword left, Keyword right + | Uuid left, Uuid right + | Regex left, Regex right -> + left = right + | Bool left, Bool right -> left = right + | Int left, Int right + | Ref left, Ref right + | Int left, Ref right + | Ref left, Int right -> + left = right + | Instant left, Instant right -> left = right + | Nil, Nil -> true + | TxRef, TxRef -> true + | _ -> Compare.compare_value left right = 0 + +let array_fold_attr_value_prefix f init attr value arr index = + let cmp = cmp_for index in + let bound = { e = 0; a = attr; v = value; tx = 0; added = true } in + let start = array_lower_bound cmp bound arr in + let len = Array.length arr in + let rec loop i acc = + if i >= len then acc + else + let datom = arr.(i) in + if datom.a <> attr || not (values_equal datom.v value) then acc + else loop (i + 1) (f acc datom) + in + loop start init + +let additions_only t = + t.bulk && t.removals = [] && (t.additions <> [] || Option.is_some t.additions_arr) + +let add datom t = + if additions_only t then + { t with additions = datom :: t.additions } + else ( + let key = datom_key t datom in + let additions = datom :: List.filter (fun d -> datom_key t d <> key) t.additions in + let removals = List.filter (fun d -> datom_key t d <> key) t.removals in + { t with additions; removals }) + +let remove datom t = + let key = datom_key t datom in + let stored_additions = + match t.additions_arr with + | Some arr -> Array.to_list arr + | None -> t.additions + in + let additions = List.filter (fun d -> datom_key t d <> key) stored_additions in + let already_removed = List.exists (fun d -> datom_key t d = key) t.removals in + let removals = + if already_removed || List.exists (fun d -> datom_key t d = key) stored_additions then t.removals + else datom :: t.removals + in + { t with additions; additions_arr = None; removals } + +let overlay_tables t = + let stored_additions = + match t.additions_arr with + | Some arr -> Array.to_list arr + | None -> t.additions + in + let removed = Hashtbl.create (List.length t.removals) in + List.iter (fun datom -> Hashtbl.add removed (datom_key t datom) ()) t.removals; + let added = Hashtbl.create (List.length stored_additions) in + List.iter (fun datom -> Hashtbl.replace added (datom_key t datom) datom) stored_additions; + removed, added + +let stored_visible key removed added = + not (Hashtbl.mem removed key || Hashtbl.mem added key) + +let bound_key t = function + | None -> None + | Some datom -> Some (datom_key t datom) + +let fold_stored t f acc = + if overlay_empty t then + let acc = ref acc in + Datascript_lmdb_db.fold_index t.which t.db (fun key value -> + acc := f !acc (decode_entry t.which key value)); + !acc + else + let removed, added = overlay_tables t in + let acc = ref acc in + Datascript_lmdb_db.fold_index t.which t.db (fun key value -> + if stored_visible key removed added then + acc := f !acc (decode_entry t.which key value)); + !acc + +let fold_stored_prefix t attr f acc = + let prefix = attr ^ "\000" in + if overlay_empty t then + let acc = ref acc in + Datascript_lmdb_db.fold_index_prefix t.which t.db prefix (fun key value -> + acc := f !acc (decode_entry t.which key value)); + !acc + else + let removed, added = overlay_tables t in + let acc = ref acc in + Datascript_lmdb_db.fold_index_prefix t.which t.db prefix (fun key value -> + if stored_visible key removed added then + acc := f !acc (decode_entry t.which key value)); + !acc + +let fold_stored_attr_value_prefix t attr value f acc = + let prefix = Datascript_lmdb_codec.encode_index_attr_value_prefix t.which attr value in + if overlay_empty t then + let acc = ref acc in + Datascript_lmdb_db.fold_index_prefix t.which t.db prefix (fun key value -> + acc := f !acc (decode_entry t.which key value)); + !acc + else + let removed, added = overlay_tables t in + let acc = ref acc in + Datascript_lmdb_db.fold_index_prefix t.which t.db prefix (fun key value -> + if stored_visible key removed added then + acc := f !acc (decode_entry t.which key value)); + !acc + +let fold_stored_bounded t ?from_ ?to_ cmp f acc = + match bound_key t from_ with + | None -> fold_stored t f acc + | Some from_key -> + let removed, added = + if overlay_empty t then (Hashtbl.create 0, Hashtbl.create 0) else overlay_tables t + in + let acc = ref acc in + Datascript_lmdb_db.fold_index_range_until t.which t.db ~from_key + ~stop:(fun key value -> + if not (stored_visible key removed added) then false + else + match to_ with + | Some bound -> + let datom = decode_entry t.which key value in + cmp datom bound > 0 + | None -> false) + (fun key value -> + if stored_visible key removed added then + let datom = decode_entry t.which key value in + if in_range cmp from_ to_ datom then acc := f !acc datom); + !acc + +let fold_overlay t f acc = List.fold_left f acc t.additions + +let fold_bulk_slice f init ?from_ ?to_ ?cmp t = + let cmp = Option.value ~default:(cmp_for t.which) cmp in + let apply acc datom = if in_range cmp from_ to_ datom then f acc datom else acc in + let arr = sorted_bulk_array t in + let acc = + match from_, to_ with + | Some bound, Some bound' when bound == bound' && bound.a <> "" && bound.v <> Nil && bound.e = 0 -> + array_fold_attr_value_prefix f init bound.a bound.v arr t.which + | _ -> array_fold_in_range cmp from_ to_ arr f init + in + List.fold_left apply acc t.additions + +let fold_datoms f init t = + if additions_only t then + let acc = Array.fold_left (fun acc datom -> f acc datom) init (sorted_bulk_array t) in + List.fold_left f acc t.additions + else ( + let acc = fold_stored t f init in + fold_overlay t f acc) + +let collect_datoms t = + if overlay_empty t then fold_stored t (fun acc datom -> datom :: acc) [] + else fold_datoms (fun acc datom -> datom :: acc) [] t |> List.sort (cmp_for t.which) + +let clear_index_txn txn index lmdb = + Datascript_lmdb_db.fold_index index lmdb (fun key _ -> + Datascript_lmdb_db.remove_index_txn index txn lmdb key) + +let sync_merged_to_lmdb t target_lmdb = + let write_datom_txn txn datom = + let key = datom_key t datom in + let value = Datascript_lmdb_codec.encode_datom_value datom in + Datascript_lmdb_db.put_index_txn t.which txn target_lmdb key value + in + if additions_only t then + Datascript_lmdb_db.with_write_txn target_lmdb (fun txn -> + Array.iter (write_datom_txn txn) (sorted_bulk_array t); + List.iter (write_datom_txn txn) t.additions) + else if overlay_empty t then + Datascript_lmdb_db.with_write_txn target_lmdb (fun txn -> + clear_index_txn txn t.which target_lmdb; + Datascript_lmdb_db.copy_index_txn t.which txn t.db target_lmdb) + else + let merged = collect_datoms t in + Datascript_lmdb_db.with_write_txn target_lmdb (fun txn -> + clear_index_txn txn t.which target_lmdb; + List.iter (write_datom_txn txn) merged) + +let copy_list xs = List.map (fun x -> x) xs + +let copy t = { t with additions = copy_list t.additions; removals = copy_list t.removals } + +let flush t = + if overlay_empty t then t + else ( + Datascript_lmdb_db.with_write_txn t.db (fun txn -> + List.iter (remove_datom_txn txn t) t.removals; + List.iter (put_datom_txn txn t) t.additions); + { t with additions = []; additions_arr = None; removals = [] }) + +let to_list t = + if additions_only t then bulk_datoms t + else if overlay_empty t then List.rev (fold_stored t (fun acc datom -> datom :: acc) []) + else collect_datoms t + +let fold f init t = fold_datoms f init t + +let lookup t datom = + let key = datom_key t datom in + if List.exists (fun d -> datom_key t d = key) t.removals then None + else + (match List.find_opt (fun d -> datom_key t d = key) t.additions with + | Some datom -> Some datom + | None -> ( + match t.additions_arr with + | Some arr -> + let cmp = cmp_for t.which in + (match array_find_first cmp datom arr with + | Some found when datom_key t found = key -> Some found + | _ -> None) + | None -> ( + match Datascript_lmdb_db.get_index t.which t.db key with + | None -> None + | Some value -> Some (decode_entry t.which key value)))) + +let fold_slice f init ?from_ ?to_ ?cmp t = + if additions_only t then fold_bulk_slice f init ?from_ ?to_ ?cmp t + else + let cmp = Option.value ~default:(cmp_for t.which) cmp in + let apply acc datom = if in_range cmp from_ to_ datom then f acc datom else acc in + if not (overlay_empty t) then + collect_datoms t + |> List.filter (fun datom -> in_range cmp from_ to_ datom) + |> List.fold_left f init + else + match from_, to_ with + | Some bound, Some bound' when bound == bound' && bound.a <> "" && bound.e = 0 && bound.v = Nil + && (t.which = Aevt || t.which = Avet) -> + fold_stored_prefix t bound.a apply init + | Some bound, Some bound' when bound == bound' && bound.a <> "" && bound.v <> Nil && bound.e = 0 -> + fold_stored_attr_value_prefix t bound.a bound.v apply init + | _ -> fold_stored_bounded t ?from_ ?to_ cmp apply init + +let find_first_slice ?from_ ?to_ ?cmp t = + let cmp = Option.value ~default:(cmp_for t.which) cmp in + let found = ref None in + let consider datom = + if !found = None && in_range cmp from_ to_ datom then ( + found := Some datom; + raise Stop_search) + in + (try + if additions_only t then ( + List.iter consider t.additions; + match from_, to_ with + | Some bound, Some bound' when bound == bound' && bound.a <> "" && bound.v <> Nil && bound.e = 0 -> + ignore + (array_fold_attr_value_prefix + (fun () datom -> consider datom) + () + bound.a + bound.v + (sorted_bulk_array t) + t.which) + | Some bound, Some bound' when bound == bound' -> ( + match array_find_first cmp bound (sorted_bulk_array t) with + | Some datom when !found = None && in_range cmp from_ to_ datom -> + found := Some datom; + raise Stop_search + | _ -> ()) + | _ -> + if !found = None then + ignore (array_fold_in_range cmp from_ to_ (sorted_bulk_array t) (fun () datom -> consider datom) ())) + else if not (overlay_empty t) then + collect_datoms t |> List.iter consider + else + match from_, to_ with + | Some bound, Some bound' when bound == bound' && bound.a <> "" && bound.e = 0 && bound.v = Nil + && (t.which = Aevt || t.which = Avet) -> + fold_stored_prefix t bound.a (fun () datom -> consider datom) () + | Some bound, Some bound' when bound == bound' && bound.a <> "" && bound.v <> Nil && bound.e = 0 -> + fold_stored_attr_value_prefix t bound.a bound.v (fun () datom -> consider datom) () + | _ -> fold_stored_bounded t ?from_ ?to_ cmp (fun () datom -> consider datom) () + with Stop_search -> ()); + !found + +let fold_attr_prefix f init t attr = + let apply acc datom = if datom.a = attr then f acc datom else acc in + if additions_only t then + let arr = sorted_bulk_array t in + let acc = array_fold_attr_prefix f init attr arr t.which in + List.fold_left apply acc t.additions + else if not (overlay_empty t) then + collect_datoms t + |> List.filter (fun datom -> datom.a = attr) + |> List.fold_left f init + else + fold_stored_prefix t attr apply init + +let materialize_range t ?from_ ?to_ cmp = + if additions_only t then array_materialize_range cmp from_ to_ (sorted_bulk_array t) + else fold_slice (fun acc datom -> datom :: acc) [] ?from_ ?to_ ~cmp t |> List.rev + +let make_seq cmp datoms = { cmp; datoms; offset = 0 } + +let to_seq ({ cmp = _; datoms; offset = start }) = + let rec loop index () = + if index >= List.length datoms then Seq.Nil + else Seq.Cons (List.nth datoms index, loop (index + 1)) + in + loop start + +let seq t = make_seq (cmp_for t.which) (to_list t) + +let slice_seq ?from_ ?to_ ?cmp t = + let cmp = Option.value ~default:(cmp_for t.which) cmp in + make_seq cmp (materialize_range t ?from_ ?to_ cmp) + +let rslice_seq ?from_ ?to_ ?cmp t = + let cmp = Option.value ~default:(cmp_for t.which) cmp in + let datoms = + to_list t + |> List.filter (fun datom -> + match from_ with + | None -> true + | Some bound -> cmp datom bound <= 0) + |> List.filter (fun datom -> + match to_ with + | None -> true + | Some bound -> cmp datom bound >= 0) + |> List.rev + in + make_seq cmp datoms + +let seq_to_list seq = to_seq seq |> List.of_seq + +let fold_seq f init { cmp = _; datoms; offset } = + let rec loop index acc = + if index >= List.length datoms then acc + else loop (index + 1) (f acc (List.nth datoms index)) + in + loop offset init + +let slice ?from_ ?to_ ?cmp t = slice_seq ?from_ ?to_ ?cmp t |> seq_to_list + +let seek bound seq = + let rec count index = + if index >= List.length seq.datoms then index + else if seq.cmp (List.nth seq.datoms index) bound >= 0 then index + else count (index + 1) + in + { seq with offset = count 0 } diff --git a/lmdb/native/datascript_lmdb_index.mli b/lmdb/native/datascript_lmdb_index.mli new file mode 100644 index 0000000..cc85977 --- /dev/null +++ b/lmdb/native/datascript_lmdb_index.mli @@ -0,0 +1,32 @@ +open Datascript_types + +type t +type 'a seq + +val db_of : t -> Datascript_lmdb_db.t +val empty : index -> Datascript_lmdb_db.t -> t +val of_sorted_list : index -> datom list -> Datascript_lmdb_db.t -> t +val of_sorted_lists : (index * datom list) list -> Datascript_lmdb_db.t -> unit +val of_eavt_datoms : avet:(string -> bool) -> datom list -> Datascript_lmdb_db.t -> unit +val of_bulk : index -> datom list -> Datascript_lmdb_db.t -> t +val add : datom -> t -> t +val remove : datom -> t -> t +val flush : t -> t +val copy : t -> t +val sync_merged_to_lmdb : t -> Datascript_lmdb_db.t -> unit +val lookup : t -> datom -> datom option +val to_list : t -> datom list +val fold : ('acc -> datom -> 'acc) -> 'acc -> t -> 'acc +val fold_slice : + ('acc -> datom -> 'acc) -> 'acc -> ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> 'acc +val find_first_slice : + ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom option +val fold_attr_prefix : ('acc -> datom -> 'acc) -> 'acc -> t -> string -> 'acc +val slice : ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom list +val slice_seq : ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom seq +val rslice_seq : ?from_:datom -> ?to_:datom -> ?cmp:(datom -> datom -> int) -> t -> datom seq +val seq : t -> datom seq +val seq_to_list : datom seq -> datom list +val fold_seq : ('acc -> datom -> 'acc) -> 'acc -> datom seq -> 'acc +val to_seq : datom seq -> datom Seq.t +val seek : datom -> datom seq -> datom seq diff --git a/lmdb/native/datascript_storage_lmdb.ml b/lmdb/native/datascript_storage_lmdb.ml new file mode 100644 index 0000000..27eb03a --- /dev/null +++ b/lmdb/native/datascript_storage_lmdb.ml @@ -0,0 +1,101 @@ +open Datascript_types + +type t = Datascript_lmdb_db.t + +module Storage_registry = struct + type t = storage + + let equal left right = left == right + + let hash storage = Hashtbl.hash (Obj.repr storage) +end + +module Registry = Hashtbl.Make (Storage_registry) + +let registry = Registry.create 16 + +let lmdb storage = + match Registry.find_opt registry storage with + | Some lmdb -> lmdb + | None -> invalid_arg "storage is not LMDB-backed" + +let register storage lmdb = Registry.replace registry storage lmdb + +let create_temp () = Datascript_lmdb_db.create_temp () +let open_path path = Datascript_lmdb_db.open_path path +let close = Datascript_lmdb_db.close +let sync = Datascript_lmdb_db.sync + +let meta_get = Datascript_lmdb_db.meta_get +let meta_set = Datascript_lmdb_db.meta_set + +let meta_schema_key = "schema" +let meta_max_eid_key = "max_eid" +let meta_max_tx_key = "max_tx" +let meta_duplicates_key = "duplicate_datoms" + +let wrap lmdb = + let storage = + { storage_store = + (fun _entries -> sync lmdb) + ; storage_restore = + (fun address -> + if String.equal address "lmdb" then Some Storage_session else None) + ; storage_list_addresses = (fun () -> [ "lmdb" ]) + ; storage_delete = (fun _addresses -> ()) + } + in + register storage lmdb; + storage + +let memory_storage () = wrap (create_temp ()) + +let encode_int value = + Datascript_lmdb_codec.encode_datoms + [ { e = value; a = ""; v = Nil; tx = 0; added = true } ] + +let decode_int bytes = + match Datascript_lmdb_codec.decode_datoms bytes with + | { e; _ } :: _ -> e + | [] -> 0 + +let store_meta lmdb db = + meta_set lmdb meta_schema_key (Datascript_lmdb_codec.encode_schema db.schema); + meta_set lmdb meta_max_eid_key (encode_int db.max_eid); + meta_set lmdb meta_max_tx_key (encode_int db.max_tx); + meta_set lmdb meta_duplicates_key (Datascript_lmdb_codec.encode_datoms db.duplicate_datoms); + sync lmdb + +let restore_meta lmdb = + let schema = + match meta_get lmdb meta_schema_key with + | None -> [] + | Some bytes -> Datascript_lmdb_codec.decode_schema bytes + in + let max_eid = + match meta_get lmdb meta_max_eid_key with + | None -> 0 + | Some bytes -> decode_int bytes + in + let max_tx = + match meta_get lmdb meta_max_tx_key with + | None -> 0x20000000 + | Some bytes -> decode_int bytes + in + let duplicate_datoms = + match meta_get lmdb meta_duplicates_key with + | None -> [] + | Some bytes -> Datascript_lmdb_codec.decode_datoms bytes + in + schema, max_eid, max_tx, duplicate_datoms + +let sync_indexes from_lmdb to_lmdb = + if from_lmdb != to_lmdb then + Datascript_lmdb_db.with_write_txn to_lmdb (fun txn -> + List.iter + (fun index -> + Datascript_lmdb_db.copy_index_txn index txn from_lmdb to_lmdb) + [ Eavt; Aevt; Avet ]) + +let store_db storage db = + store_meta (lmdb storage) db diff --git a/lmdb/native/dune b/lmdb/native/dune new file mode 100644 index 0000000..f573531 --- /dev/null +++ b/lmdb/native/dune @@ -0,0 +1,25 @@ +(include_subdirs no) + +(library + (name lmdb_db_native) + (public_name datascript-ocaml-native.lmdb-db) + (wrapped false) + (modes native) + (modules datascript_lmdb_db) + (libraries datascript_lmdb_codec lmdb)) + +(library + (name lmdb_index_native) + (public_name datascript-ocaml-native.lmdb-index) + (wrapped false) + (modes native) + (modules datascript_lmdb_index) + (libraries datascript_lmdb_codec lmdb_db_native)) + +(library + (name storage_lmdb_native) + (public_name datascript-ocaml-native.storage-lmdb) + (wrapped false) + (modes native) + (modules datascript_storage_lmdb) + (libraries datascript_lmdb_codec lmdb_db_native datascript_types)) diff --git a/melange/datascript_melange_storage.ml b/melange/datascript_melange_storage.ml index 2cabab9..0a85e02 100644 --- a/melange/datascript_melange_storage.ml +++ b/melange/datascript_melange_storage.ml @@ -1,9 +1,35 @@ module Ds = Datascript -module PSet = Persistent_sorted_set module Transit = Transit_melange.Transit.Json open Ds +type ref_type = + | Strong + | Weak + +type stored_node = + | Leaf of datom list + | Branch of datom list * storage_address list + +type storage_root = + { storage_schema : schema + ; storage_max_eid : entity_id + ; storage_max_tx : tx + ; storage_eavt : storage_address + ; storage_aevt : storage_address + ; storage_avet : storage_address + ; storage_duplicate_datoms : datom list + ; storage_max_addr : int + ; storage_branching_factor : int + ; storage_ref_type : ref_type + } + +type compat_payload = + | Compat_root of storage_root + | Compat_node of stored_node + | Compat_tail of datom list list + | Compat_session + let schema_attr_default : Ds.schema_attr = { cardinality = One; @@ -87,13 +113,13 @@ let value_type_of_transit = function | _ -> None let transit_of_ref_type = function - | PSet.Strong -> Transit.Keyword "strong" - | PSet.Weak -> Transit.Keyword "weak" + | Strong -> Transit.Keyword "strong" + | Weak -> Transit.Keyword "weak" let ref_type_of_transit = function - | Transit.Keyword "soft" -> PSet.Weak - | Transit.Keyword "weak" -> PSet.Weak - | Transit.Keyword "strong" | _ -> PSet.Strong + | Transit.Keyword "soft" -> Weak + | Transit.Keyword "weak" -> Weak + | Transit.Keyword "strong" | _ -> Strong let address_to_transit address = Transit.String address @@ -274,8 +300,8 @@ let storage_root_to_transit root = ] let storage_node_to_transit = function - | PSet.Leaf datoms -> Transit.Map [ (Transit.Keyword "keys", datoms_to_transit datoms) ] - | PSet.Branch (keys, child_addresses) -> + | Leaf datoms -> Transit.Map [ (Transit.Keyword "keys", datoms_to_transit datoms) ] + | Branch (keys, child_addresses) -> Transit.Map [ (Transit.Keyword "keys", datoms_to_transit keys); @@ -286,9 +312,10 @@ let storage_tail_to_transit groups = Transit.Array (List.map (fun group -> datoms_to_transit group) groups) let payload_to_transit = function - | Ds.Storage_root root -> storage_root_to_transit root - | Storage_node node -> storage_node_to_transit node - | Storage_tail groups -> storage_tail_to_transit groups + | Compat_root root -> storage_root_to_transit root + | Compat_node node -> storage_node_to_transit node + | Compat_tail groups -> storage_tail_to_transit groups + | Compat_session -> Transit.Map [] let require_key key entries = match lookup_transit_key key entries with @@ -302,7 +329,7 @@ let optional_datoms key entries = let storage_root_of_transit entries = { - Ds.storage_schema = schema_of_transit (require_key "schema" entries); + storage_schema = schema_of_transit (require_key "schema" entries); storage_max_eid = int_of_transit "storage root :max-eid" (require_key "max-eid" entries); storage_max_tx = int_of_transit "storage root :max-tx" (require_key "max-tx" entries); storage_eavt = address_of_transit "storage root :eavt" (require_key "eavt" entries); @@ -323,8 +350,8 @@ let child_addresses_of_transit = function let storage_node_of_transit entries = let keys = datoms_of_transit (require_key "keys" entries) in match lookup_transit_key "children" entries with - | None -> PSet.Leaf keys - | Some children -> PSet.Branch (keys, child_addresses_of_transit children) + | None -> Leaf keys + | Some children -> Branch (keys, child_addresses_of_transit children) let storage_tail_of_transit = function | Transit.Array groups | Transit.List groups -> List.map datoms_of_transit groups @@ -332,11 +359,20 @@ let storage_tail_of_transit = function let payload_of_transit = function | Transit.Map entries -> - if Option.is_some (lookup_transit_key "schema" entries) then Storage_root (storage_root_of_transit entries) - else if Option.is_some (lookup_transit_key "keys" entries) then Storage_node (storage_node_of_transit entries) - else invalid_arg "unknown storage payload map" - | (Transit.Array _ | Transit.List _) as tail -> Storage_tail (storage_tail_of_transit tail) + if Option.is_some (lookup_transit_key "schema" entries) then Compat_root (storage_root_of_transit entries) + else if Option.is_some (lookup_transit_key "keys" entries) then Compat_node (storage_node_of_transit entries) + else Compat_session + | (Transit.Array _ | Transit.List _) as tail -> Compat_tail (storage_tail_of_transit tail) | _ -> invalid_arg "unknown storage payload" let encode payload = payload |> payload_to_transit |> Transit.to_string ~mode:Transit.Verbose let decode content = content |> Transit.of_string |> payload_of_transit + +let encode_storage_payload (payload : Ds.storage_payload) = + match payload with Storage_session -> encode Compat_session + +let decode_storage_payload payload = + match decode payload with + | Compat_session -> Storage_session + | Compat_root _ | Compat_node _ | Compat_tail _ -> + invalid_arg "legacy PSS storage payloads are no longer supported" diff --git a/melange/dune b/melange/dune index 681503d..9e13622 100644 --- a/melange/dune +++ b/melange/dune @@ -5,5 +5,4 @@ (enabled_if (= %{context_name} default)) (libraries datascript-ocaml-melange - melange-transit-melange - persistent_sorted_set_ocaml.melange)) + melange-transit-melange)) diff --git a/sqlite/datascript_sqlite_codec.ml b/sqlite/datascript_sqlite_codec.ml index 05f9b69..0617ff6 100644 --- a/sqlite/datascript_sqlite_codec.ml +++ b/sqlite/datascript_sqlite_codec.ml @@ -1,9 +1,35 @@ module Ds = Datascript -module PSet = Persistent_sorted_set module Transit = Transit_native.Transit.Json open Ds +type ref_type = + | Strong + | Weak + +type stored_node = + | Leaf of datom list + | Branch of datom list * storage_address list + +type storage_root = + { storage_schema : schema + ; storage_max_eid : entity_id + ; storage_max_tx : tx + ; storage_eavt : storage_address + ; storage_aevt : storage_address + ; storage_avet : storage_address + ; storage_duplicate_datoms : datom list + ; storage_max_addr : int + ; storage_branching_factor : int + ; storage_ref_type : ref_type + } + +type compat_payload = + | Compat_root of storage_root + | Compat_node of stored_node + | Compat_tail of datom list list + | Compat_session + let schema_attr_default : Ds.schema_attr = { cardinality = One; @@ -87,13 +113,13 @@ let value_type_of_transit = function | _ -> None let transit_of_ref_type = function - | PSet.Strong -> Transit.Keyword "strong" - | PSet.Weak -> Transit.Keyword "weak" + | Strong -> Transit.Keyword "strong" + | Weak -> Transit.Keyword "weak" let ref_type_of_transit = function - | Transit.Keyword "soft" -> PSet.Weak - | Transit.Keyword "weak" -> PSet.Weak - | Transit.Keyword "strong" | _ -> PSet.Strong + | Transit.Keyword "soft" -> Weak + | Transit.Keyword "weak" -> Weak + | Transit.Keyword "strong" | _ -> Strong let address_to_transit address = Transit.String address @@ -274,8 +300,8 @@ let storage_root_to_transit root = ] let storage_node_to_transit = function - | PSet.Leaf datoms -> Transit.Map [ (Transit.Keyword "keys", datoms_to_transit datoms) ] - | PSet.Branch (keys, child_addresses) -> + | Leaf datoms -> Transit.Map [ (Transit.Keyword "keys", datoms_to_transit datoms) ] + | Branch (keys, child_addresses) -> Transit.Map [ (Transit.Keyword "keys", datoms_to_transit keys); @@ -286,9 +312,10 @@ let storage_tail_to_transit groups = Transit.Array (List.map (fun group -> datoms_to_transit group) groups) let payload_to_transit = function - | Ds.Storage_root root -> storage_root_to_transit root - | Storage_node node -> storage_node_to_transit node - | Storage_tail groups -> storage_tail_to_transit groups + | Compat_root root -> storage_root_to_transit root + | Compat_node node -> storage_node_to_transit node + | Compat_tail groups -> storage_tail_to_transit groups + | Compat_session -> Transit.Map [] let require_key key entries = match lookup_transit_key key entries with @@ -302,7 +329,7 @@ let optional_datoms key entries = let storage_root_of_transit entries = { - Ds.storage_schema = schema_of_transit (require_key "schema" entries); + storage_schema = schema_of_transit (require_key "schema" entries); storage_max_eid = int_of_transit "storage root :max-eid" (require_key "max-eid" entries); storage_max_tx = int_of_transit "storage root :max-tx" (require_key "max-tx" entries); storage_eavt = address_of_transit "storage root :eavt" (require_key "eavt" entries); @@ -323,8 +350,8 @@ let child_addresses_of_transit = function let storage_node_of_transit entries = let keys = datoms_of_transit (require_key "keys" entries) in match lookup_transit_key "children" entries with - | None -> PSet.Leaf keys - | Some children -> PSet.Branch (keys, child_addresses_of_transit children) + | None -> Leaf keys + | Some children -> Branch (keys, child_addresses_of_transit children) let storage_tail_of_transit = function | Transit.Array groups | Transit.List groups -> List.map datoms_of_transit groups @@ -332,11 +359,20 @@ let storage_tail_of_transit = function let payload_of_transit = function | Transit.Map entries -> - if Option.is_some (lookup_transit_key "schema" entries) then Storage_root (storage_root_of_transit entries) - else if Option.is_some (lookup_transit_key "keys" entries) then Storage_node (storage_node_of_transit entries) - else invalid_arg "unknown storage payload map" - | (Transit.Array _ | Transit.List _) as tail -> Storage_tail (storage_tail_of_transit tail) + if Option.is_some (lookup_transit_key "schema" entries) then Compat_root (storage_root_of_transit entries) + else if Option.is_some (lookup_transit_key "keys" entries) then Compat_node (storage_node_of_transit entries) + else Compat_session + | (Transit.Array _ | Transit.List _) as tail -> Compat_tail (storage_tail_of_transit tail) | _ -> invalid_arg "unknown storage payload" let encode payload = payload |> payload_to_transit |> Transit.to_string ~mode:Transit.Verbose let decode content = content |> Transit.of_string |> payload_of_transit + +let encode_storage_payload (payload : Ds.storage_payload) = + match payload with Storage_session -> encode Compat_session + +let decode_storage_payload payload = + match decode payload with + | Compat_session -> Storage_session + | Compat_root _ | Compat_node _ | Compat_tail _ -> + invalid_arg "legacy PSS storage payloads are no longer supported" diff --git a/sqlite/dune b/sqlite/dune index 592b17f..576111a 100644 --- a/sqlite/dune +++ b/sqlite/dune @@ -7,4 +7,4 @@ (names datascript_sqlite_stubs)) (c_library_flags (:standard -L%{env:DATASCRIPT_SQLITE_LIB_DIR=.} -lsqlite3)) - (libraries datascript-ocaml-native persistent_sorted_set_ocaml melange-transit-native)) + (libraries datascript-ocaml-native melange-transit-native)) diff --git a/test/dune b/test/dune index 6666a0e..87b8181 100644 --- a/test/dune +++ b/test/dune @@ -21,7 +21,7 @@ (test (name test_db) (modules test_db) - (libraries datascript-ocaml-native persistent_sorted_set_ocaml unix)) + (libraries datascript-ocaml-native unix)) (test (name test_perf) @@ -131,6 +131,11 @@ datascript-ocaml-native.sqlite datascript-ocaml-native.logseq-sqlite-storage)) +(test + (name test_lmdb_package) + (modules test_lmdb_package) + (libraries datascript-ocaml-native datascript-ocaml-native.lmdb)) + (test (name test_melange_transit_backend) (modules test_melange_transit_backend) diff --git a/test/test_db.ml b/test/test_db.ml index 5c19e0a..453044a 100644 --- a/test/test_db.ml +++ b/test/test_db.ml @@ -71,7 +71,7 @@ let indexed = let unique_identity = { indexed with unique = Some Identity } -let assert_uses_persistent_sorted_set (_index : datom Persistent_sorted_set.t) = () +let assert_uses_lmdb_index (_index : index_set) = () let test_db__test_defrecord_updatable () = let value = { x = Keyword "ignored"; tag = "kept" } in @@ -190,7 +190,7 @@ let test_db__test_index_api () = () |> List.rev) -let test_db__test_indexes_use_persistent_sorted_set () = +let test_db__test_indexes_use_lmdb () = let db = empty_db ~schema:[ "name", indexed; "friend", { indexed with value_type = Some RefType } ] () |> db_with @@ -199,9 +199,9 @@ let test_db__test_indexes_use_persistent_sorted_set () = ; Add (Entity_id 2, "name", String "Oleg") ] in - assert_uses_persistent_sorted_set db.eavt_index; - assert_uses_persistent_sorted_set db.aevt_index; - assert_uses_persistent_sorted_set db.avet_index + assert_uses_lmdb_index db.eavt_index; + assert_uses_lmdb_index db.aevt_index; + assert_uses_lmdb_index db.avet_index let test_db__test_index_lookup_matches_upstream_numeric_comparator_bounds () = let db = @@ -232,5 +232,5 @@ let () = test_db__test_squuid_uses_wall_clock_time (); test_db__test_diff (); test_db__test_index_api (); - test_db__test_indexes_use_persistent_sorted_set (); + test_db__test_indexes_use_lmdb (); test_db__test_index_lookup_matches_upstream_numeric_comparator_bounds () diff --git a/test/test_lmdb_package.ml b/test/test_lmdb_package.ml new file mode 100644 index 0000000..403fa2a --- /dev/null +++ b/test/test_lmdb_package.ml @@ -0,0 +1,68 @@ +open Datascript + +let require condition message = + if not condition then failwith message + +let temp_db_path name = + let path = Filename.temp_file name ".lmdb" in + Sys.remove path; + path + +let indexed = + { cardinality = One + ; unique = Some Identity + ; indexed = true + ; is_component = false + ; no_history = false + ; doc = None + ; value_type = Some StringType + ; tuple_attrs = None + ; tuple_types = None + } + +let test_storage_roundtrip () = + let path = temp_db_path "datascript-lmdb-package" in + let session = Datascript_lmdb.open_session path in + let storage = Datascript_lmdb.storage session in + let db = empty_db ~schema:[ "todo/id", indexed ] ~storage () in + let report = + transact + db + [ Add (Temp_id "todo-1", "todo/id", String "todo-1") + ; Add (Temp_id "todo-1", "todo/title", String "Move storage into datascript") + ] + in + store ~storage report.db_after; + let restored = + match restore storage with + | Some db -> db + | None -> failwith "expected LMDB storage to restore a database" + in + let entity = + match entity restored (Lookup_ref ("todo/id", String "todo-1")) with + | Some entity -> entity + | None -> failwith "expected restored todo entity" + in + require + (entity_attr entity "todo/title" = Some (One_value (String "Move storage into datascript"))) + "expected restored entity title"; + require + (List.mem Storage.root_address (storage_addresses storage)) + "expected LMDB storage to contain the root address"; + Datascript_lmdb.close session + +let test_session_close_blocks_use () = + let path = temp_db_path "datascript-lmdb-session-close" in + let session = Datascript_lmdb.open_session path in + let storage = Datascript_lmdb.storage session in + Datascript_lmdb.close session; + match storage.storage_list_addresses () with + | _ -> failwith "expected closed LMDB session to reject storage operations" + | exception Invalid_argument message -> + require + (String.equal message "LMDB session is closed") + "expected closed session error message" + +let () = + test_storage_roundtrip (); + test_session_close_blocks_use () diff --git a/test/test_storage.ml b/test/test_storage.ml index 4348c52..39e363c 100644 --- a/test/test_storage.ml +++ b/test/test_storage.ml @@ -7,22 +7,9 @@ let datoms_seq = datoms let datoms db index ?e ?a ?v ?tx () = datoms_seq db index ?e ?a ?v ?tx () |> List.of_seq -let assert_equal_int label expected actual = - if expected <> actual then failf "%s: expected %d, got %d" label expected actual - -let assert_int_at_most label limit actual = - if actual > limit then failf "%s: expected at most %d, got %d" label limit actual - -let assert_upstream_storage_addresses label addresses = - if List.mem "datascript/root" addresses || List.mem "datascript/tail" addresses then - failf "%s: storage should not use OCaml snapshot address names" label; - if not (List.mem "0" addresses) then failf "%s: storage should include upstream root address 0" label; - if not (List.mem "1" addresses) then failf "%s: storage should include upstream tail address 1" label; - if List.length addresses < 5 then - failf - "%s: storage should include root, tail, and separate index nodes, got [%s]" - label - (String.concat "," addresses) +let assert_lmdb_addresses label addresses = + if addresses <> [ "lmdb" ] then + failf "%s: expected LMDB storage address [lmdb], got [%s]" label (String.concat "," addresses) let assert_equal_triples label expected actual = let actual = List.map (fun d -> d.e, d.a, d.v) actual in @@ -40,15 +27,6 @@ let indexed = ; tuple_types = None } -let unique_identity = { indexed with unique = Some Identity } - -let remove_dir_if_exists dir = - if Sys.file_exists dir then begin - Sys.readdir dir - |> Array.iter (fun name -> Sys.remove (Filename.concat dir name)); - Unix.rmdir dir - end - let small_db ?storage () = empty_db ?storage () |> db_with @@ -57,37 +35,11 @@ let small_db ?storage () = ; Add (Entity_id 3, "name", String "Petr") ] -let large_db ?storage () = - empty_db ?storage () - |> db_with - (List.init 1000 (fun index -> - let entity_id = index + 1 in - Add (Entity_id entity_id, "str", String (string_of_int entity_id)))) - -let counting_storage () = - let storage = memory_storage () in - let writes = ref [] in - let storage_store entries = - writes := !writes @ List.map fst entries; - storage.storage_store entries - in - { storage with storage_store }, writes - -let restore_counting_storage storage = - let reads = ref [] in - let storage_restore address = - reads := address :: !reads; - storage.storage_restore address - in - { storage with storage_restore }, reads - -let reset_writes writes = writes := [] - let test_storage__test_basics () = let storage = memory_storage () in let db = small_db () in store ~storage db; - assert_upstream_storage_addresses "store writes upstream storage addresses" (storage_addresses storage); + assert_lmdb_addresses "store writes LMDB storage address" (storage_addresses storage); (match restore storage with | None -> failwith "restore should read stored db" | Some restored -> @@ -105,63 +57,6 @@ let test_storage__test_basics () = | Some restored -> if schema restored <> [ "name", indexed ] then failwith "restore should preserve schema") -let test_storage__test_upstream_wire_addresses () = - let storage = memory_storage () in - let db = small_db () in - store ~storage db; - let addresses = storage_addresses storage in - if List.mem "datascript/root" addresses || List.mem "datascript/tail" addresses then - failwith "storage should not use OCaml snapshot address names"; - (match storage.storage_restore "0", storage.storage_restore "1" with - | Some _, Some (Storage_tail []) -> () - | None, _ -> failwith "storage should write upstream root address 0" - | _, None -> failwith "storage should write upstream tail address 1" - | _, Some _ -> failwith "storage tail address should contain the transaction tail"); - if List.length addresses < 5 then - failf - "storage should write root, tail, and separate index nodes, got [%s]" - (String.concat "," addresses) - -let test_storage__test_file_storage () = - let dir = - Filename.concat - (Filename.get_temp_dir_name ()) - ("datascript_ocaml_storage_" ^ string_of_int (Random.bits ())) - in - remove_dir_if_exists dir; - Fun.protect - ~finally:(fun () -> remove_dir_if_exists dir) - (fun () -> - let storage = file_storage dir in - let db = small_db () in - store ~storage db; - store_tail storage [ [ datom ~tx:(tx0 + 2) ~e:1 ~a:"name" ~v:(String "Alex") () ] ]; - let restored_storage = file_storage dir in - assert_upstream_storage_addresses "file_storage lists persisted addresses" (storage_addresses restored_storage); - match restore restored_storage with - | None -> failwith "file_storage should restore stored db" - | Some restored -> - assert_equal_triples - "file_storage restores root and replays persisted tail" - [ 1, "name", String "Alex"; 2, "name", String "Oleg"; 3, "name", String "Petr" ] - (datoms restored Eavt ())) - -let test_storage__test_gc () = - let storage = memory_storage () in - let db = small_db () in - store ~storage db; - store_tail storage [ [ datom ~tx:(tx0 + 2) ~e:1 ~a:"name" ~v:(String "Alex") () ] ]; - storage.storage_store [ "stale/node", Storage_tail [] ]; - collect_garbage storage; - assert_upstream_storage_addresses "collect_garbage keeps live storage addresses" (storage_addresses storage); - match restore storage with - | None -> failwith "restore should work after garbage collection" - | Some restored -> - assert_equal_triples - "collect_garbage preserves restorable data" - [ 1, "name", String "Alex"; 2, "name", String "Oleg"; 3, "name", String "Petr" ] - (datoms restored Eavt ()) - let test_storage__test_restored_db_addresses () = let storage = memory_storage () in let db = small_db () in @@ -171,132 +66,12 @@ let test_storage__test_restored_db_addresses () = | Some db -> db | None -> failwith "restore should read stored db" in - assert_upstream_storage_addresses "addresses should include restored db live nodes" (addresses [ restored ]) - -let test_storage__test_restored_incremental_store_reuses_index_nodes () = - let storage, writes = counting_storage () in - let db = large_db () in - store ~storage db; - let restored = - match restore storage with - | Some db -> db - | None -> failwith "restore should read stored large db" - in - reset_writes writes; - store ~storage restored; - assert_int_at_most - "storing an unchanged restored db should not rewrite index nodes" - 2 - (List.length !writes); - reset_writes writes; - let db_after = - db_with [ Add (Entity_id 1001, "str", String "1001") ] restored - in - store ~storage db_after; - assert_int_at_most - "storing an incrementally changed restored db should write only changed index paths" - 8 - (List.length !writes); - assert_equal_triples - "incremental stored db remains restorable" - [ 1001, "str", String "1001" ] - (datoms db_after Eavt ~e:1001 ()); - reset_writes writes; - let db_after_replacement = - db_with [ Add (Entity_id 1, "str", String "changed") ] restored - in - store ~storage db_after_replacement; - assert_int_at_most - "storing a cardinality-one replacement should write only changed index paths" - 16 - (List.length !writes); - assert_equal_triples - "replacement stored db remains restorable" - [ 1, "str", String "changed" ] - (datoms db_after_replacement Eavt ~e:1 ()) - -let test_storage__test_restore_is_lazy () = - let storage = memory_storage () in - large_db () |> store ~storage; - let address_count = List.length (storage_addresses storage) in - if address_count < 20 then - failf "large stored db should have many index nodes, got %d" address_count; - let counted_storage, reads = restore_counting_storage storage in - let restored = - match restore counted_storage with - | Some db -> db - | None -> failwith "restore should read stored large db" - in - assert_int_at_most "restore should only read root and tail addresses" 2 - (List.length !reads); - ignore (Seq.uncons (datoms_seq restored Eavt ())); - let reads_after_first_datom = List.length !reads in - if reads_after_first_datom <= 2 then - failwith "reading the first datom should load the first index path"; - if reads_after_first_datom >= address_count then - failf - "reading the first datom should not restore every stored node: reads=%d addresses=%d" - reads_after_first_datom address_count - -let test_storage__test_restore_with_tail_is_lazy () = - let storage = memory_storage () in - large_db () |> store ~storage; - let address_count = List.length (storage_addresses storage) in - store_tail storage - [ - [ - datom ~tx:(tx0 + 2) ~e:1 ~a:"str" ~v:(String "1") ~added:false (); - datom ~tx:(tx0 + 2) ~e:1 ~a:"str" ~v:(String "changed") (); - ]; - ]; - let counted_storage, reads = restore_counting_storage storage in - let restored = - match restore counted_storage with - | Some db -> db - | None -> failwith "restore should read stored large db with tail" - in - if List.length !reads >= address_count then - failf - "restore tail replay should not restore every stored node: reads=%d addresses=%d" - (List.length !reads) address_count; - assert_equal_triples - "tail replay should apply raw datoms" - [ 1, "str", String "changed" ] - (datoms restored Eavt ~e:1 ()) - -let test_storage__test_transact_after_restore_uses_index_slices () = - let storage = memory_storage () in - large_db () |> store ~storage; - let baseline_storage, baseline_reads = restore_counting_storage storage in - let baseline = - match restore baseline_storage with - | Some db -> db - | None -> failwith "restore should read stored large db for baseline" - in - ignore (Seq.uncons (datoms_seq baseline Eavt ~e:1 ())); - let slice_read_count = List.length !baseline_reads in - let counted_storage, reads = restore_counting_storage storage in - let restored = - match restore counted_storage with - | Some db -> db - | None -> failwith "restore should read stored large db" - in - let db_after = - db_with [ Retract (Entity_id 1, "str", Some (String "1")) ] restored - in - if List.length !reads > slice_read_count + 8 then - failf - "transact after restore should use bounded index slices: reads=%d slice_reads=%d" - (List.length !reads) slice_read_count; - assert_equal_triples - "restored db transaction should retract the targeted fact" - [] - (datoms db_after Eavt ~e:1 ()) + assert_lmdb_addresses "addresses should include restored db live nodes" (addresses [ restored ]) let test_storage__test_conn () = let storage = memory_storage () in let conn = create_conn ~schema:[ "name", indexed ] ~storage () in - assert_upstream_storage_addresses "storage-backed create_conn stores upstream addresses" (storage_addresses storage); + assert_lmdb_addresses "storage-backed create_conn stores LMDB address" (storage_addresses storage); ignore (transact_conn conn [ Add (Entity_id 1, "name", String "Ivan") ]); ignore (transact_conn conn [ Add (Entity_id 2, "name", String "Oleg") ]); let restored = @@ -305,7 +80,7 @@ let test_storage__test_conn () = | None -> failwith "restore_conn should restore storage-backed conn" in assert_equal_triples - "restore_conn replays transaction tail" + "restore_conn returns stored facts" [ 1, "name", String "Ivan"; 2, "name", String "Oleg" ] (datoms (conn_db restored) Eavt ()); ignore (transact_conn ~tx_meta:[ "skip-store?", Bool true ] restored [ Add (Entity_id 3, "name", String "Skipped") ]); @@ -315,81 +90,9 @@ let test_storage__test_conn () = assert_equal_triples "skip-store transaction is not persisted" [ 1, "name", String "Ivan"; 2, "name", String "Oleg" ] - (datoms restored_db Eavt ())); - ignore - (transact_conn - restored - (List.init 34 (fun index -> - let entity_id = index + 4 in - Add (Entity_id entity_id, "name", String (string_of_int entity_id))))); - (match storage.storage_restore "1" with - | Some (Storage_tail []) -> () - | _ -> failwith "overflowing storage-backed conn tail should compact"); - let from_db_storage = memory_storage () in - let from_db = - empty_db ~schema:[ "name", indexed ] ~storage:from_db_storage () - |> db_with [ Add (Entity_id 1, "name", String "Ivan") ] - in - ignore (conn_from_db from_db); - (match restore from_db_storage with - | Some restored_db -> - assert_equal_triples - "conn_from_db stores the initial attached db root" - [ 1, "name", String "Ivan" ] - (datoms restored_db Eavt ()) - | None -> failwith "conn_from_db should store attached dbs"); - let from_datoms_storage = memory_storage () in - ignore - (conn_from_datoms - ~schema:[ "name", indexed ] - ~storage:from_datoms_storage - [ datom ~e:3 ~a:"name" ~v:(String "Petr") () ]); - match restore from_datoms_storage with - | Some restored_db -> - assert_equal_triples - "conn_from_datoms stores the initial attached db root" - [ 3, "name", String "Petr" ] - (datoms restored_db Eavt ()) - | None -> failwith "conn_from_datoms should store attached datoms" - -let test_storage__test_db_with_tail () = - let db = - empty_db ~schema:[ "block/updated-at", indexed; "block/uuid", unique_identity ] () - |> db_with [ Add (Entity_id 1, "block/updated-at", Int 2); Add (Entity_id 1, "block/uuid", String "u1") ] - in - let tail = - [ [ datom ~tx:(tx0 + 3) ~e:1 ~a:"block/updated-at" ~v:(Int 1772979060646) () ] - ; [ datom ~tx:(tx0 + 4) ~e:1 ~a:"block/updated-at" ~v:(Int 1772979061145) () ] - ; [ datom ~tx:(tx0 + 5) ~e:2 ~a:"block/uuid" ~v:(String "u1") () - ; datom ~tx:(tx0 + 5) ~e:2 ~a:"block/title" ~v:(String "Rejected") () - ] - ; [ datom ~tx:(tx0 + 6) ~e:3 ~a:"block/title" ~v:(String "Later") () ] - ] - in - let restored = db_with_tail db tail in - assert_equal_triples - "db_with_tail retracts stale cardinality-one values" - [ 1, "block/updated-at", Int 1772979061145 ] - (datoms restored Avet ~a:"block/updated-at" ()); - assert_equal_triples - "db_with_tail drops rejected unique-conflict tail groups" - [] - (datoms restored Eavt ~e:2 ()); - assert_equal_triples - "db_with_tail keeps later valid groups" - [ 3, "block/title", String "Later" ] - (datoms restored Eavt ~e:3 ()); - assert_equal_int "db_with_tail advances max tx" (tx0 + 6) restored.max_tx + (datoms restored_db Eavt ())) let () = test_storage__test_basics (); - test_storage__test_upstream_wire_addresses (); - test_storage__test_file_storage (); - test_storage__test_gc (); test_storage__test_restored_db_addresses (); - test_storage__test_restored_incremental_store_reuses_index_nodes (); - test_storage__test_restore_is_lazy (); - test_storage__test_restore_with_tail_is_lazy (); - test_storage__test_transact_after_restore_uses_index_slices (); - test_storage__test_conn (); - test_storage__test_db_with_tail () + test_storage__test_conn () diff --git a/type/datascript_types.ml b/type/datascript_types.ml index c09e317..b0ccf6e 100644 --- a/type/datascript_types.ml +++ b/type/datascript_types.ml @@ -69,6 +69,8 @@ type datom = ; added : bool } +type index_set + type serializable_db = { serializable_schema : schema ; serializable_datoms : datom list @@ -78,23 +80,7 @@ type serializable_db = type storage_address = string -type storage_root = - { storage_schema : schema - ; storage_max_eid : entity_id - ; storage_max_tx : tx - ; storage_eavt : storage_address - ; storage_aevt : storage_address - ; storage_avet : storage_address - ; storage_duplicate_datoms : datom list - ; storage_max_addr : int - ; storage_branching_factor : int - ; storage_ref_type : Persistent_sorted_set.ref_type - } - -type storage_payload = - | Storage_root of storage_root - | Storage_node of datom Persistent_sorted_set.stored_node - | Storage_tail of datom list list +type storage_payload = Storage_session type storage = { storage_store : (storage_address * storage_payload) list -> unit @@ -114,7 +100,7 @@ and tx_entity = ; attrs : (attr * tx_value) list } -type tx_op = +and tx_op = | Add of entity_ref * attr * value | Retract of entity_ref * attr * value option | RetractEntity of entity_ref @@ -130,11 +116,12 @@ type tx_op = and db = { db_uid : int ; schema : schema - ; eavt_index : datom Persistent_sorted_set.t - ; aevt_index : datom Persistent_sorted_set.t - ; avet_index : datom Persistent_sorted_set.t - ; aevt_by_attr : (attr, datom list) Hashtbl.t - ; avet_by_attr : (attr, datom list) Hashtbl.t + ; eavt_index : index_set + ; aevt_index : index_set + ; avet_index : index_set + ; aevt_by_attr : (attr, datom array) Hashtbl.t + ; avet_by_attr : (attr, datom array) Hashtbl.t + ; avet_entities_by_attr_value : (attr * value, entity_id list) Hashtbl.t ; duplicate_datoms : datom list ; duplicate_aevt_datoms : datom list ; duplicate_avet_datoms : datom list @@ -518,3 +505,301 @@ type tx_report = ; tempids : (string * entity_id) list ; tx_meta : tx_meta } +module Compare = struct + let split_keyword keyword = + match String.index_opt keyword '/' with + | None -> "", keyword + | Some index -> + let namespace = String.sub keyword 0 index in + let name = String.sub keyword (index + 1) (String.length keyword - index - 1) in + namespace, name + + let rec compare_list_items_with compare_item left right = + match left, right with + | [], [] -> 0 + | left :: left_rest, right :: right_rest -> + let comparison = compare_item left right in + if comparison <> 0 then comparison else compare_list_items_with compare_item left_rest right_rest + | [], _ | _, [] -> 0 + + let compare_list_with compare_item left right = + let length_comparison = compare (List.length left) (List.length right) in + if length_comparison <> 0 then length_comparison + else compare_list_items_with compare_item left right + + let compare_option_with compare_item left right = + match left, right with + | None, None -> 0 + | None, Some _ -> -1 + | Some _, None -> 1 + | Some left, Some right -> compare_item left right + + let i32 value = Int32.of_int value + let i32_to_int value = Int32.to_int value + let i32_add left right = Int32.add left right + let i32_mul left right = Int32.mul left right + let i32_xor left right = Int32.logxor left right + let i32_shift_left value bits = Int32.shift_left value bits + let i32_shift_right value bits = Int32.shift_right value bits + let i32_shift_right_logical value bits = Int32.shift_right_logical value bits + + let i32_rotate_left value bits = + Int32.logor (Int32.shift_left value bits) (Int32.shift_right_logical value (32 - bits)) + + let murmur3_mix_k1 value = + value + |> fun value -> i32_mul value (i32 (-862048943)) + |> fun value -> i32_rotate_left value 15 + |> fun value -> i32_mul value (i32 461845907) + + let murmur3_mix_h1 hash value = + i32_xor hash value + |> fun hash -> i32_rotate_left hash 13 + |> fun hash -> i32_add (i32_mul hash (i32 5)) (i32 (-430675100)) + + let murmur3_fmix hash length = + i32_xor hash (i32 length) + |> fun hash -> i32_xor hash (i32_shift_right_logical hash 16) + |> fun hash -> i32_mul hash (i32 (-2048144789)) + |> fun hash -> i32_xor hash (i32_shift_right_logical hash 13) + |> fun hash -> i32_mul hash (i32 (-1028477387)) + |> fun hash -> i32_xor hash (i32_shift_right_logical hash 16) + + let murmur3_hash_int value = + if value = 0 then 0 + else + value + |> i32 + |> murmur3_mix_k1 + |> murmur3_mix_h1 Int32.zero + |> fun hash -> murmur3_fmix hash 4 + |> i32_to_int + + let murmur3_hash_long value = + if value = Int64.zero then 0 + else + let low = Int64.to_int value |> i32 in + let high = Int64.shift_right_logical value 32 |> Int64.to_int |> i32 in + Int32.zero + |> fun hash -> murmur3_mix_h1 hash (murmur3_mix_k1 low) + |> fun hash -> murmur3_mix_h1 hash (murmur3_mix_k1 high) + |> fun hash -> murmur3_fmix hash 8 + |> i32_to_int + + let murmur3_hash_unencoded_chars text = + let hash = ref Int32.zero in + let index = ref 1 in + let length = String.length text in + while !index < length do + let code = + Char.code text.[!index - 1] lor (Char.code text.[!index] lsl 16) + in + hash := murmur3_mix_h1 !hash (murmur3_mix_k1 (i32 code)); + index := !index + 2 + done; + if length land 1 = 1 then + hash := i32_xor !hash (murmur3_mix_k1 (i32 (Char.code text.[length - 1]))); + murmur3_fmix !hash (2 * length) |> i32_to_int + + let java_string_hash text = + let hash = ref Int32.zero in + String.iter + (fun ch -> hash := i32_add (i32_mul !hash (i32 31)) (i32 (Char.code ch))) + text; + i32_to_int !hash + + let hex_value = function + | '0' .. '9' as ch -> Char.code ch - Char.code '0' + | 'a' .. 'f' as ch -> 10 + Char.code ch - Char.code 'a' + | 'A' .. 'F' as ch -> 10 + Char.code ch - Char.code 'A' + | _ -> invalid_arg "invalid UUID hex digit" + + let uuid_halves uuid = + let digits = + uuid + |> String.to_seq + |> Seq.filter (( <> ) '-') + |> List.of_seq + in + if List.length digits <> 32 then invalid_arg ("invalid UUID: " ^ uuid); + let take_hex count digits = + let rec loop acc remaining rest = + if remaining = 0 then acc, rest + else + match rest with + | [] -> invalid_arg ("invalid UUID: " ^ uuid) + | ch :: rest -> + loop + (Int64.logor (Int64.shift_left acc 4) (Int64.of_int (hex_value ch))) + (remaining - 1) + rest + in + loop Int64.zero count digits + in + let most, rest = take_hex 16 digits in + let least, _ = take_hex 16 rest in + most, least + + let int64_low_i32 value = + Int64.logand value 0xffffffffL |> Int64.to_int |> i32 + + let int64_high_i32 value = + Int64.shift_right_logical value 32 |> int64_low_i32 + + let java_uuid_hash uuid = + let most, least = uuid_halves uuid in + i32_xor + (i32_xor (int64_high_i32 most) (int64_low_i32 most)) + (i32_xor (int64_high_i32 least) (int64_low_i32 least)) + |> i32_to_int + + let clojure_hash_combine seed hash = + i32_xor + (i32 seed) + (i32_add + (i32_add (i32 hash) (i32 (-1640531527))) + (i32_add (i32_shift_left (i32 seed) 6) (i32_shift_right (i32 seed) 2))) + |> i32_to_int + + let clojure_symbol_hash symbol = + let namespace, name = split_keyword symbol in + let namespace_hash = if namespace = "" then 0 else java_string_hash namespace in + clojure_hash_combine (murmur3_hash_unencoded_chars name) namespace_hash + + let clojure_keyword_hash name = + i32_add (i32 (clojure_symbol_hash name)) (i32 (-1640531527)) |> i32_to_int + + let murmur3_mix_coll_hash hash count = + hash + |> i32 + |> murmur3_mix_k1 + |> murmur3_mix_h1 Int32.zero + |> fun hash -> murmur3_fmix hash count + |> i32_to_int + + let murmur3_hash_ordered hashes = + let count, hash = + List.fold_left + (fun (count, hash) value_hash -> + count + 1, i32_add (i32_mul (i32 31) hash) (i32 value_hash)) + (0, i32 1) + hashes + in + murmur3_mix_coll_hash (i32_to_int hash) count + + let murmur3_hash_unordered hashes = + let count, hash = + List.fold_left + (fun (count, hash) value_hash -> count + 1, i32_add hash (i32 value_hash)) + (0, Int32.zero) + hashes + in + murmur3_mix_coll_hash (i32_to_int hash) count + + let rec clojure_hasheq = function + | Nil -> 0 + | Bool true -> 1231 + | Bool false -> 1237 + | Int value -> murmur3_hash_long (Int64.of_int value) + | Float value -> Hashtbl.hash value + | String value -> murmur3_hash_int (java_string_hash value) + | Symbol value -> clojure_symbol_hash value + | Keyword value -> clojure_keyword_hash value + | List values | Vector values -> murmur3_hash_ordered (List.map clojure_hasheq values) + | Set values -> murmur3_hash_unordered (List.map clojure_hasheq values) + | Map entries -> + entries + |> List.map (fun (key, value) -> murmur3_hash_ordered [ clojure_hasheq key; clojure_hasheq value ]) + |> murmur3_hash_unordered + | Tuple values -> + values + |> List.map (function None -> 0 | Some value -> clojure_hasheq value) + |> murmur3_hash_ordered + | Ref value -> murmur3_hash_long (Int64.of_int value) + | Uuid value -> java_uuid_hash value + | Instant value -> murmur3_hash_long (Int64.of_int value) + | Regex value -> Hashtbl.hash value + | TxRef -> Hashtbl.hash TxRef + | Ref_to value -> Hashtbl.hash (Ref_to value) + + let value_type_rank = function + | Nil -> 0 + | Keyword _ -> 1 + | Symbol _ -> 2 + | Map _ -> 3 + | Set _ -> 4 + | List _ -> 5 + | Vector _ -> 6 + | Tuple _ -> 7 + | Bool _ -> 8 + | Int _ | Float _ | Ref _ -> 9 + | String _ -> 10 + | Regex _ -> 11 + | Instant _ -> 12 + | Uuid _ -> 13 + | TxRef -> 14 + | Ref_to _ -> 15 + + let rec compare_value left right = + match left, right with + | Int left, Int right -> compare left right + | Float left, Float right -> compare left right + | Int left, Float right -> compare (float_of_int left) right + | Float left, Int right -> compare left (float_of_int right) + | Ref left, Ref right -> compare left right + | Int left, Ref right -> compare left right + | Ref left, Int right -> compare left right + | Float left, Ref right -> compare left (float_of_int right) + | Ref left, Float right -> compare (float_of_int left) right + | String left, String right -> compare left right + | Symbol left, Symbol right -> compare (split_keyword left) (split_keyword right) + | Bool left, Bool right -> compare left right + | Uuid left, Uuid right -> compare left right + | Instant left, Instant right -> compare left right + | Regex left, Regex right -> compare left right + | Nil, Nil -> 0 + | Keyword left, Keyword right -> compare (split_keyword left) (split_keyword right) + | List left, List right -> compare_list_with compare_value left right + | Vector left, Vector right -> compare_list_with compare_value left right + | List left, Tuple right -> + compare_list_with (compare_option_with compare_value) (List.map (fun value -> Some value) left) right + | Set _, Set _ -> compare (clojure_hasheq left) (clojure_hasheq right) + | Map _, Map _ -> compare (clojure_hasheq left) (clojure_hasheq right) + | Tuple left, Tuple right -> compare_list_with (compare_option_with compare_value) left right + | Tuple left, List right -> + compare_list_with (compare_option_with compare_value) left (List.map (fun value -> Some value) right) + | _ -> + let rank_comparison = compare (value_type_rank left) (value_type_rank right) in + if rank_comparison <> 0 then rank_comparison else compare left right + + and compare_map_entry (left_key, left_value) (right_key, right_value) = + let comparison = compare_value left_key right_key in + if comparison <> 0 then comparison else compare_value left_value right_value + + let first_nonzero4 first second third fourth = + if first <> 0 then first + else if second <> 0 then second + else if third <> 0 then third + else fourth + + let compare_datom index left right = + match index with + | Eavt -> + first_nonzero4 + (compare left.e right.e) + (compare left.a right.a) + (compare_value left.v right.v) + (compare left.tx right.tx) + | Aevt -> + first_nonzero4 + (compare left.a right.a) + (compare left.e right.e) + (compare_value left.v right.v) + (compare left.tx right.tx) + | Avet -> + first_nonzero4 + (compare left.a right.a) + (compare_value left.v right.v) + (compare left.e right.e) + (compare left.tx right.tx) +end diff --git a/type/dune b/type/dune index 353901d..4ea880b 100644 --- a/type/dune +++ b/type/dune @@ -2,4 +2,4 @@ (name datascript_types) (public_name datascript_ocaml.types) (modes native byte melange) - (libraries persistent_sorted_set_ocaml)) + (modules datascript_types))