diff --git a/.gitignore b/.gitignore index ef15086ad3..54d04c0419 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,8 @@ build *.py[cod] *$py.class __pycache__/ +3rdparty/loro/target +3rdparty/mogan-loro-ffi/target # editor files *.sln @@ -60,3 +62,6 @@ record # ImGui imgui.ini + +# node +tools/loro-relay/node_modules/ diff --git a/3rdparty/loro/Cargo.lock b/3rdparty/loro/Cargo.lock new file mode 100644 index 0000000000..e9d15f5aba --- /dev/null +++ b/3rdparty/loro/Cargo.lock @@ -0,0 +1,2716 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "addr2line" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "getrandom 0.3.3", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstyle" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd" + +[[package]] +name = "anyhow" +version = "1.0.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0674a1ddeecb70197781e945de4b3b8ffb61fa939a5597bcf48503737663100" + +[[package]] +name = "append-only-bytes" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac436601d6bdde674a0d7fb593e829ffe7b3387c351b356dd20e2d40f5bf3ee5" + +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + +[[package]] +name = "arref" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ccd462b64c3c72f1be8305905a85d85403d768e8690c9b8bd3b9009a5761679" + +[[package]] +name = "atomic-polyfill" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cf2bce30dfe09ef0bfaef228b9d414faaf7e563035494d7fe092dba54b300f4" +dependencies = [ + "critical-section", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi 0.1.19", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "backtrace" +version = "0.3.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6806a6321ec58106fea15becdad98371e28d92ccbc7c8f1b3b6dd724fe8f1002" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide 0.8.9", + "object", + "rustc-demangle", + "windows-targets 0.52.6", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2261d10cca569e4643e526d8dc2e62e433cc8aba21ab764233731f8d369bf394" + +[[package]] +name = "bitmaps" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031043d04099746d8db04daf1fa424b2bc8bd69d92b25962dcde24da39ab64a2" +dependencies = [ + "typenum", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" + +[[package]] +name = "bytecount" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "cc" +version = "1.2.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5252b3d2648e5eedbc1a6f501e3c795e07025c1e93bbf8bbdd6eef7f447a6d54" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fd1289c04a9ea8cb22300a459a72a385d7c73d3259e2ed7dcb2af674838cfa9" + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clap" +version = "3.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" +dependencies = [ + "bitflags 1.3.2", + "clap_lex 0.2.4", + "indexmap", + "textwrap", +] + +[[package]] +name = "clap" +version = "4.5.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eac00902d9d136acd712710d71823fb8ac8004ca445a89e73a41d45aa712931" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.5.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ad9bbf750e73b5884fb8a211a9424a1906c1e156724260fdae972f31d70e1d6" +dependencies = [ + "anstyle", + "clap_lex 0.7.5", +] + +[[package]] +name = "clap_lex" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" +dependencies = [ + "os_str_bytes", +] + +[[package]] +name = "clap_lex" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" + +[[package]] +name = "cobs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1" +dependencies = [ + "thiserror 2.0.16", +] + +[[package]] +name = "color-backtrace" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "150fd80a270c0671379f388c8204deb6a746bb4eac8a6c03fe2460b2c0127ea0" +dependencies = [ + "backtrace", + "termcolor", +] + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "criterion" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c76e09c1aae2bc52b3d2f29e13c6572553b30c4aa1b8a49fd70de6412654cb" +dependencies = [ + "anes", + "atty", + "cast", + "ciborium", + "clap 3.2.25", + "criterion-plot 0.5.0", + "itertools 0.10.5", + "lazy_static", + "num-traits", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap 4.5.47", + "criterion-plot 0.5.0", + "is-terminal", + "itertools 0.10.5", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1c047a62b0cc3e145fa84415a3191f628e980b194c2755aa12300a4e6cbd928" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap 4.5.47", + "criterion-plot 0.6.0", + "itertools 0.13.0", + "num-traits", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools 0.10.5", +] + +[[package]] +name = "criterion-plot" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b1bcc0dc7dfae599d84ad0b1a55f80cde8af3725da8313b528da95ef783e338" +dependencies = [ + "cast", + "itertools 0.13.0", +] + +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "ctor" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d2301688392eb071b0bf1a37be05c469d3cc4dbbd95df672fe28ab021e6a096" +dependencies = [ + "quote", + "syn 1.0.109", +] + +[[package]] +name = "ctor" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a2785755761f3ddc1492979ce1e48d2c00d09311c39e4466429188f3dd6501" +dependencies = [ + "quote", + "syn 2.0.106", +] + +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.106", +] + +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "diff" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "embedded-io" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" + +[[package]] +name = "embedded-io" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" + +[[package]] +name = "ensure-cov" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33753185802e107b8fa907192af1f0eca13b1fb33327a59266d650fef29b2b4e" + +[[package]] +name = "enum-as-inner" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9720bba047d567ffc8a3cba48bf19126600e249ab7f128e9233e6376976a116" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "enum-as-inner" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "enum_dispatch" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa18ce2bc66555b3218614519ac839ddb759a7d6720732f979ef8d13be147ecd" +dependencies = [ + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad" +dependencies = [ + "libc", + "windows-sys 0.60.2", +] + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "find-msvc-tools" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fd99930f64d146689264c637b5af2f0233a933bef0d8570e2526bf9e083192d" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "fractional_index" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce01fad85c73f947e62a39152c7b0f785af7f44859da36e421c42a3023bcbe3a" +dependencies = [ + "serde", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-executor" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "generator" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "605183a538e3e2a9c1038635cc5c2d194e2ee8fd0d1b66b8349fad7dbacce5a2" +dependencies = [ + "cc", + "cfg-if", + "libc", + "log", + "rustversion", + "windows", +] + +[[package]] +name = "generic-array" +version = "0.14.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "generic-btree" +version = "0.10.7" +dependencies = [ + "arbitrary", + "arref", + "heapless 0.9.1", + "itertools 0.11.0", + "loro-thunderdome", + "proc-macro2", + "rand 0.8.5", + "rustc-hash", +] + +[[package]] +name = "getrandom" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi 0.11.1+wasi-snapshot-preview1", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasi 0.14.4+wasi-0.2.4", +] + +[[package]] +name = "gimli" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" + +[[package]] +name = "half" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9" +dependencies = [ + "cfg-if", + "crunchy", +] + +[[package]] +name = "hash32" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c35f58762feb77d74ebe43bdbc3210f09be9fe6742234d573bacc26ed92b67" +dependencies = [ + "byteorder", +] + +[[package]] +name = "hash32" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d60b12902ba28e2730cd37e95b8c9223af2808df9e902d4df49588d1470606" +dependencies = [ + "byteorder", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" + +[[package]] +name = "heapless" +version = "0.7.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdc6457c0eb62c71aac4bc17216026d8410337c4126773b9c5daba343f17964f" +dependencies = [ + "atomic-polyfill", + "hash32 0.2.1", + "rustc_version", + "serde", + "spin", + "stable_deref_trait", +] + +[[package]] +name = "heapless" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bfb9eb618601c89945a70e254898da93b13be0388091d42117462b265bb3fad" +dependencies = [ + "hash32 0.3.1", + "stable_deref_trait", +] + +[[package]] +name = "heapless" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1edcd5a338e64688fbdcb7531a846cfd3476a54784dcb918a0844682bc7ada5" +dependencies = [ + "hash32 0.3.1", + "stable_deref_trait", +] + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "im" +version = "15.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0acd33ff0285af998aaf9b57342af478078f53492322fafc47450e09397e0e9" +dependencies = [ + "bitmaps", + "rand_core 0.6.4", + "rand_xoshiro", + "serde", + "sized-chunks", + "typenum", + "version_check", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + +[[package]] +name = "is-terminal" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9" +dependencies = [ + "hermit-abi 0.5.2", + "libc", + "windows-sys 0.59.0", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.3", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "leb128" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" + +[[package]] +name = "libc" +version = "0.2.175" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543" + +[[package]] +name = "linux-raw-sys" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" + +[[package]] +name = "loom" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "serde", + "serde_json", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "loro" +version = "1.13.7" +dependencies = [ + "anyhow", + "base64", + "ctor 0.2.9", + "enum-as-inner 0.6.1", + "generic-btree", + "loom", + "loro-common", + "loro-delta", + "loro-internal", + "loro-kv-store", + "pretty_assertions", + "rand 0.8.5", + "rustc-hash", + "serde_json", + "serial_test", + "tracing", +] + +[[package]] +name = "loro-common" +version = "1.13.1" +dependencies = [ + "arbitrary", + "enum-as-inner 0.6.1", + "js-sys", + "leb128", + "loro-rle", + "nonmax", + "rustc-hash", + "serde", + "serde_columnar", + "serde_json", + "thiserror 1.0.69", + "tracing", + "wasm-bindgen", +] + +[[package]] +name = "loro-delta" +version = "1.13.0" +dependencies = [ + "arrayvec", + "color-backtrace", + "criterion 0.4.0", + "ctor 0.2.9", + "enum-as-inner 0.5.1", + "generic-btree", + "heapless 0.8.0", + "rand 0.8.5", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "loro-internal" +version = "1.13.7" +dependencies = [ + "append-only-bytes", + "arbitrary", + "arref", + "bytes", + "criterion 0.7.0", + "ctor 0.1.26", + "either", + "ensure-cov", + "enum-as-inner 0.6.1", + "enum_dispatch", + "generic-btree", + "getrandom 0.2.16", + "im", + "itertools 0.12.1", + "js-sys", + "leb128", + "loom", + "loro-common", + "loro-delta", + "loro-kv-store", + "loro-rle", + "loro_fractional_index", + "md5", + "miniz_oxide 0.7.4", + "nonmax", + "num", + "num-traits", + "once_cell", + "parking_lot", + "pest", + "pest_derive", + "postcard", + "pretty_assertions", + "rand 0.8.5", + "rustc-hash", + "serde", + "serde_columnar", + "serde_json", + "smallvec", + "tabled", + "thiserror 1.0.69", + "thread_local", + "tracing", + "wasm-bindgen", + "xxhash-rust", + "zstd", +] + +[[package]] +name = "loro-kv-store" +version = "1.13.7" +dependencies = [ + "bytes", + "ctor 0.2.9", + "ensure-cov", + "loro-common", + "lz4_flex", + "once_cell", + "quick_cache", + "rand 0.8.5", + "rustc-hash", + "tracing", + "xxhash-rust", +] + +[[package]] +name = "loro-rle" +version = "1.6.0" +dependencies = [ + "append-only-bytes", + "ctor 0.1.26", + "num", + "proptest", + "rand 0.8.5", + "smallvec", + "smartstring", + "static_assertions", +] + +[[package]] +name = "loro-thunderdome" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f3d053a135388e6b1df14e8af1212af5064746e9b87a06a345a7a779ee9695a" + +[[package]] +name = "loro_fractional_index" +version = "1.13.0" +dependencies = [ + "criterion 0.5.1", + "fractional_index", + "once_cell", + "rand 0.8.5", + "serde", + "serde_json", +] + +[[package]] +name = "lz4_flex" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08ab2867e3eeeca90e844d1940eab391c9dc5228783db2ed999acbc0a9ed375a" +dependencies = [ + "twox-hash", +] + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "md5" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" + +[[package]] +name = "memchr" +version = "2.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" + +[[package]] +name = "miniz_oxide" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" +dependencies = [ + "adler", +] + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", +] + +[[package]] +name = "nonmax" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "610a5acd306ec67f907abe5567859a3c693fb9886eb1f012ab8f2a47bef3db51" + +[[package]] +name = "nu-ansi-term" +version = "0.50.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4a28e057d01f97e61255210fcff094d74ed0466038633e95017f5beb68e4399" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "object" +version = "0.36.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + +[[package]] +name = "os_str_bytes" +version = "6.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" + +[[package]] +name = "papergrid" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1526bb6aa9f10ec339fb10360f22c57edf81d5678d0278e93bc12a47ffbe4b01" +dependencies = [ + "bytecount", + "fnv", + "unicode-width", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link 0.2.1", +] + +[[package]] +name = "pest" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "989e7521a040efde50c3ab6bbadafbe15ab6dc042686926be59ac35d74607df4" +dependencies = [ + "memchr", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "187da9a3030dbafabbbfb20cb323b976dc7b7ce91fcd84f2f74d6e31d378e2de" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49b401d98f5757ebe97a26085998d6c0eecec4995cad6ab7fc30ffdf4b052843" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "pest_meta" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f27a2cfee9f9039c4d86faa5af122a0ac3851441a34865b8a043b46be0065a" +dependencies = [ + "pest", + "sha2", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + +[[package]] +name = "postcard" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24" +dependencies = [ + "cobs", + "embedded-io 0.4.0", + "embedded-io 0.6.1", + "heapless 0.7.17", + "serde", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "pretty_assertions" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d" +dependencies = [ + "diff", + "yansi", +] + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + +[[package]] +name = "proc-macro2" +version = "1.0.101" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "proptest" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fcdab19deb5195a31cf7726a210015ff1496ba1464fd42cb4f537b8b01b471f" +dependencies = [ + "bit-set", + "bit-vec", + "bitflags 2.9.4", + "lazy_static", + "num-traits", + "rand 0.9.2", + "rand_chacha 0.9.0", + "rand_xorshift", + "regex-syntax", + "rusty-fork", + "tempfile", + "unarray", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quick_cache" +version = "0.6.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ad6644cb07b7f3488b9f3d2fde3b4c0a7fa367cafefb39dff93a659f76eb786" +dependencies = [ + "ahash", + "equivalent", + "hashbrown 0.15.5", + "parking_lot", +] + +[[package]] +name = "quote" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.3", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.3", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.16", +] + +[[package]] +name = "rand_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +dependencies = [ + "getrandom 0.3.3", +] + +[[package]] +name = "rand_xorshift" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" +dependencies = [ + "rand_core 0.9.3", +] + +[[package]] +name = "rand_xoshiro" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa" +dependencies = [ + "rand_core 0.6.4", +] + +[[package]] +name = "rayon" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "redox_syscall" +version = "0.5.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5407465600fb0548f1442edf71dd20683c6ed326200ace4b1ef0763521bb3b77" +dependencies = [ + "bitflags 2.9.4", +] + +[[package]] +name = "regex" +version = "1.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23d7fd106d8c02486a8d64e778353d1cffe08ce79ac2e82f540c86d0facf6912" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b9458fa0bfeeac22b5ca447c63aaf45f28439a709ccd244698632f9aa6394d6" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caf4aa5b0f434c91fe5c7f1ecb6a5ece2130b02ad2a590589dda5146df959001" + +[[package]] +name = "rustc-demangle" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace" + +[[package]] +name = "rustc-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11181fbabf243db407ef8df94a6ce0b2f9a733bd8be4ad02b4eda9602296cac8" +dependencies = [ + "bitflags 2.9.4", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.60.2", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "rusty-fork" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb3dcc6e454c328bb824492db107ab7c0ae8fcffe4ad210136ef014458c1bc4f" +dependencies = [ + "fnv", + "quick-error", + "tempfile", + "wait-timeout", +] + +[[package]] +name = "ryu" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scc" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46e6f046b7fef48e2660c57ed794263155d713de679057f2d0c169bfc6e756cc" +dependencies = [ + "sdd", +] + +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "sdd" +version = "3.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490dcfcbfef26be6800d11870ff2df8774fa6e86d047e3e8c8a76b25655e41ca" + +[[package]] +name = "semver" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" + +[[package]] +name = "serde" +version = "1.0.226" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dca6411025b24b60bfa7ec1fe1f8e710ac09782dca409ee8237ba74b51295fd" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_columnar" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a16e404f17b16d0273460350e29b02d76ba0d70f34afdc9a4fa034c97d6c6eb" +dependencies = [ + "itertools 0.11.0", + "postcard", + "serde", + "serde_columnar_derive", + "thiserror 1.0.69", +] + +[[package]] +name = "serde_columnar_derive" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45958fce4903f67e871fbf15ac78e289269b21ebd357d6fecacdba233629112e" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "serde_core" +version = "1.0.226" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba2ba63999edb9dac981fb34b3e5c0d111a69b0924e253ed29d83f7c99e966a4" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.226" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8db53ae22f34573731bafa1db20f04027b2d25e02d8205921b569171699cdb33" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "serde_json" +version = "1.0.143" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d401abef1d108fbd9cbaebc3e46611f4b1021f714a0597a71f41ee463f5f4a5a" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "serial_test" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d0b343e184fc3b7bb44dff0705fffcf4b3756ba6aff420dddd8b24ca145e555" +dependencies = [ + "futures-executor", + "futures-util", + "log", + "once_cell", + "parking_lot", + "scc", + "serial_test_derive", +] + +[[package]] +name = "serial_test_derive" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f50427f258fb77356e4cd4aa0e87e2bd2c66dbcee41dc405282cae2bfc26c83" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "sized-chunks" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16d69225bde7a69b235da73377861095455d298f2b970996eec25ddbb42b3d1e" +dependencies = [ + "bitmaps", + "typenum", +] + +[[package]] +name = "slab" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +dependencies = [ + "serde", +] + +[[package]] +name = "smartstring" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fb72c633efbaa2dd666986505016c32c3044395ceaf881518399d2f4127ee29" +dependencies = [ + "autocfg", + "static_assertions", + "version_check", +] + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tabled" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56c3ee73732ffceaea7b8f6b719ce3bb17f253fa27461ffeaf568ebd0cdb4b85" +dependencies = [ + "papergrid", + "tabled_derive", + "unicode-width", +] + +[[package]] +name = "tabled_derive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "beca1b4eaceb4f2755df858b88d9b9315b7ccfd1ffd0d7a48a52602301f01a57" +dependencies = [ + "heck 0.4.1", + "proc-macro-error", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "tempfile" +version = "3.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b61f8f20e3a6f7e0649d825294eaf317edce30f82cf6026e7e4cb9222a7d1e" +dependencies = [ + "fastrand", + "getrandom 0.3.3", + "once_cell", + "rustix", + "windows-sys 0.60.2", +] + +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "textwrap" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c13547615a44dc9c452a8a534638acdf07120d4b6847c8178705da06306a3057" + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3467d614147380f2e4e374161426ff399c91084acd2363eaf549172b3d5e60c0" +dependencies = [ + "thiserror-impl 2.0.16", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c5e1be1c48b9172ee610da68fd9cd2770e7a4056cb3fc98710ee6906f0c7960" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "tracing" +version = "0.1.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "tracing-core" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2054a14f5307d601f88daf0553e1cbf472acc4f2c51afab632431cdcd72124d5" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "twox-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" + +[[package]] +name = "typenum" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" + +[[package]] +name = "ucd-trie" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" + +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + +[[package]] +name = "unicode-ident" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" + +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasi" +version = "0.14.4+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88a5f4a424faf49c3c2c344f166f0662341d470ea185e939657aaff130f0ec4a" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" +dependencies = [ + "bumpalo", + "log", + "proc-macro2", + "quote", + "syn 2.0.106", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0978bf7171b3d90bac376700cb56d606feb40f251a475a5d6634613564460b22" +dependencies = [ + "windows-sys 0.60.2", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows" +version = "0.61.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" +dependencies = [ + "windows-collections", + "windows-core", + "windows-future", + "windows-link 0.1.3", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8" +dependencies = [ + "windows-core", +] + +[[package]] +name = "windows-core" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link 0.1.3", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-future" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" +dependencies = [ + "windows-core", + "windows-link 0.1.3", + "windows-threading", +] + +[[package]] +name = "windows-implement" +version = "0.60.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "windows-interface" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "windows-link" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-numerics" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1" +dependencies = [ + "windows-core", + "windows-link 0.1.3", +] + +[[package]] +name = "windows-result" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" +dependencies = [ + "windows-link 0.1.3", +] + +[[package]] +name = "windows-strings" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" +dependencies = [ + "windows-link 0.1.3", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.3", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5fe6031c4041849d7c496a8ded650796e7b6ecc19df1a431c1a363342e5dc91" +dependencies = [ + "windows-link 0.1.3", + "windows_aarch64_gnullvm 0.53.0", + "windows_aarch64_msvc 0.53.0", + "windows_i686_gnu 0.53.0", + "windows_i686_gnullvm 0.53.0", + "windows_i686_msvc 0.53.0", + "windows_x86_64_gnu 0.53.0", + "windows_x86_64_gnullvm 0.53.0", + "windows_x86_64_msvc 0.53.0", +] + +[[package]] +name = "windows-threading" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b66463ad2e0ea3bbf808b7f1d371311c80e115c0b71d60efc142cafbcfb057a6" +dependencies = [ + "windows-link 0.1.3", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" + +[[package]] +name = "wit-bindgen" +version = "0.45.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c573471f125075647d03df72e026074b7203790d41351cd6edc96f46bcccd36" + +[[package]] +name = "xxhash-rust" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" + +[[package]] +name = "yansi" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" + +[[package]] +name = "zerocopy" +version = "0.8.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0894878a5fa3edfd6da3f88c4805f4c8558e2b996227a3d864f47fe11e38282c" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/3rdparty/loro/Cargo.toml b/3rdparty/loro/Cargo.toml new file mode 100644 index 0000000000..92988f52f0 --- /dev/null +++ b/3rdparty/loro/Cargo.toml @@ -0,0 +1,33 @@ +[workspace] +members = [ + "crates/loro", + "crates/rle", + "crates/loro-common", + "crates/loro-internal", + "crates/fractional_index", + "crates/delta", + "crates/kv-store", + "crates/generic-btree", +] +resolver = "2" + +# Use the in-tree fork of generic-btree (loro-dev maintains it). This redirects +# every `generic-btree` dependency in the graph to the workspace crate. +[patch.crates-io] +generic-btree = { path = "crates/generic-btree" } + +[workspace.dependencies] +enum_dispatch = "0.3.11" +enum-as-inner = "0.6.0" +rustc-hash = "2.1.1" +tracing = { version = "0.1" } +serde_columnar = { version = "0.3.14", default-features = false } +serde_json = "1.0" +thiserror = "1" +smallvec = { version = "1.8.0", features = ["serde"] } +itertools = "0.12.1" +serde = "1" +bytes = "1" +once_cell = "1.18.0" +xxhash-rust = { version = "0.8.12", features = ["xxh32"] } +ensure-cov = "0.1.0" diff --git a/3rdparty/loro/LICENSE b/3rdparty/loro/LICENSE new file mode 100644 index 0000000000..bffe7781be --- /dev/null +++ b/3rdparty/loro/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 Loro + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/3rdparty/loro/README.md b/3rdparty/loro/README.md new file mode 100644 index 0000000000..9c40381489 --- /dev/null +++ b/3rdparty/loro/README.md @@ -0,0 +1,169 @@ +

+ + + + + +

+

+Loro +

+

+ Make your JSON data collaborative and version-controlled 🦜 +

+

+ loro-dev%2Floro | Trendshift +

+

+ + Documentation + + | + + Getting Started + + | + + Rust Doc + +

+

+ + + + + + + + + +

+ +https://github.com/loro-dev/loro/assets/18425020/fe246c47-a120-44b3-91d4-1e7232a5b4ac + +

+ ✨ Loro 1.0 is out! Read the announcement. +

+ +Loro is a [CRDTs(Conflict-free Replicated Data Types)](https://crdt.tech/) library that makes building [local-first][local-first] and collaborative apps easier. You can now use it in Rust, JS (via WASM), and Swift. + +# Features + +**Features Provided by CRDTs** + +- P2P Synchronization +- Automatic Merging +- Local Availability +- Scalability +- Delta Updates + +**Supported CRDT Algorithms** + +- 📝 Text Editing with [Fugue] +- 📙 [Rich Text CRDT](https://loro.dev/blog/loro-richtext) +- 🌲 [Moveable Tree](https://loro.dev/docs/tutorial/tree) +- 🚗 [Moveable List](https://loro.dev/docs/tutorial/list) +- 🗺️ [Last-Write-Wins Map](https://loro.dev/docs/tutorial/map) +- 🤝 Mergeable map-key children via `ensureMergeable*` / `ensure_mergeable_*` for lazy child container creation + +**Advanced Features in Loro** + +- 🚀 [Fast Document Loading](https://loro.dev/blog/v1.0) +- ⏱️ Fast [Time Travel](https://loro.dev/docs/tutorial/time_travel) Through History +- 🏛️ [Version Control with Real-Time Collaboration](https://loro.dev/blog/v1.0#version-control) +- 📦 [Shallow Snapshot](https://loro.dev/docs/advanced/shallow_snapshot) that Works like Git Shallow Clone + +https://github.com/user-attachments/assets/68e0017a-4987-4f71-b2cf-4ed28a210987 + +> In this example, we demonstrate importing an entire Loro codebase into a Loro-powered +> version controller, preserving the complete Git DAG history while enabling fast version switching. + +# Example + +[![Open in StackBlitz](https://developer.stackblitz.com/img/open_in_stackblitz.svg)](https://stackblitz.com/edit/loro-basic-test?file=test%2Floro-sync.test.ts) + +```ts +import { expect, test } from "vitest"; +import { LoroDoc, LoroList } from "loro-crdt"; + +test("sync example", () => { + // Sync two docs with two rounds of exchanges + + // Initialize document A + const docA = new LoroDoc(); + const listA: LoroList = docA.getList("list"); + listA.insert(0, "A"); + listA.insert(1, "B"); + listA.insert(2, "C"); + + // Export all updates from docA + const bytes: Uint8Array = docA.export({ mode: "update" }); + + // Simulate sending `bytes` across the network to another peer, B + + const docB = new LoroDoc(); + // Peer B imports the updates from A + docB.import(bytes); + + // B's state matches A's state + expect(docB.toJSON()).toStrictEqual({ + list: ["A", "B", "C"], + }); + + // Get the current version of docB + const version = docB.oplogVersion(); + + // Simulate editing at B: delete item 'B' + const listB: LoroList = docB.getList("list"); + listB.delete(1, 1); + + // Export the updates from B since the last sync point + const bytesB: Uint8Array = docB.export({ mode: "update", from: version }); + + // Simulate sending `bytesB` back across the network to A + + // A imports the updates from B + docA.import(bytesB); + + // A has the same state as B + expect(docA.toJSON()).toStrictEqual({ + list: ["A", "C"], + }); +}); +``` + +# DevTools + +## Loro Inspector + +You can use the [Loro Inspector](https://inspector.loro.dev) to inspect the state and history of a Loro document. + +https://github.com/user-attachments/assets/ceeb7450-80ce-42f2-aef4-2e08fa2d1f1b + +# Bindings + +You can find bindings for other programming languages in [loro-ffi](https://github.com/loro-dev/loro-ffi). + +# Blog + +- [Loro 1.0](https://loro.dev/blog/v1.0) +- [Movable tree CRDTs and Loro's implementation](https://loro.dev/blog/movable-tree) +- [Introduction to Loro's Rich Text CRDT](https://loro.dev/blog/loro-richtext) +- [Loro: Reimagine State Management with CRDTs](https://loro.dev/blog/loro-now-open-source) + +# Sponsors + +[![Sponsors](./sponsorkit/sponsors.svg)](https://github.com/sponsors/loro-dev) + +# Credits + +Loro draws inspiration from the innovative work of the following projects and individuals: + +- [Diamond-types](https://github.com/josephg/diamond-types): The [Event Graph Walker (Eg-walker)](https://loro.dev/docs/advanced/event_graph_walker) algorithm from @josephg has been adapted to reduce the computation and space usage of CRDTs. +- [Automerge](https://github.com/automerge/automerge): Their use of columnar encoding for CRDTs has informed our strategies for efficient data encoding. +- [Yjs](https://github.com/yjs/yjs): We have incorporated a similar algorithm for effectively merging collaborative editing operations, thanks to their pioneering work. +- [Matthew Weidner](https://mattweidner.com/): His work on the [Fugue](https://arxiv.org/abs/2305.00583) algorithm has been invaluable, enhancing our text editing capabilities. +- [Martin Kleppmann](https://martin.kleppmann.com/): His work on CRDTs has significantly influenced our comprehension of the field. + +[local-first]: https://www.inkandswitch.com/local-first/ +[Fugue]: https://arxiv.org/abs/2305.00583 diff --git a/3rdparty/loro/crates/delta/Cargo.toml b/3rdparty/loro/crates/delta/Cargo.toml new file mode 100644 index 0000000000..df79eaadf4 --- /dev/null +++ b/3rdparty/loro/crates/delta/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "loro-delta" +version = "1.13.0" +edition = "2021" +license = "MIT" +description = "Loro internal library. It's used as a delta between two versions of text or lists. It's also an efficient Rope." +homepage = "https://loro.dev" +repository = "https://github.com/loro-dev/loro/" +authors = ["Zixuan Chen", "Liang Zhao"] +categories = ["data-structures"] +keywords = ["rope"] + +[dependencies] +arrayvec = "0.7.4" +generic-btree = { version = "^0.10.7" } +heapless = "0.8.0" +enum-as-inner = "0.5.1" + +[dev-dependencies] +tracing = "0.1.40" +criterion = { version = "0.4", features = ["html_reports"] } +rand = { version = "0.8.5" } +tracing-subscriber = "0.3.18" +color-backtrace = "0.6.1" +ctor = "0.2" + + +[[bench]] +name = "rope" +harness = false diff --git a/3rdparty/loro/crates/delta/LICENSE b/3rdparty/loro/crates/delta/LICENSE new file mode 100644 index 0000000000..afe68a4136 --- /dev/null +++ b/3rdparty/loro/crates/delta/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Loro + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/3rdparty/loro/crates/delta/fuzz/.gitignore b/3rdparty/loro/crates/delta/fuzz/.gitignore new file mode 100644 index 0000000000..1a45eee776 --- /dev/null +++ b/3rdparty/loro/crates/delta/fuzz/.gitignore @@ -0,0 +1,4 @@ +target +corpus +artifacts +coverage diff --git a/3rdparty/loro/crates/delta/fuzz/Cargo.lock b/3rdparty/loro/crates/delta/fuzz/Cargo.lock new file mode 100644 index 0000000000..0e11c3881d --- /dev/null +++ b/3rdparty/loro/crates/delta/fuzz/Cargo.lock @@ -0,0 +1,1002 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "addr2line" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "arbitrary" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5a26814d8dcb93b0e5a0ff3c6d80a8843bafb21b39e8e18a6f05471870e110" +dependencies = [ + "derive_arbitrary", +] + +[[package]] +name = "arrayvec" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" + +[[package]] +name = "arref" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ccd462b64c3c72f1be8305905a85d85403d768e8690c9b8bd3b9009a5761679" + +[[package]] +name = "atomic-polyfill" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cf2bce30dfe09ef0bfaef228b9d414faaf7e563035494d7fe092dba54b300f4" +dependencies = [ + "critical-section", +] + +[[package]] +name = "autocfg" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" + +[[package]] +name = "backtrace" +version = "0.3.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b05800d2e817c8b3b4b54abd461726265fa9789ae34330622f2db9ee696f9d" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] +name = "bumpalo" +version = "3.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "cc" +version = "1.0.98" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41c270e7540d725e65ac7f1b212ac8ce349719624d7bcff99f8e2e488e8cf03f" +dependencies = [ + "jobserver", + "libc", + "once_cell", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-targets 0.52.5", +] + +[[package]] +name = "color-backtrace" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "150fd80a270c0671379f388c8204deb6a746bb4eac8a6c03fe2460b2c0127ea0" +dependencies = [ + "backtrace", + "termcolor", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" + +[[package]] +name = "critical-section" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7059fff8937831a9ae6f0fe4d658ffabf58f2ca96aa9dec1c889f936f705f216" + +[[package]] +name = "ctor" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edb49164822f3ee45b17acd4a208cfc1251410cf0cad9a833234c9890774dd9f" +dependencies = [ + "quote", + "syn 2.0.65", +] + +[[package]] +name = "derive_arbitrary" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.65", +] + +[[package]] +name = "dev-utils" +version = "0.1.0" +dependencies = [ + "chrono", + "color-backtrace", + "rand", + "tracing", + "tracing-chrome", + "tracing-subscriber", + "tracing-tree", +] + +[[package]] +name = "either" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dca9240753cf90908d7e4aac30f630662b02aebaa1b58a3cadabdb23385b58b" + +[[package]] +name = "enum-as-inner" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9720bba047d567ffc8a3cba48bf19126600e249ab7f128e9233e6376976a116" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + +[[package]] +name = "generic-btree" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "210507e6dec78bb1304e52a174bd99efdd83894219bf20d656a066a0ce2fedc5" +dependencies = [ + "arref", + "fxhash", + "heapless 0.7.17", + "itertools", + "loro-thunderdome", + "proc-macro2", +] + +[[package]] +name = "getrandom" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "gimli" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" + +[[package]] +name = "hash32" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c35f58762feb77d74ebe43bdbc3210f09be9fe6742234d573bacc26ed92b67" +dependencies = [ + "byteorder", +] + +[[package]] +name = "hash32" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d60b12902ba28e2730cd37e95b8c9223af2808df9e902d4df49588d1470606" +dependencies = [ + "byteorder", +] + +[[package]] +name = "heapless" +version = "0.7.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdc6457c0eb62c71aac4bc17216026d8410337c4126773b9c5daba343f17964f" +dependencies = [ + "atomic-polyfill", + "hash32 0.2.1", + "rustc_version", + "spin", + "stable_deref_trait", +] + +[[package]] +name = "heapless" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bfb9eb618601c89945a70e254898da93b13be0388091d42117462b265bb3fad" +dependencies = [ + "hash32 0.3.1", + "stable_deref_trait", +] + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "iana-time-zone" +version = "0.1.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "itertools" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" + +[[package]] +name = "jobserver" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2b099aaa34a9751c5bf0878add70444e1ed2dd73f347be99003d4577277de6e" +dependencies = [ + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.155" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" + +[[package]] +name = "libfuzzer-sys" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a96cfd5557eb82f2b83fed4955246c988d331975a002961b07c81584d107e7f7" +dependencies = [ + "arbitrary", + "cc", + "once_cell", +] + +[[package]] +name = "lock_api" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" + +[[package]] +name = "loro-delta" +version = "0.5.1" +dependencies = [ + "arrayvec", + "enum-as-inner", + "generic-btree", + "heapless 0.8.0", + "tracing", +] + +[[package]] +name = "loro-delta-fuzz" +version = "0.0.0" +dependencies = [ + "arbitrary", + "ctor", + "dev-utils", + "libfuzzer-sys", + "loro-delta", + "tracing", +] + +[[package]] +name = "loro-thunderdome" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f3d053a135388e6b1df14e8af1212af5064746e9b87a06a345a7a779ee9695a" + +[[package]] +name = "memchr" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" + +[[package]] +name = "miniz_oxide" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87dfd01fe195c66b572b37921ad8803d010623c0aca821bea2302239d155cdae" +dependencies = [ + "adler", +] + +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + +[[package]] +name = "nu-ansi-term" +version = "0.49.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c073d3c1930d0751774acf49e66653acecb416c3a54c6ec095a9b11caddb5a68" +dependencies = [ + "windows-sys 0.48.0", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "object" +version = "0.32.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + +[[package]] +name = "pin-project-lite" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "proc-macro2" +version = "1.0.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b33eb56c327dec362a9e55b3ad14f9d2f0904fb5a5b03b513ab5465399e9f43" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" + +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver", +] + +[[package]] +name = "ryu" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "semver" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" + +[[package]] +name = "serde" +version = "1.0.202" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "226b61a0d411b2ba5ff6d7f73a476ac4f8bb900373459cd00fab8512828ba395" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.202" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6048858004bcff69094cd972ed40a32500f153bd3be9f716b2eed2e8217c4838" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.65", +] + +[[package]] +name = "serde_json" +version = "1.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "smallvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2863d96a84c6439701d7a38f9de935ec562c8832cc55d1dde0f513b52fad106" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "thread_local" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" +dependencies = [ + "cfg-if", + "once_cell", +] + +[[package]] +name = "tracing" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.65", +] + +[[package]] +name = "tracing-chrome" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf0a738ed5d6450a9fb96e86a23ad808de2b727fd1394585da5cdd6788ffe724" +dependencies = [ + "serde_json", + "tracing-core", + "tracing-subscriber", +] + +[[package]] +name = "tracing-core" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" +dependencies = [ + "nu-ansi-term 0.46.0", + "sharded-slab", + "smallvec", + "thread_local", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "tracing-tree" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65139ecd2c3f6484c3b99bc01c77afe21e95473630747c7aca525e78b0666675" +dependencies = [ + "nu-ansi-term 0.49.0", + "tracing-core", + "tracing-log", + "tracing-subscriber", +] + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "valuable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.65", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.65", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets 0.52.5", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.5", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" +dependencies = [ + "windows_aarch64_gnullvm 0.52.5", + "windows_aarch64_msvc 0.52.5", + "windows_i686_gnu 0.52.5", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.5", + "windows_x86_64_gnu 0.52.5", + "windows_x86_64_gnullvm 0.52.5", + "windows_x86_64_msvc 0.52.5", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" diff --git a/3rdparty/loro/crates/delta/fuzz/Cargo.toml b/3rdparty/loro/crates/delta/fuzz/Cargo.toml new file mode 100644 index 0000000000..32dbe20fe0 --- /dev/null +++ b/3rdparty/loro/crates/delta/fuzz/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "loro-delta-fuzz" +version = "0.0.0" +publish = false +edition = "2021" + +[package.metadata] +cargo-fuzz = true + +[dependencies] +arbitrary = { version = "1.3.2", features = ["derive"] } +libfuzzer-sys = "0.4" +dev-utils = { path = "../../dev-utils" } +tracing = "0.1.40" +ctor = "0.2.8" + +[dependencies.loro-delta] +path = ".." + +# Prevent this from interfering with workspaces +[workspace] +members = ["."] + +[profile.release] +debug = 1 + +[[bin]] +name = "ot" +path = "fuzz_targets/ot.rs" +test = false +doc = false diff --git a/3rdparty/loro/crates/delta/fuzz/fuzz_targets/ot.rs b/3rdparty/loro/crates/delta/fuzz/fuzz_targets/ot.rs new file mode 100644 index 0000000000..5764478b99 --- /dev/null +++ b/3rdparty/loro/crates/delta/fuzz/fuzz_targets/ot.rs @@ -0,0 +1,6 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; +use loro_delta_fuzz::{run, Op}; + +fuzz_target!(|ops: Vec| run(ops, 5)); diff --git a/3rdparty/loro/crates/delta/fuzz/src/lib.rs b/3rdparty/loro/crates/delta/fuzz/src/lib.rs new file mode 100644 index 0000000000..b0a8bc8a98 --- /dev/null +++ b/3rdparty/loro/crates/delta/fuzz/src/lib.rs @@ -0,0 +1,159 @@ +use arbitrary::Arbitrary; +use loro_delta::{ + text_delta::{TextChunk, TextDelta}, + DeltaItem, +}; +use tracing::{debug_span, instrument, trace}; + +#[derive(Debug, Arbitrary)] +pub enum Op { + Insert { site: u8, pos: u16, text: u16 }, + Delete { site: u8, pos: u16, len: u16 }, + Sync { site: u8 }, +} + +pub struct Actor { + rope: TextDelta, + last_sync_version: usize, + pending: TextDelta, +} + +pub struct Manager { + server: TextDelta, + versions: Vec, + actors: Vec, +} + +#[instrument(skip(m))] +fn sync(m: &mut Manager, site: usize) { + let actor = &mut m.actors[site]; + let mut server_ops = TextDelta::new(); + for t in &m.versions[actor.last_sync_version..] { + server_ops.compose(t); + } + + let client_to_apply = server_ops.transform(&actor.pending, true); + + let client_ops = std::mem::take(&mut actor.pending); + + let server_to_apply = client_ops.transform(&server_ops, false); + + actor.rope.compose(&client_to_apply); + m.server.compose(&server_to_apply); + m.versions.push(server_to_apply); + actor.last_sync_version = m.versions.len(); +} + +pub fn run(mut ops: Vec, site_num: usize) { + let mut m = Manager { + server: TextDelta::new(), + versions: vec![], + actors: vec![], + }; + for _ in 0..site_num { + m.actors.push(Actor { + rope: TextDelta::new(), + last_sync_version: 0, + pending: TextDelta::new(), + }) + } + + for op in &mut ops { + match op { + Op::Insert { site, pos, text } => { + *site = ((*site as usize) % site_num) as u8; + let actor = &mut m.actors[*site as usize]; + let len = actor.rope.len(); + *pos = ((*pos as usize) % (len + 1)) as u16; + let pos = *pos as usize; + + actor.rope.insert_str(pos, text.to_string().as_str()); + if actor.pending.len() < pos { + actor.pending.push_retain(pos, ()); + } + actor.pending.insert_values( + pos, + TextChunk::from_long_str(text.to_string().as_str()).map(|chunk| { + DeltaItem::Replace { + value: chunk, + attr: Default::default(), + delete: 0, + } + }), + ); + } + Op::Delete { + site, + pos, + len: del_len, + } => { + *site = ((*site as usize) % site_num) as u8; + let actor = &mut m.actors[*site as usize]; + let len = actor.rope.len(); + if len == 0 { + continue; + } + *pos = ((*pos as usize) % len) as u16; + let pos = *pos as usize; + *del_len = ((*del_len as usize) % len) as u16; + let del_len = *del_len as usize; + let mut del = TextDelta::new(); + del.push_retain(pos, ()).push_delete(del_len); + actor.rope.compose(&del); + actor.pending.compose(&del); + } + Op::Sync { site } => { + *site = ((*site as usize) % site_num) as u8; + let site = *site as usize; + sync(&mut m, site); + } + } + } + + debug_span!("Round 1").in_scope(|| { + for i in 0..site_num { + sync(&mut m, i); + } + }); + debug_span!("Round 2").in_scope(|| { + for i in 0..site_num { + sync(&mut m, i); + } + }); + + let server_str = m.server.try_to_string().unwrap(); + for i in 0..site_num { + let actor = &m.actors[i]; + let rope_str = actor.rope.try_to_string().unwrap(); + assert_eq!(rope_str, server_str, "site {} ops={:#?}", i, &ops); + } +} + +#[cfg(test)] +mod tests { + use super::Op::*; + use super::*; + + #[ctor::ctor] + fn init() { + dev_utils::setup_test_log(); + } + + #[test] + fn test_run() { + let ops = vec![ + Insert { + site: 1, + pos: 0, + text: 65535, + }, + Sync { site: 1 }, + Insert { + site: 1, + pos: 5, + text: 0, + }, + ]; + run(ops, 2); + } +} diff --git a/3rdparty/loro/crates/delta/src/array_vec.rs b/3rdparty/loro/crates/delta/src/array_vec.rs new file mode 100644 index 0000000000..c5c485e68b --- /dev/null +++ b/3rdparty/loro/crates/delta/src/array_vec.rs @@ -0,0 +1,280 @@ +use std::fmt::Debug; +use std::ops::{Deref, DerefMut}; + +use generic_btree::rle::{HasLength, Mergeable, Sliceable, TryInsert}; +use heapless::Vec; + +use crate::delta_trait::{DeltaAttr, DeltaValue}; +use crate::{DeltaRope, DeltaRopeBuilder}; + +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[repr(transparent)] +pub struct ArrayVec { + vec: Vec, +} + +impl ArrayVec { + pub fn new() -> Self { + Self { vec: Vec::new() } + } + + pub fn insert_many(&mut self, pos: usize, values: Self) -> Result<(), Self> { + let self_len = self.len(); + let values_len = values.len(); + let Some(new_len) = self_len.checked_add(values_len) else { + return Err(values); + }; + + if pos > self_len || C < new_len { + return Err(values); + } + + // SAFETY: We have the ownership of the values + unsafe { + let ptr_start = self.vec.as_mut_ptr().add(pos); + ptr_start.copy_to(ptr_start.add(values_len), self_len - pos); + ptr_start.copy_from_nonoverlapping(values.as_ptr(), values_len); + self.vec.set_len(new_len); + } + + // This will not cause memory leak because the vec is on the stack + // And we already take the ownership of the elements inside the vec + std::mem::forget(values.vec); + Ok(()) + } + + pub fn from_many(mut iter: impl Iterator) -> impl Iterator { + let mut new = Self::new(); + std::iter::from_fn(move || { + let mut v = iter.next(); + while let Some(iv) = v { + if new.vec.push(iv).is_err() { + unreachable!() + } + + if new.vec.is_full() { + break; + } + + v = iter.next(); + } + + if new.is_empty() { + return None; + } + + Some(std::mem::take(&mut new)) + }) + } + + // The type system doesn't allow us to use unexported types in trait's associated types + // So we don't impl the trait for this + #[allow(clippy::should_implement_trait)] + pub fn into_iter(self) -> impl Iterator { + self.vec.into_iter() + } +} + +impl DeltaRope, Attr> { + pub fn from_many(iter: impl Iterator) -> Self { + let mut rope = DeltaRope::new(); + rope.insert_values( + 0, + ArrayVec::from_many(iter).map(|x| crate::DeltaItem::Replace { + value: x, + attr: Default::default(), + delete: 0, + }), + ); + + rope + } +} + +impl DeltaRopeBuilder, Attr> { + pub fn insert_many(mut self, value_iter: impl IntoIterator, attr: Attr) -> Self { + let iter = ArrayVec::from_many(value_iter.into_iter()); + for value in iter { + self = self.insert(value, attr.clone()); + } + self + } +} + +impl Default for ArrayVec { + fn default() -> Self { + Self::new() + } +} + +impl Deref for ArrayVec { + type Target = Vec; + + fn deref(&self) -> &Self::Target { + &self.vec + } +} + +impl DerefMut for ArrayVec { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.vec + } +} + +impl HasLength for ArrayVec { + fn rle_len(&self) -> usize { + self.len() + } +} + +impl Sliceable for ArrayVec { + fn _slice(&self, range: std::ops::Range) -> Self { + Self { + vec: Vec::from_slice(&self.vec.as_slice()[range]).unwrap(), + } + } + + fn split(&mut self, pos: usize) -> Self { + let right = self.slice(pos..); + self.vec.truncate(pos); + right + } +} + +impl Mergeable for ArrayVec { + fn can_merge(&self, rhs: &Self) -> bool { + C >= self.len() + rhs.len() + } + + fn merge_right(&mut self, rhs: &Self) { + self.extend_from_slice(rhs.as_slice()).unwrap(); + } + + fn merge_left(&mut self, left: &Self) { + match self.insert_many(0, left.clone()) { + Ok(_) => {} + Err(_) => unreachable!(), + } + } +} + +impl TryInsert for ArrayVec { + fn try_insert(&mut self, pos: usize, elem: Self) -> Result<(), Self> + where + Self: Sized, + { + self.insert_many(pos, elem) + } +} + +impl DeltaValue for ArrayVec where T: Clone + Debug {} + +impl From<[T; A]> for ArrayVec +where + T: Clone, +{ + fn from(array: [T; A]) -> Self { + let vec = Vec::from_slice(&array).unwrap(); + ArrayVec { vec } + } +} + +#[cfg(test)] +mod test { + use std::sync::Arc; + + use super::*; + + #[test] + fn test_insert() { + let mut array_vec: ArrayVec, 8> = + ArrayVec::from([Arc::new(1), Arc::new(2), Arc::new(3)]); + let b = ArrayVec::from([Arc::new(1), Arc::new(2), Arc::new(3)]); + array_vec.insert_many(1, b).unwrap(); + } + + #[cfg(miri)] + #[test] + fn miri_insert_many_rejects_out_of_bounds_position_before_pointer_arithmetic() { + let mut array_vec: ArrayVec = ArrayVec::from([1, 2, 3]); + let values = ArrayVec::from([4]); + + assert!(array_vec.insert_many(usize::MAX, values).is_err()); + assert_eq!(array_vec.as_slice(), &[1, 2, 3]); + } + + #[test] + fn test_slice() { + let array_vec: ArrayVec = ArrayVec::from([1, 2, 3, 4, 5]); + let sliced = array_vec._slice(1..3); + assert_eq!(sliced.as_slice(), &[2, 3]); + } + + #[test] + fn test_split() { + let mut array_vec: ArrayVec = ArrayVec::from([1, 2, 3, 4, 5]); + let right = array_vec.split(2); + assert_eq!(array_vec.as_slice(), &[1, 2]); + assert_eq!(right.as_slice(), &[3, 4, 5]); + } + + #[test] + fn test_can_merge() { + let array_vec1: ArrayVec = ArrayVec::from([1, 2, 3]); + let array_vec2: ArrayVec = ArrayVec::from([4, 5, 6]); + assert!(array_vec1.can_merge(&array_vec2)); + } + + #[test] + fn test_merge_right() { + let mut array_vec1: ArrayVec = ArrayVec::from([1, 2, 3]); + let array_vec2: ArrayVec = ArrayVec::from([4, 5, 6]); + array_vec1.merge_right(&array_vec2); + assert_eq!(array_vec1.as_slice(), &[1, 2, 3, 4, 5, 6]); + } + + #[test] + fn test_merge_left() { + let mut array_vec1: ArrayVec = ArrayVec::from([4, 5, 6]); + let array_vec2: ArrayVec = ArrayVec::from([1, 2, 3]); + array_vec1.merge_left(&array_vec2); + assert_eq!(array_vec1.as_slice(), &[1, 2, 3, 4, 5, 6]); + } + + #[test] + fn test_try_insert() { + let mut array_vec: ArrayVec = ArrayVec::from([1, 2, 5]); + let result = array_vec.try_insert(2, ArrayVec::from([3, 4])); + assert!(result.is_ok()); + assert_eq!(array_vec.as_slice(), &[1, 2, 3, 4, 5]); + } + + #[test] + fn test_merge_fail() { + let array_vec1: ArrayVec = ArrayVec::from([1, 2, 3, 4, 5]); + let array_vec2: ArrayVec = ArrayVec::from([6, 7, 8]); + let result = array_vec1.can_merge(&array_vec2); + assert!(!result); + } + + #[test] + fn test_try_insert_fail() { + let mut array_vec: ArrayVec = ArrayVec::from([1, 2, 4, 5, 6]); + let result = array_vec.try_insert(2, ArrayVec::from([3])); + assert!(result.is_err()); + } + + #[test] + fn test_insert_many_fail() { + let mut array_vec: ArrayVec = ArrayVec::from([1, 2, 3]); + let result = array_vec.insert_many(1, ArrayVec::from([4, 5, 6])); + assert!(result.is_err()); + } + + #[test] + #[should_panic] + fn merge_right_overflow() { + let mut array_vec: ArrayVec = ArrayVec::from([1, 2, 3]); + array_vec.merge_right(&ArrayVec::from([4, 5, 6])); + } +} diff --git a/3rdparty/loro/crates/delta/src/delta_item.rs b/3rdparty/loro/crates/delta/src/delta_item.rs new file mode 100644 index 0000000000..b64e3149e7 --- /dev/null +++ b/3rdparty/loro/crates/delta/src/delta_item.rs @@ -0,0 +1,374 @@ +use std::ops::RangeBounds; + +use super::*; +use generic_btree::rle::{CanRemove, TryInsert}; + +impl DeltaItem { + /// Including the delete length + pub fn delta_len(&self) -> usize { + match self { + DeltaItem::Retain { len, .. } => *len, + DeltaItem::Replace { + value, + attr: _, + delete, + } => value.rle_len() + delete, + } + } + + /// The real length of the item in the delta, excluding the delete length + pub fn data_len(&self) -> usize { + match self { + DeltaItem::Retain { len, .. } => *len, + DeltaItem::Replace { value, .. } => value.rle_len(), + } + } + + pub fn new_insert(value: V, attr: Attr) -> Self { + DeltaItem::Replace { + value, + attr, + delete: 0, + } + } +} + +impl DeltaItem { + pub fn new_delete(len: usize) -> Self { + DeltaItem::Replace { + value: Default::default(), + attr: Default::default(), + delete: len, + } + } +} + +impl HasLength for DeltaItem { + /// This would treat the len of the Delete as 0 + fn rle_len(&self) -> usize { + self.delta_len() + } +} + +impl Mergeable for DeltaItem { + fn can_merge(&self, rhs: &Self) -> bool { + match (self, rhs) { + (DeltaItem::Retain { attr: attr1, .. }, DeltaItem::Retain { attr: attr2, .. }) => { + attr1 == attr2 + } + ( + DeltaItem::Replace { + value: value1, + attr: attr1, + delete: _del1, + }, + DeltaItem::Replace { + value: value2, + attr: attr2, + delete: _del2, + }, + ) => value1.can_merge(value2) && attr1 == attr2, + _ => false, + } + } + + fn merge_right(&mut self, rhs: &Self) { + match (self, rhs) { + (DeltaItem::Retain { len: len1, .. }, DeltaItem::Retain { len: len2, .. }) => { + *len1 += len2 + } + ( + DeltaItem::Replace { + value: value1, + delete: del1, + .. + }, + DeltaItem::Replace { + value: value2, + delete: del2, + .. + }, + ) => { + value1.merge_right(value2); + *del1 += *del2; + } + _ => unreachable!(), + } + } + + fn merge_left(&mut self, left: &Self) { + match (self, left) { + (DeltaItem::Retain { len: len1, .. }, DeltaItem::Retain { len: len2, .. }) => { + *len1 += len2 + } + ( + DeltaItem::Replace { + value: value1, + delete: del1, + .. + }, + DeltaItem::Replace { + value: value2, + delete: del2, + .. + }, + ) => { + value1.merge_left(value2); + *del1 += del2; + } + _ => unreachable!(), + } + } +} + +impl Sliceable for DeltaItem { + fn _slice(&self, range: std::ops::Range) -> Self { + assert!(range.end <= self.rle_len()); + match self { + DeltaItem::Retain { len, attr } => { + assert!(range.end <= *len); + DeltaItem::Retain { + len: range.len(), + attr: attr.clone(), + } + } + DeltaItem::Replace { + value, + attr, + delete, + } => { + if range.end <= value.rle_len() { + let value = value._slice(range.clone()); + DeltaItem::Replace { + value, + attr: attr.clone(), + delete: 0, + } + } else if range.start >= value.rle_len() { + debug_assert!(range.end <= delete + value.rle_len()); + debug_assert!(range.len() <= *delete); + DeltaItem::new_delete(range.len()) + } else { + let delete_len = range.end - value.rle_len(); + debug_assert!(delete_len <= *delete); + let value = value._slice(range.start..value.rle_len()); + DeltaItem::Replace { + delete: delete_len, + value, + attr: attr.clone(), + } + } + } + } + } + + /// slice in-place + #[inline(always)] + fn slice_(&mut self, range: impl RangeBounds) { + *self = self.slice(range); + } + + fn split(&mut self, pos: usize) -> Self { + match self { + DeltaItem::Retain { len, attr } => { + let right_len = *len - pos; + *len = pos; + DeltaItem::Retain { + len: right_len, + attr: attr.clone(), + } + } + DeltaItem::Replace { + value, + attr, + delete, + } => { + if pos < value.rle_len() { + let right = value.split(pos); + let right_delete = *delete; + *delete = 0; + DeltaItem::Replace { + value: right, + attr: attr.clone(), + delete: right_delete, + } + } else { + let right_len = value.rle_len() + *delete - pos; + let right = DeltaItem::new_delete(right_len); + *delete -= right_len; + right + } + } + } + } + + /// Update the slice in the given range. + /// This method may split `self` into two or three parts. + /// If so, it will make `self` the leftmost part and return the next split parts. + /// + /// # Example + /// + /// If `self.rle_len() == 10`, `self.update(1..5)` will split self into three parts and update the middle part. + /// It returns the middle and the right part. + fn update_with_split( + &mut self, + range: impl RangeBounds, + f: impl FnOnce(&mut Self), + ) -> (Option, Option) { + let start = match range.start_bound() { + std::ops::Bound::Included(x) => *x, + std::ops::Bound::Excluded(x) => x + 1, + std::ops::Bound::Unbounded => 0, + }; + + let end = match range.end_bound() { + std::ops::Bound::Included(x) => x + 1, + std::ops::Bound::Excluded(x) => *x, + std::ops::Bound::Unbounded => self.rle_len(), + }; + + match (start == 0, end == self.rle_len()) { + (true, true) => { + f(self); + (None, None) + } + (true, false) => { + let right = self.split(end); + f(self); + if self.can_merge(&right) { + self.merge_right(&right); + (None, None) + } else { + (Some(right), None) + } + } + (false, true) => { + let mut right = self.split(start); + f(&mut right); + if self.can_merge(&right) { + self.merge_right(&right); + (None, None) + } else { + (Some(right), None) + } + } + (false, false) => { + let right = self.split(end); + let mut middle = self.split(start); + f(&mut middle); + if middle.can_remove() { + if self.can_merge(&right) { + self.merge_right(&right); + (None, None) + } else { + (Some(right), None) + } + } else if middle.can_merge(&right) { + middle.merge_right(&right); + if self.can_merge(&middle) { + self.merge_right(&middle); + (None, None) + } else { + (Some(middle), None) + } + } else if self.can_merge(&middle) { + self.merge_right(&middle); + (Some(right), None) + } else { + (Some(middle), Some(right)) + } + } + } + } +} + +impl TryInsert for DeltaItem { + fn try_insert(&mut self, pos: usize, elem: Self) -> Result<(), Self> + where + Self: Sized, + { + match (self, elem) { + ( + DeltaItem::Retain { len, attr }, + DeltaItem::Retain { + len: len_b, + attr: attr_b, + }, + ) => { + if attr == &attr_b { + *len += len_b; + Ok(()) + } else { + Err(DeltaItem::Retain { + len: len_b, + attr: attr_b, + }) + } + } + ( + DeltaItem::Replace { + value: l_value, + attr: l_attr, + delete: l_delete, + }, + DeltaItem::Replace { + value: r_value, + attr: r_attr, + delete: r_delete, + }, + ) => { + if l_value.rle_len() == 0 && r_value.rle_len() == 0 { + *l_delete += r_delete; + return Ok(()); + } + + if l_attr == &r_attr { + match l_value.try_insert(pos, r_value) { + Ok(_) => { + *l_delete += r_delete; + return Ok(()); + } + Err(r_value) => { + return Err(DeltaItem::Replace { + value: r_value, + attr: r_attr, + delete: r_delete, + }) + } + } + } + + Err(DeltaItem::Replace { + value: r_value, + attr: r_attr, + delete: r_delete, + }) + } + (_, a) => Err(a), + } + } +} + +impl CanRemove for DeltaItem { + fn can_remove(&self) -> bool { + match self { + DeltaItem::Retain { len, .. } => *len == 0, + DeltaItem::Replace { + value, + attr: _, + delete, + } => value.rle_len() == 0 && *delete == 0, + } + } +} + +impl Default for DeltaItem { + fn default() -> Self { + DeltaItem::Retain { + len: 0, + attr: Default::default(), + } + } +} + +impl DeltaValue for DeltaItem {} diff --git a/3rdparty/loro/crates/delta/src/delta_rope.rs b/3rdparty/loro/crates/delta/src/delta_rope.rs new file mode 100644 index 0000000000..66e9d2683f --- /dev/null +++ b/3rdparty/loro/crates/delta/src/delta_rope.rs @@ -0,0 +1,580 @@ +use std::{fmt::Debug, ops::Range}; + +use generic_btree::{rle::Sliceable, Cursor}; + +use crate::{ + delta_rope::rle_tree::LengthFinder, + delta_trait::{DeltaAttr, DeltaValue}, + DeltaItem, DeltaRope, DeltaRopeBuilder, +}; + +use self::rle_tree::Len; + +use super::iter::Iter; + +mod compose; +pub(crate) mod rle_tree; + +impl DeltaRope { + pub fn new() -> Self { + Self { + tree: Default::default(), + } + } + + pub fn first(&self) -> Option<&DeltaItem> { + let leaf = self.tree.first_leaf()?; + self.tree.get_elem(leaf) + } + + pub fn iter(&self) -> impl Iterator> { + self.tree.iter() + } + + /// Returns the length of the delta rope (insertions + retains). + pub fn len(&self) -> usize { + self.tree.root_cache().data_len as usize + } + + /// Returns the length of the delta rope (deletions + retains). + pub fn old_len(&self) -> usize { + self.tree.root_cache().delta_len as usize + } + + pub fn is_empty(&self) -> bool { + self.tree.is_empty() + } + + pub fn push_insert(&mut self, v: V, attr: Attr) -> &mut Self { + if v.rle_len() == 0 { + return self; + } + + let Some(leaf) = self.tree.last_leaf() else { + self.tree.push(DeltaItem::Replace { + value: v, + attr, + delete: 0, + }); + return self; + }; + let mut inserted = false; + self.tree.update_leaf(leaf, |item| { + if let DeltaItem::Replace { + value, + attr: a, + delete: _, + } = item + { + if value.can_merge(&v) && a == &attr { + value.merge_right(&v); + inserted = true; + return (true, None, None); + } + } + (false, None, None) + }); + + if !inserted { + self.tree.push(DeltaItem::Replace { + value: v, + attr, + delete: 0, + }); + } + + self + } + + pub fn push_retain(&mut self, retain: usize, attr: Attr) -> &mut Self { + if retain == 0 { + return self; + } + + let Some(leaf) = self.tree.last_leaf() else { + self.tree.push(DeltaItem::Retain { len: retain, attr }); + return self; + }; + + let mut inserted = false; + self.tree.update_leaf(leaf, |item| { + if let DeltaItem::Retain { len, attr: a } = item { + if a == &attr { + *len += retain; + inserted = true; + return (true, None, None); + } + } + (false, None, None) + }); + + if !inserted { + self.tree.push(DeltaItem::Retain { len: retain, attr }); + } + + self + } + + pub fn push_replace(&mut self, value: V, attr: Attr, delete: usize) -> &mut Self { + if value.rle_len() == 0 && delete == 0 { + return self; + } + + let Some(leaf) = self.tree.last_leaf() else { + self.tree.push(DeltaItem::Replace { + value, + attr, + delete, + }); + return self; + }; + + let mut inserted = false; + self.tree.update_leaf(leaf, |item| { + if let DeltaItem::Replace { + value: v, + attr: a, + delete: d, + } = item + { + if a == &attr && v.can_merge(&value) { + v.merge_right(&value); + *d += delete; + inserted = true; + return (true, None, None); + } + } + (false, None, None) + }); + + if !inserted { + self.tree.push(DeltaItem::Replace { + value, + attr, + delete, + }); + } + + self + } + + pub fn push_delete(&mut self, len: usize) -> &mut Self { + if len == 0 { + return self; + } + + let Some(leaf) = self.tree.last_leaf() else { + self.tree.push(DeltaItem::Replace { + value: Default::default(), + attr: Default::default(), + delete: len, + }); + return self; + }; + + let mut inserted = false; + self.tree.update_leaf(leaf, |item| { + if let DeltaItem::Replace { + value: _, + attr: _, + delete, + } = item + { + *delete += len; + inserted = true; + return (true, None, None); + } + (false, None, None) + }); + + if !inserted { + self.tree.push(DeltaItem::new_delete(len)); + } + + self + } + + pub fn push(&mut self, item: DeltaItem) -> &mut Self { + match item { + DeltaItem::Retain { len, attr } => self.push_retain(len, attr), + DeltaItem::Replace { + value, + attr, + delete, + } => self.push_replace(value, attr, delete), + } + } + + /// Returns an iterator that can iterate over the delta rope with a custom length. + /// + /// It's more controllable compared to the default iterator. + /// + /// - Iterating over the delta rope with a custom length. + /// - You can peek the next item. + /// + /// It's useful to implement algorithms related to Delta + pub fn iter_with_len(&self) -> Iter<'_, V, Attr> { + Iter::new(self) + } + + pub fn chop(&mut self) { + let mut last_leaf = self.tree.last_leaf(); + while let Some(last_leaf_idx) = last_leaf { + let elem = self.tree.get_elem(last_leaf_idx).unwrap(); + match elem { + DeltaItem::Retain { len: _, attr } if attr.attr_is_empty() => { + self.tree.remove_leaf(Cursor { + leaf: last_leaf_idx, + offset: 0, + }); + last_leaf = self.tree.last_leaf(); + } + _ => return, + } + } + } + + pub fn transform(&self, other: &Self, left_prior: bool) -> Self { + let mut this_iter = self.iter_with_len(); + let mut other_iter = other.iter_with_len(); + let mut transformed_delta = DeltaRope::new(); + + while this_iter.peek().is_some() || other_iter.peek().is_some() { + if this_iter.peek_is_insert() && (left_prior || !other_iter.peek_is_insert()) { + let insert_length; + match this_iter.peek().unwrap() { + DeltaItem::Replace { value, attr, .. } => { + insert_length = value.rle_len(); + transformed_delta.push_insert(value.clone(), attr.clone()); + } + DeltaItem::Retain { .. } => unreachable!(), + } + this_iter.next_with(insert_length).unwrap(); + } else if other_iter.peek_is_insert() { + let insert_length = other_iter.peek_insert_length(); + transformed_delta.push_retain(insert_length, Default::default()); + other_iter.next_with(insert_length).unwrap(); + } else { + // It's now either retains or deletes + let length = this_iter.peek_length().min(other_iter.peek_length()); + let this_op_peek = this_iter.peek().cloned(); + let other_op_peek = other_iter.peek().cloned(); + let _ = this_iter.next_with(length); + let _ = other_iter.next_with(length); + if other_op_peek.map(|x| x.is_delete()).unwrap_or(false) { + // It makes our deletes or retains redundant + continue; + } else if this_op_peek + .as_ref() + .map(|x| x.is_delete()) + .unwrap_or(false) + { + transformed_delta.push_delete(length); + } else { + transformed_delta.push_retain( + length, + this_op_peek + .map(|x| x.into_retain().unwrap().1) + .unwrap_or_default(), + ); + // FIXME: transform the attributes + } + } + } + + transformed_delta.chop(); + transformed_delta + } + + /// Transforms operation `self` against another operation `other` in such a way that the + /// impact of `other` is effectively included in `self`. + pub fn transform_(&mut self, other: &Self, left_prior: bool) { + *self = self.transform(other, left_prior); + } + + /// Transforms the position based on self + pub fn transform_pos(&self, mut pos: usize, left_prior: bool) -> usize { + let mut index = 0; + let mut iter = self.iter_with_len(); + while iter.peek().is_some() { + if iter.peek_is_retain() { + let DeltaItem::Retain { len, attr: _ } = iter.next().unwrap() else { + unreachable!() + }; + index += len; + if index > pos || (index == pos && !left_prior) { + return pos; + } + } else if iter.peek_is_insert() { + if index == pos && !left_prior { + return pos; + } + + let insert_len; + match iter.peek().unwrap() { + DeltaItem::Replace { value, .. } => { + insert_len = value.rle_len(); + } + _ => { + unreachable!() + } + } + + pos += insert_len; + index += insert_len; + iter.next_with(insert_len).unwrap(); + } else { + // Delete + match iter.next().unwrap() { + DeltaItem::Replace { delete, .. } => { + pos = pos.saturating_sub(delete); + if pos < index { + pos = index; + return pos; + } + } + DeltaItem::Retain { .. } => unreachable!(), + } + } + } + + pos + } +} + +impl PartialEq for DeltaRope { + fn eq(&self, other: &Self) -> bool { + if self.len() != other.len() { + return false; + } + + let mut a = self.iter_with_len(); + let mut b = other.iter_with_len(); + while let (Some(x), Some(y)) = (a.peek(), b.peek()) { + let len = x.delta_len().min(y.delta_len()); + match (x, y) { + ( + DeltaItem::Replace { + value: va, + attr: attr_a, + delete: d_a, + }, + DeltaItem::Replace { + value: vb, + attr: attr_b, + delete: d_b, + }, + ) => { + if attr_a != attr_b { + return false; + } + + let va_empty = va.rle_len() == 0; + let vb_empty = vb.rle_len() == 0; + if vb_empty || va_empty { + // both deletions + let min_del_len = (*d_a).min(*d_b); + if min_del_len == 0 { + return false; + } + + a.next_with_del(min_del_len).unwrap(); + b.next_with_del(min_del_len).unwrap(); + } else { + let len = (va.rle_len()).min(vb.rle_len()); + let va_slice = va.slice(..len); + let vb_slice = vb.slice(..len); + if va_slice != vb_slice { + return false; + } + + a.next_with(len).unwrap(); + b.next_with(len).unwrap(); + } + } + (DeltaItem::Retain { attr, .. }, DeltaItem::Retain { attr: b_attr, .. }) + if *attr == *b_attr => + { + a.next_with(len).unwrap(); + b.next_with(len).unwrap(); + } + _ => return false, + } + } + + a.peek().is_none() && b.peek().is_none() + } +} + +impl Default for DeltaRope { + fn default() -> Self { + Self::new() + } +} + +impl DeltaRope { + pub fn insert_values( + &mut self, + pos: usize, + values: impl IntoIterator>, + ) { + if self.is_empty() { + for value in values { + self.tree.push(value.clone()); + } + return; + } + + let pos = self.tree.query::(&pos).unwrap(); + // This would crash if values's number is large + self.tree + .insert_many_by_cursor(Some(pos.cursor), values.into_iter()); + } + + pub fn insert_value(&mut self, pos: usize, value: V, attr: Attr) { + if self.len() < pos { + self.push_retain(pos - self.len(), Default::default()); + } + + if pos == self.len() { + self.tree.push(DeltaItem::Replace { + value, + attr, + delete: 0, + }); + return; + } + + let pos = self.tree.query::(&pos).unwrap(); + self.tree.insert_by_path( + pos.cursor, + DeltaItem::Replace { + value, + attr, + delete: 0, + }, + ); + } + + fn update_attr_in_range(&mut self, range: Range, attr: &Attr) { + if range.start == range.end || self.is_empty() { + return; + } + + let from = self.tree.query::(&range.start).unwrap(); + let to = self.tree.query::(&range.end).unwrap(); + self.tree.update(from.cursor..to.cursor, &mut |item| { + match item { + DeltaItem::Retain { attr: a, .. } => { + a.compose(attr); + } + DeltaItem::Replace { attr: a, .. } => a.compose(attr), + } + + None + }); + } +} + +impl DeltaRopeBuilder { + pub fn new() -> Self { + Self { items: vec![] } + } + + pub fn insert(mut self, v: V, attr: Attr) -> Self { + if v.rle_len() == 0 { + return self; + } + + if let Some(DeltaItem::Replace { value, attr: a, .. }) = self.items.last_mut() { + if value.can_merge(&v) && a == &attr { + value.merge_right(&v); + return self; + } + } + + self.items.push(DeltaItem::Replace { + value: v, + attr, + delete: 0, + }); + self + } + + pub fn retain(mut self, retain: usize, attr: Attr) -> Self { + if retain == 0 { + return self; + } + + if let Some(DeltaItem::Retain { len, attr: a }) = self.items.last_mut() { + if *a == attr { + *len += retain; + return self; + } + } + + self.items.push(DeltaItem::Retain { len: retain, attr }); + self + } + + pub fn delete(mut self, len: usize) -> Self { + if len == 0 { + return self; + } + + if let Some(DeltaItem::Replace { delete, .. }) = self.items.last_mut() { + *delete += len; + return self; + } + + self.items.push(DeltaItem::new_delete(len)); + self + } + + pub fn replace(mut self, value: V, attr: Attr, delete: usize) -> Self { + if delete == 0 && value.rle_len() == 0 { + return self; + } + + if let Some(DeltaItem::Replace { + value: last_value, + attr: last_attr, + delete: last_delete, + }) = self.items.last_mut() + { + if last_value.can_merge(&value) && &attr == last_attr { + last_value.merge_right(&value); + *last_delete += delete; + return self; + } + } + + self.items.push(DeltaItem::Replace { + value, + attr, + delete, + }); + self + } + + pub fn build(self) -> DeltaRope { + let mut rope = DeltaRope::new(); + for item in self.items { + rope.tree.push(item); + } + + rope + } +} + +impl Default for DeltaRopeBuilder { + fn default() -> Self { + Self::new() + } +} + +impl Debug for DeltaRope { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_list().entries(self.iter()).finish() + } +} diff --git a/3rdparty/loro/crates/delta/src/delta_rope/compose.rs b/3rdparty/loro/crates/delta/src/delta_rope/compose.rs new file mode 100644 index 0000000000..7cf0967076 --- /dev/null +++ b/3rdparty/loro/crates/delta/src/delta_rope/compose.rs @@ -0,0 +1,221 @@ +use super::*; + +struct DeltaReplace<'a, V, Attr> { + value: &'a V, + attr: &'a Attr, + delete: usize, +} + +impl DeltaRope { + pub fn compose(&mut self, other: &Self) { + // TODO: Need to implement a slow mode that is guaranteed to be correct, then we can fuzz on it + if self.is_empty() { + *self = other.clone(); + return; + } + + // trace!("Composing {:#?}\n{:#?}", &self, &other); + let mut index = 0; + + let mut push_rest = false; + for item in other.iter() { + if index >= self.len() { + self.push_retain(index - self.len(), Attr::default()); + push_rest = true; + } + + if push_rest { + self.push(item.clone()); + continue; + } + + match item { + DeltaItem::Retain { len, attr } => { + if self.len() < index + len { + self.push_retain(index + len - self.len(), Default::default()); + } + if !attr.attr_is_empty() { + self.update_attr_in_range(index..index + len, attr); + } + index += len; + } + DeltaItem::Replace { + value: this_value, + attr: this_attr, + delete, + } => { + self._compose_replace( + DeltaReplace { + value: this_value, + attr: this_attr, + delete: *delete, + }, + &mut index, + ); + } + } + } + + // trace!("Composed {:#?}", &self); + } + + pub fn delete(&mut self, mut index: usize, len: usize) { + self._compose_replace( + DeltaReplace { + value: &Default::default(), + attr: &Default::default(), + delete: len, + }, + &mut index, + ); + } + + fn _compose_replace( + &mut self, + delta_replace_item @ DeltaReplace { + value: this_value, + attr: this_attr, + delete, + }: DeltaReplace, + index: &mut usize, + ) { + let mut should_insert = this_value.rle_len() > 0; + let mut left_del_len = delete; + if *index > self.len() { + self.push_retain(*index - self.len(), Attr::default()); + } + + if delete > 0 && *index != self.len() { + let range = *index..(*index + left_del_len).min(self.len()); + let from = self.tree.query::(&range.start).unwrap(); + let to = self.tree.query::(&range.end).unwrap(); + if from.cursor.leaf == to.cursor.leaf { + self._replace_on_single_leaf(from, to, left_del_len, delta_replace_item); + should_insert = false; + left_del_len = 0; + } else { + self._replace_batch_leaves(from, to, &mut left_del_len); + } + } + + if left_del_len > 0 || should_insert { + let replace = DeltaItem::Replace { + value: if should_insert { + this_value.clone() + } else { + Default::default() + }, + attr: if should_insert { + this_attr.clone() + } else { + Default::default() + }, + delete: left_del_len, + }; + self.insert_values(*index, [replace]); + } + + *index += this_value.rle_len(); + } + + fn _replace_batch_leaves( + &mut self, + from: generic_btree::QueryResult, + to: generic_btree::QueryResult, + left_del_len: &mut usize, + ) { + self.tree.update(from.cursor..to.cursor, &mut |item| { + // This method will split the leaf node before calling this closure. + // So it's guaranteed that the item is contained in the range. + if *left_del_len == 0 { + return None; + } + + match item { + DeltaItem::Retain { len, .. } => { + assert!(*len <= *left_del_len); + *left_del_len -= *len; + let diff = -(*len as isize); + *item = DeltaItem::Replace { + delete: *len, + value: Default::default(), + attr: Default::default(), + }; + Some(Len { + data_len: diff, + delta_len: 0, + }) + } + DeltaItem::Replace { value, attr, .. } => { + if *left_del_len >= value.rle_len() { + let diff = value.rle_len() as isize; + *left_del_len -= value.rle_len(); + *value = Default::default(); + *attr = Default::default(); + Some(Len { + data_len: -diff, + delta_len: -diff, + }) + } else { + unreachable!() + } + } + } + }); + } + + fn _replace_on_single_leaf( + &mut self, + from: generic_btree::QueryResult, + to: generic_btree::QueryResult, + left_del_len: usize, + DeltaReplace { + value: this_value, + attr: this_attr, + delete: _, + }: DeltaReplace, + ) { + self.tree.update_leaf(from.cursor.leaf, |item| match item { + DeltaItem::Retain { + len: retain_len, .. + } => { + let start = from.cursor.offset; + let end = to.cursor.offset; + debug_assert!(end <= *retain_len); + let (l, r) = item.update_with_split(start..end, |item| { + *item = DeltaItem::Replace { + delete: left_del_len, + value: this_value.clone(), + attr: this_attr.clone(), + }; + }); + + (true, l, r) + } + DeltaItem::Replace { value, delete, .. } => { + let start = from.cursor.offset; + let end = to.cursor.offset; + let value_len = value.rle_len(); + let value_start = start.min(value_len); + let value_end = value_len.min(end); + { + // We need to remove the part of value that is between start and end. + // If the range is out of the bounds of the value, we record extra deletions + // on the `delete` field of this item. + let left = left_del_len.saturating_sub(value_end - value_start); + *delete += left; + } + + let (l, r) = item.update_with_split(value_start..value_end, |item| { + *item = DeltaItem::Replace { + value: this_value.clone(), + attr: this_attr.clone(), + delete: 0, + }; + }); + + (true, l, r) + } + }); + } +} diff --git a/3rdparty/loro/crates/delta/src/delta_rope/rle_tree.rs b/3rdparty/loro/crates/delta/src/delta_rope/rle_tree.rs new file mode 100644 index 0000000000..5dcd6b9ceb --- /dev/null +++ b/3rdparty/loro/crates/delta/src/delta_rope/rle_tree.rs @@ -0,0 +1,199 @@ +use std::{ + fmt::Debug, + iter::Sum, + ops::{Add, AddAssign, Sub}, +}; + +use generic_btree::{rle::CanRemove, ArenaIndex, BTreeTrait, Child, FindResult, Query}; + +use crate::{ + delta_trait::{DeltaAttr, DeltaValue}, + DeltaItem, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub(crate) struct Len { + /// The length of insertions + retains + pub data_len: isize, + /// The length of deletions + retains + insertions + pub delta_len: isize, +} + +impl CanRemove for Len { + fn can_remove(&self) -> bool { + self.data_len == 0 && self.delta_len == 0 + } +} + +impl Add for Len { + type Output = Self; + + fn add(self, rhs: Self) -> Self { + Self { + data_len: self.data_len + rhs.data_len, + delta_len: self.delta_len + rhs.delta_len, + } + } +} + +impl AddAssign for Len { + fn add_assign(&mut self, rhs: Self) { + self.data_len += rhs.data_len; + self.delta_len += rhs.delta_len; + } +} + +impl Sub for Len { + type Output = Self; + + fn sub(self, rhs: Self) -> Self { + Self { + data_len: self.data_len - rhs.data_len, + delta_len: self.delta_len - rhs.delta_len, + } + } +} + +impl Sum for Len { + fn sum>(iter: I) -> Self { + iter.fold( + Self { + data_len: 0, + delta_len: 0, + }, + |acc, x| acc + x, + ) + } +} + +pub(crate) struct DeltaTreeTrait { + _phantom: std::marker::PhantomData<(V, Attr)>, +} + +impl BTreeTrait for DeltaTreeTrait { + type Elem = DeltaItem; + + type Cache = Len; + + type CacheDiff = Len; + + fn calc_cache_internal( + cache: &mut Self::Cache, + caches: &[generic_btree::Child], + ) -> Self::CacheDiff { + let new = caches.iter().map(|c| c.cache).sum(); + let diff = new - *cache; + *cache = new; + diff + } + + fn apply_cache_diff(cache: &mut Self::Cache, diff: &Self::CacheDiff) { + *cache += *diff; + } + + fn merge_cache_diff(diff1: &mut Self::CacheDiff, diff2: &Self::CacheDiff) { + *diff1 += *diff2; + } + + fn get_elem_cache(elem: &Self::Elem) -> Self::Cache { + match elem { + DeltaItem::Retain { len, attr: _ } => Len { + data_len: *len as isize, + delta_len: *len as isize, + }, + DeltaItem::Replace { + value, + attr: _, + delete, + } => Len { + data_len: value.rle_len() as isize, + delta_len: *delete as isize + value.rle_len() as isize, + }, + } + } + + fn new_cache_to_diff(cache: &Self::Cache) -> Self::CacheDiff { + *cache + } + + fn sub_cache(cache_lhs: &Self::Cache, cache_rhs: &Self::Cache) -> Self::CacheDiff { + *cache_lhs - *cache_rhs + } +} + +/// A generic length finder +pub struct LengthFinder { + pub left: usize, + pub slot: u8, + pub parent: Option, +} + +impl LengthFinder { + #[inline(always)] + pub fn new() -> Self { + Self { + left: 0, + slot: 0, + parent: None, + } + } +} + +impl Default for LengthFinder { + #[inline(always)] + fn default() -> Self { + Self::new() + } +} + +impl Query> + for LengthFinder +{ + type QueryArg = usize; + + #[inline(always)] + fn init(target: &Self::QueryArg) -> Self { + Self { + left: *target, + slot: 0, + parent: None, + } + } + + #[inline(always)] + fn find_node( + &mut self, + _: &Self::QueryArg, + child_caches: &[Child>], + ) -> FindResult { + let mut last_left = self.left; + let is_internal = matches!(child_caches.first().unwrap().arena, ArenaIndex::Internal(_)); + for (i, cache) in child_caches.iter().enumerate() { + let len = cache.cache.data_len as usize; + if self.left >= len { + last_left = self.left; + self.left -= len; + } else { + if is_internal { + self.parent = Some(cache.arena); + } else { + self.slot = i as u8; + } + return FindResult::new_found(i, self.left); + } + } + + self.left = last_left; + if is_internal { + self.parent = Some(child_caches.last().unwrap().arena); + } else { + self.slot = child_caches.len() as u8 - 1; + } + FindResult::new_missing(child_caches.len() - 1, last_left) + } + + #[inline(always)] + fn confirm_elem(&mut self, _: &Self::QueryArg, elem: &DeltaItem) -> (usize, bool) { + (self.left, self.left < elem.data_len()) + } +} diff --git a/3rdparty/loro/crates/delta/src/delta_trait.rs b/3rdparty/loro/crates/delta/src/delta_trait.rs new file mode 100644 index 0000000000..cb0bca35fe --- /dev/null +++ b/3rdparty/loro/crates/delta/src/delta_trait.rs @@ -0,0 +1,41 @@ +use generic_btree::rle::{HasLength, Mergeable, Sliceable, TryInsert}; +use std::hash::{BuildHasher, Hash}; +use std::{collections::HashMap, fmt::Debug}; + +pub trait DeltaValue: + HasLength + Sliceable + Mergeable + TryInsert + Debug + Clone + Default +{ +} + +pub trait DeltaAttr: Clone + PartialEq + Debug + Default { + fn compose(&mut self, other: &Self); + fn attr_is_empty(&self) -> bool; +} + +mod implementations { + use super::*; + + impl DeltaAttr for () { + fn compose(&mut self, _other: &Self) {} + fn attr_is_empty(&self) -> bool { + true + } + } + + impl DeltaAttr for HashMap + where + K: Eq + Hash + Debug + Clone, + V: Debug + PartialEq + Clone, + S: BuildHasher + Default + Clone, + { + fn compose(&mut self, other: &Self) { + for (key, value) in other { + self.insert(key.clone(), value.clone()); + } + } + + fn attr_is_empty(&self) -> bool { + self.is_empty() + } + } +} diff --git a/3rdparty/loro/crates/delta/src/iter.rs b/3rdparty/loro/crates/delta/src/iter.rs new file mode 100644 index 0000000000..faba92ba4f --- /dev/null +++ b/3rdparty/loro/crates/delta/src/iter.rs @@ -0,0 +1,161 @@ +use generic_btree::{Cursor, LeafIndex}; + +use crate::{ + delta_trait::{DeltaAttr, DeltaValue}, + DeltaItem, DeltaRope, +}; + +pub struct Iter<'a, V: DeltaValue, Attr: DeltaAttr> { + delta: &'a DeltaRope, + cursor: Option, + current: Option>, +} + +impl<'a, V: DeltaValue, Attr: DeltaAttr> Iter<'a, V, Attr> { + pub fn new(delta: &'a DeltaRope) -> Self { + let leaf = delta.tree.first_leaf(); + let mut current = None; + if let Some(leaf) = leaf { + current = delta.tree.get_elem(leaf).cloned(); + } + + Self { + delta, + cursor: leaf, + current, + } + } + + pub fn peek(&self) -> Option<&'_ DeltaItem> { + self.current.as_ref() + } + + pub fn peek_is_replace(&self) -> bool { + self.peek().map(|x| x.is_replace()).unwrap_or(false) + } + + pub fn peek_is_insert(&self) -> bool { + self.peek().map(|x| x.is_insert()).unwrap_or(false) + } + + pub fn peek_is_delete(&self) -> bool { + self.peek().map(|x| x.is_delete()).unwrap_or(false) + } + + pub fn peek_is_retain(&self) -> bool { + self.peek().map(|x| x.is_retain()).unwrap_or(false) + } + + pub fn peek_length(&self) -> usize { + self.peek().map(|x| x.delta_len()).unwrap_or(usize::MAX) + } + + pub fn peek_insert_length(&self) -> usize { + self.peek() + .map(|x| match x { + DeltaItem::Retain { .. } => 0, + DeltaItem::Replace { value, .. } => value.rle_len(), + }) + .unwrap_or(0) + } + + pub fn next_with(&mut self, mut len: usize) -> Result<(), usize> { + while len > 0 { + let Some(current) = self.current.as_mut() else { + return Err(len); + }; + + if len >= current.delta_len() { + len -= current.delta_len(); + self.cursor = self + .delta + .tree + .next_elem(Cursor { + leaf: self.cursor.unwrap(), + offset: 0, + }) + .map(|x| x.leaf); + if let Some(leaf) = self.cursor { + self.current = self.delta.tree.get_elem(leaf).cloned(); + } else { + self.current = None; + } + } else { + match current { + DeltaItem::Retain { + len: retain, + attr: _, + } => { + *retain -= len; + } + DeltaItem::Replace { + value, + attr: _, + delete, + } => { + if value.rle_len() > 0 { + value.slice_(len..); + } else { + *delete -= len; + } + } + } + len = 0; + } + } + + Ok(()) + } + + /// Consume next `len` deletions in the current item + pub(crate) fn next_with_del(&mut self, mut len: usize) -> Result<(), usize> { + let Some(current) = self.current.as_mut() else { + return Err(len); + }; + + match current { + DeltaItem::Retain { .. } => return Err(len), + DeltaItem::Replace { delete, .. } => { + if *delete >= len { + *delete -= len; + len = 0; + } else { + len -= *delete; + *delete = 0; + } + } + } + + if current.delta_len() == 0 { + self.next(); + } + + if len > 0 { + Err(len) + } else { + Ok(()) + } + } +} + +impl Iterator for Iter<'_, V, Attr> { + type Item = DeltaItem; + + fn next(&mut self) -> Option { + self.cursor = self + .delta + .tree + .next_elem(Cursor { + leaf: self.cursor.unwrap(), + offset: 0, + }) + .map(|x| x.leaf); + let old_current = std::mem::take(&mut self.current); + if let Some(c) = self.cursor { + self.current = self.delta.tree.get_elem(c).cloned(); + } else { + self.current = None; + } + old_current + } +} diff --git a/3rdparty/loro/crates/delta/src/lib.rs b/3rdparty/loro/crates/delta/src/lib.rs new file mode 100644 index 0000000000..eaec2c3ac5 --- /dev/null +++ b/3rdparty/loro/crates/delta/src/lib.rs @@ -0,0 +1,78 @@ +use delta_rope::rle_tree::DeltaTreeTrait; +use delta_trait::{DeltaAttr, DeltaValue}; +use enum_as_inner::EnumAsInner; +use generic_btree::{ + rle::{HasLength, Mergeable, Sliceable}, + BTree, +}; +use std::fmt::Debug; + +pub mod array_vec; +mod delta_item; +mod delta_rope; +pub mod delta_trait; +pub mod iter; +pub mod text_delta; +pub mod utf16; + +/// A [DeltaRope] is a rope-like data structure that can be used to represent +/// a sequence of [DeltaItem]. It has efficient operations for composing other +/// [DeltaRope]s. It can also be used as a rope, where it only contains insertions. +#[derive(Clone)] +pub struct DeltaRope { + tree: BTree>, +} + +pub struct DeltaRopeBuilder { + items: Vec>, +} + +#[derive(Debug, Clone, PartialEq, Eq, EnumAsInner)] +pub enum DeltaItem { + Retain { + len: usize, + attr: Attr, + }, + /// This is the combined of a delete and an insert. + /// + /// They are two separate operations in the original Quill Delta format. + /// But the order of two neighboring delete and insert operations can be + /// swapped without changing the result. So Quill requires that the insert + /// always comes before the delete. So it creates room for invalid deltas + /// by the type system. Using Replace is a way to avoid this. + Replace { + value: V, + attr: Attr, + delete: usize, + }, +} + +impl DeltaItem { + fn is_insert(&self) -> bool { + match self { + DeltaItem::Retain { .. } => false, + DeltaItem::Replace { value, .. } => value.rle_len() > 0, + } + } + + fn is_delete(&self) -> bool { + match self { + DeltaItem::Retain { .. } => false, + DeltaItem::Replace { value, delete, .. } => value.rle_len() == 0 && *delete > 0, + } + } + + fn is_replace(&self) -> bool { + match self { + DeltaItem::Retain { .. } => false, + DeltaItem::Replace { .. } => true, + } + } + + fn is_retain(&self) -> bool { + match self { + DeltaItem::Retain { .. } => true, + DeltaItem::Replace { .. } => false, + } + } +} diff --git a/3rdparty/loro/crates/delta/src/text_delta.rs b/3rdparty/loro/crates/delta/src/text_delta.rs new file mode 100644 index 0000000000..48d0b62fe4 --- /dev/null +++ b/3rdparty/loro/crates/delta/src/text_delta.rs @@ -0,0 +1,191 @@ +use crate::{ + delta_trait::{DeltaAttr, DeltaValue}, + DeltaItem, DeltaRope, +}; +use arrayvec::ArrayString; +use generic_btree::rle::{HasLength, Mergeable, Sliceable, TryInsert}; + +#[cfg(test)] +const MAX_STRING_SIZE: usize = 8; +#[cfg(not(test))] +const MAX_STRING_SIZE: usize = 128; + +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct TextChunk(ArrayString); +pub type TextDelta = DeltaRope; + +impl TextDelta { + pub fn insert_str(&mut self, index: usize, s: &str) { + if s.is_empty() || index == self.len() { + self.push_str_insert(s); + return; + } + + self.insert_values( + index, + TextChunk::from_long_str(s).map(|chunk| DeltaItem::Replace { + value: chunk, + attr: Default::default(), + delete: 0, + }), + ); + } + + pub fn push_str_insert(&mut self, s: &str) -> &mut Self { + self.push_str_insert_with_attr(s, Default::default()) + } + + pub fn push_str_insert_with_attr(&mut self, s: &str, attr: Attr) -> &mut Self { + if s.is_empty() { + return self; + } + + if s.len() <= MAX_STRING_SIZE { + self.push_insert(TextChunk(ArrayString::from(s).unwrap()), attr); + return self; + } + + let mut split_end = 128; + let mut split_start = 0; + while split_end != s.len() { + while !s.is_char_boundary(split_end) { + split_end -= 1; + } + + let chunk = TextChunk(ArrayString::from(&s[split_start..split_end]).unwrap()); + self.push_insert(chunk, attr.clone()); + split_start = split_end; + split_end = (split_end + 128).min(s.len()); + } + + self + } + + pub fn try_to_string(&self) -> Option { + let mut ans = String::with_capacity(self.len()); + for item in self.iter() { + match item { + crate::DeltaItem::Retain { .. } => return None, + crate::DeltaItem::Replace { value, .. } => { + ans.push_str(&value.0); + } + } + } + + Some(ans) + } +} + +impl TextChunk { + pub(crate) fn try_insert(&mut self, pos: usize, s: &str) -> Result<(), ()> { + if self.0.len() + s.len() > MAX_STRING_SIZE { + return Err(()); + } + + assert!(self.0.is_char_boundary(pos)); + let new_len = self.0.len() + s.len(); + unsafe { + let ptr = self.0.as_mut_ptr().add(pos); + ptr.copy_to(ptr.add(s.len()), self.0.len() - pos); + ptr.copy_from_nonoverlapping(s.as_ptr(), s.len()); + self.0.set_len(new_len); + } + + Ok(()) + } + + pub fn try_from_str(s: &str) -> Option { + Some(TextChunk(ArrayString::from(s).ok()?)) + } + + pub fn from_long_str(s: &str) -> impl Iterator + '_ { + let mut text_iter = s.chars(); + std::iter::from_fn(move || { + let mut chunk = Self::default(); + for c in text_iter.by_ref() { + let mut bytes = [0, 0, 0, 0]; + chunk.0.push_str(c.encode_utf8(&mut bytes)); + if chunk.0.is_full() { + break; + } + } + + if chunk.0.is_empty() { + return None; + } + + Some(chunk) + }) + } +} + +impl HasLength for TextChunk { + fn rle_len(&self) -> usize { + self.0.len() + } +} + +impl Mergeable for TextChunk { + fn can_merge(&self, rhs: &Self) -> bool { + MAX_STRING_SIZE >= self.0.len() + rhs.0.len() + } + + fn merge_right(&mut self, rhs: &Self) { + self.0.push_str(&rhs.0) + } + + fn merge_left(&mut self, left: &Self) { + assert!(self.can_merge(left), "TextChunk cannot merge over capacity"); + let ptr = self.0.as_mut_ptr(); + // Safety: `self.0` is a valid `ArrayString` and `left.0` is a valid `ArrayString`. + unsafe { + ptr.copy_to(ptr.add(left.0.len()), self.0.len()); + ptr.copy_from_nonoverlapping(left.0.as_ptr(), left.0.len()); + self.0.set_len(self.0.len() + left.0.len()); + } + } +} + +impl Sliceable for TextChunk { + fn _slice(&self, range: std::ops::Range) -> Self { + let mut new = ArrayString::new(); + new.push_str(&self.0.as_str()[range]); + TextChunk(new) + } + + fn split(&mut self, pos: usize) -> Self { + let mut right = ArrayString::new(); + right.push_str(&self.0.as_str()[pos..]); + self.0.truncate(pos); + TextChunk(right) + } +} + +impl TryInsert for TextChunk { + fn try_insert(&mut self, pos: usize, elem: Self) -> Result<(), Self> + where + Self: Sized, + { + match self.try_insert(pos, elem.0.as_str()) { + Ok(_) => Ok(()), + Err(_) => Err(elem), + } + } +} + +impl DeltaValue for TextChunk {} + +#[cfg(all(test, miri))] +mod miri_tests { + use super::TextChunk; + use generic_btree::rle::Mergeable; + + #[test] + #[should_panic(expected = "TextChunk cannot merge over capacity")] + fn merge_left_rejects_over_capacity_before_raw_copy() { + let mut right = TextChunk::try_from_str("12345678").unwrap(); + let left = TextChunk::try_from_str("9").unwrap(); + + right.merge_left(&left); + } +} diff --git a/3rdparty/loro/crates/delta/src/utf16.rs b/3rdparty/loro/crates/delta/src/utf16.rs new file mode 100644 index 0000000000..72b3218f62 --- /dev/null +++ b/3rdparty/loro/crates/delta/src/utf16.rs @@ -0,0 +1,133 @@ +use generic_btree::rle::{HasLength, Mergeable, Sliceable, TryInsert}; + +use crate::delta_trait::DeltaValue; + +#[derive(Debug, Clone)] +pub struct AsUtf16Index> { + s: S, + utf16_len: usize, +} + +impl> AsUtf16Index { + pub fn new(s: S) -> Self { + let utf16_len = s + .as_ref() + .chars() + .fold(0, |prev, cur| prev + cur.len_utf16()); + Self { s, utf16_len } + } + + fn convert_utf16_to_utf8(&self, index: usize) -> usize { + if index == 0 { + return 0; + } + + let s = self.s.as_ref(); + if index == self.utf16_len { + return s.len(); + } + + if index > self.utf16_len { + panic!("Index out of bounds"); + } + + let mut utf16_index = 0; + for (i, c) in s.char_indices() { + if utf16_index >= index { + return i; + } + utf16_index += c.len_utf16(); + } + + unreachable!(); + } + + #[allow(unused)] + fn convert_utf8_to_utf16(&self, index: usize) -> usize { + if index == 0 { + return 0; + } + + let s = self.s.as_ref(); + if index == s.len() { + return self.utf16_len; + } + + if index > s.len() { + panic!("Index out of bounds"); + } + + let mut utf16_index = 0; + for (i, c) in s.char_indices() { + if i >= index { + return utf16_index; + } + utf16_index += c.len_utf16(); + } + + unreachable!(); + } +} + +impl> HasLength for AsUtf16Index { + fn rle_len(&self) -> usize { + self.utf16_len + } +} + +impl + Mergeable> Mergeable for AsUtf16Index { + fn can_merge(&self, rhs: &Self) -> bool { + self.s.can_merge(&rhs.s) + } + + fn merge_right(&mut self, rhs: &Self) { + self.s.merge_right(&rhs.s); + self.utf16_len += rhs.utf16_len; + } + + fn merge_left(&mut self, left: &Self) { + self.s.merge_left(&left.s); + self.utf16_len += left.utf16_len; + } +} + +impl + Sliceable> Sliceable for AsUtf16Index { + fn _slice(&self, range: std::ops::Range) -> Self { + let start = self.convert_utf16_to_utf8(range.start); + let end = self.convert_utf16_to_utf8(range.end); + Self { + s: self.s._slice(start..end), + utf16_len: range.len(), + } + } +} + +impl + Sliceable + TryInsert> TryInsert for AsUtf16Index { + fn try_insert(&mut self, pos: usize, elem: Self) -> Result<(), Self> + where + Self: Sized, + { + let start = self.convert_utf16_to_utf8(pos); + match self.s.try_insert(start, elem.s) { + Ok(()) => { + self.utf16_len += elem.utf16_len; + Ok(()) + } + Err(e) => Err(Self { + s: e, + utf16_len: elem.utf16_len, + }), + } + } +} + +impl + DeltaValue> Default for AsUtf16Index { + fn default() -> Self { + Self { + s: S::default(), + utf16_len: 0, + } + } +} + +impl + DeltaValue> DeltaValue for AsUtf16Index {} diff --git a/3rdparty/loro/crates/fractional_index/Cargo.toml b/3rdparty/loro/crates/fractional_index/Cargo.toml new file mode 100644 index 0000000000..63ee685626 --- /dev/null +++ b/3rdparty/loro/crates/fractional_index/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "loro_fractional_index" +version = "1.13.0" +edition = "2021" +license = "MIT" +description = "Fraction index for Loro. Fork and modified from jamsocket/fractional_index" +repository = "https://github.com/loro-dev/loro/" +authors = ["Liang Zhao", "Zixuan Chen"] +categories = [] +keywords = ["crdt", "local-first"] + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +serde = { workspace = true, features = ["derive", "rc"], optional = true } +rand = { version = "^0.8" } +once_cell = { workspace = true } + +[dev-dependencies] +fraction_index = { version = "^2.0", package = "fractional_index" } +criterion = "^0.5.0" +serde_json = { workspace = true } + + +[[bench]] +name = "fractional_index" +harness = false diff --git a/3rdparty/loro/crates/fractional_index/LICENSE b/3rdparty/loro/crates/fractional_index/LICENSE new file mode 100644 index 0000000000..3aff770ead --- /dev/null +++ b/3rdparty/loro/crates/fractional_index/LICENSE @@ -0,0 +1,21 @@ +MIT License Copyright (c) 2024 Loro + +Permission is hereby granted, free of +charge, to any person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the Software without +restriction, including without limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to the +following conditions: + +The above copyright notice and this permission notice +(including the next paragraph) shall be included in all copies or substantial +portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO +EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/3rdparty/loro/crates/fractional_index/README.md b/3rdparty/loro/crates/fractional_index/README.md new file mode 100644 index 0000000000..d7f1490572 --- /dev/null +++ b/3rdparty/loro/crates/fractional_index/README.md @@ -0,0 +1,3 @@ +# Fractional Index + +This crate is forked from [drifting-in-space/fractional_index](https://github.com/drifting-in-space/fractional_index) diff --git a/3rdparty/loro/crates/fractional_index/src/jitter.rs b/3rdparty/loro/crates/fractional_index/src/jitter.rs new file mode 100644 index 0000000000..54ebd56020 --- /dev/null +++ b/3rdparty/loro/crates/fractional_index/src/jitter.rs @@ -0,0 +1,103 @@ +use std::sync::Arc; + +use crate::{new_after, new_before, new_between, FractionalIndex, TERMINATOR}; +use rand::Rng; + +impl FractionalIndex { + pub fn jitter_default(rng: &mut impl Rng, jitter: u8) -> Self { + Self::jitter(Vec::new(), rng, jitter) + } + + pub fn new_jitter( + lower: Option<&FractionalIndex>, + upper: Option<&FractionalIndex>, + rng: &mut R, + jitter: u8, + ) -> Option { + match (lower, upper) { + (Some(lower), Some(upper)) => Self::new_between_jitter(lower, upper, rng, jitter), + (Some(lower), None) => Self::new_after_jitter(lower, rng, jitter).into(), + (None, Some(upper)) => Self::new_before_jitter(upper, rng, jitter).into(), + (None, None) => FractionalIndex::jitter_default(rng, jitter).into(), + } + } + + fn jitter(mut bytes: Vec, rng: &mut R, jitter: u8) -> FractionalIndex { + bytes.push(TERMINATOR); + bytes.extend((0..jitter).map(|_| rng.gen::())); + FractionalIndex(Arc::new(bytes)) + } + + pub fn new_before_jitter( + FractionalIndex(bytes): &FractionalIndex, + rng: &mut R, + jitter: u8, + ) -> Self { + Self::jitter(new_before(bytes), rng, jitter) + } + + pub fn new_after_jitter( + FractionalIndex(bytes): &FractionalIndex, + rng: &mut R, + jitter: u8, + ) -> Self { + Self::jitter(new_after(bytes), rng, jitter) + } + + pub fn new_between_jitter( + FractionalIndex(left): &FractionalIndex, + FractionalIndex(right): &FractionalIndex, + rng: &mut R, + jitter: u8, + ) -> Option { + new_between(left, right, jitter as usize + 1).map(|x| Self::jitter(x, rng, jitter)) + } + + pub fn generate_n_evenly_jitter( + lower: Option<&FractionalIndex>, + upper: Option<&FractionalIndex>, + n: usize, + rng: &mut R, + jitter: u8, + ) -> Option> { + fn generate( + lower: Option<&FractionalIndex>, + upper: Option<&FractionalIndex>, + n: usize, + push: &mut impl FnMut(FractionalIndex), + rng: &mut impl Rng, + jitter: u8, + ) { + if n == 0 { + return; + } + + let mid = n / 2; + let mid_ans = FractionalIndex::new_jitter(lower, upper, rng, jitter).unwrap(); + if n == 1 { + push(mid_ans); + return; + } + + generate(lower, Some(&mid_ans), mid, push, rng, jitter); + push(mid_ans.clone()); + if n - mid - 1 == 0 { + return; + } + generate(Some(&mid_ans), upper, n - mid - 1, push, rng, jitter); + } + + if n == 0 { + return Some(Vec::new()); + } + + match (lower, upper) { + (Some(a), Some(b)) if a >= b => return None, + _ => {} + } + + let mut ans = Vec::with_capacity(n); + generate(lower, upper, n, &mut |v| ans.push(v), rng, jitter); + Some(ans) + } +} diff --git a/3rdparty/loro/crates/fractional_index/src/lib.rs b/3rdparty/loro/crates/fractional_index/src/lib.rs new file mode 100644 index 0000000000..7673450874 --- /dev/null +++ b/3rdparty/loro/crates/fractional_index/src/lib.rs @@ -0,0 +1,225 @@ +use std::{ + fmt::{Display, Write}, + sync::Arc, +}; + +#[cfg(feature = "serde")] +use serde::{Deserialize, Serialize}; + +mod jitter; + +const TERMINATOR: u8 = 128; +static DEFAULT_FRACTIONAL_INDEX: once_cell::sync::Lazy = + once_cell::sync::Lazy::new(|| FractionalIndex(Arc::new(vec![TERMINATOR]))); + +#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +pub struct FractionalIndex(Arc>); + +impl Default for FractionalIndex { + fn default() -> Self { + DEFAULT_FRACTIONAL_INDEX.clone() + } +} + +impl FractionalIndex { + fn from_vec_unterminated(mut bytes: Vec) -> Self { + bytes.push(TERMINATOR); + FractionalIndex(Arc::new(bytes)) + } + + pub fn from_bytes(bytes: Vec) -> Self { + FractionalIndex(Arc::new(bytes)) + } + + pub fn from_hex_string>(str: T) -> Self { + let s = str.as_ref(); + let mut bytes = Vec::with_capacity(s.len() / 2); + for i in 0..s.len() / 2 { + let byte = u8::from_str_radix(&s[i * 2..i * 2 + 2], 16).unwrap(); + bytes.push(byte); + } + FractionalIndex::from_bytes(bytes) + } + + pub fn as_bytes(&self) -> &[u8] { + &self.0 + } +} + +pub(crate) fn new_before(bytes: &[u8]) -> Vec { + for i in 0..bytes.len() { + if bytes[i] > TERMINATOR { + return bytes[..i].into(); + } + if bytes[i] > u8::MIN { + let mut ans: Vec = bytes[..=i].into(); + ans[i] -= 1; + return ans; + } + } + unreachable!() +} + +pub(crate) fn new_after(bytes: &[u8]) -> Vec { + for i in 0..bytes.len() { + if bytes[i] < TERMINATOR { + return bytes[0..i].into(); + } + if bytes[i] < u8::MAX { + let mut ans: Vec = bytes[0..=i].into(); + ans[i] += 1; + return ans; + } + } + unreachable!() +} + +pub(crate) fn new_between(left: &[u8], right: &[u8], extra_capacity: usize) -> Option> { + let shorter_len = left.len().min(right.len()) - 1; + for i in 0..shorter_len { + let left_byte = left[i]; + let right_byte = right[i]; + let diff = i16::from(right_byte) - i16::from(left_byte); + if diff > 1 { + let mut ans: Vec = left[0..=i].into(); + ans[i] += (right_byte - left_byte) / 2; + return ans.into(); + } + if diff == 1 { + let (prefix, suffix) = left.split_at(i + 1); + let new_suffix = new_after(suffix); + let mut ans = Vec::with_capacity(prefix.len() + new_suffix.len() + extra_capacity); + ans.extend_from_slice(prefix); + ans.extend_from_slice(&new_suffix); + return ans.into(); + } + if diff < 0 { + return None; + } + } + + match left.len().cmp(&right.len()) { + std::cmp::Ordering::Less => { + let (prefix, suffix) = right.split_at(shorter_len + 1); + if prefix.last().unwrap() < &TERMINATOR { + return None; + } + let new_suffix = new_before(suffix); + let mut ans = Vec::with_capacity(new_suffix.len() + prefix.len() + extra_capacity); + ans.extend_from_slice(prefix); + ans.extend_from_slice(&new_suffix); + ans.into() + } + std::cmp::Ordering::Equal => None, + std::cmp::Ordering::Greater => { + let (prefix, suffix) = left.split_at(shorter_len + 1); + if prefix.last().unwrap() >= &TERMINATOR { + return None; + } + let new_suffix = new_after(suffix); + let mut ans = Vec::with_capacity(new_suffix.len() + prefix.len() + extra_capacity); + ans.extend_from_slice(prefix); + ans.extend_from_slice(&new_suffix); + ans.into() + } + } +} +impl FractionalIndex { + pub fn new(lower: Option<&FractionalIndex>, upper: Option<&FractionalIndex>) -> Option { + match (lower, upper) { + (Some(lower), Some(upper)) => Self::new_between(lower, upper), + (Some(lower), None) => Self::new_after(lower).into(), + (None, Some(upper)) => Self::new_before(upper).into(), + (None, None) => FractionalIndex::default().into(), + } + } + + pub fn new_before(FractionalIndex(bytes): &FractionalIndex) -> Self { + FractionalIndex::from_vec_unterminated(new_before(bytes)) + } + + pub fn new_after(FractionalIndex(bytes): &FractionalIndex) -> Self { + FractionalIndex::from_vec_unterminated(new_after(bytes)) + } + + pub fn new_between( + FractionalIndex(left): &FractionalIndex, + FractionalIndex(right): &FractionalIndex, + ) -> Option { + new_between(left, right, 1).map(FractionalIndex::from_vec_unterminated) + } + + pub fn generate_n_evenly( + lower: Option<&FractionalIndex>, + upper: Option<&FractionalIndex>, + n: usize, + ) -> Option> { + fn generate( + lower: Option<&FractionalIndex>, + upper: Option<&FractionalIndex>, + n: usize, + push: &mut impl FnMut(FractionalIndex), + ) { + if n == 0 { + return; + } + + let mid = n / 2; + let mid_ans = FractionalIndex::new(lower, upper).unwrap(); + if n == 1 { + push(mid_ans); + return; + } + + generate(lower, Some(&mid_ans), mid, push); + push(mid_ans.clone()); + if n - mid - 1 == 0 { + return; + } + generate(Some(&mid_ans), upper, n - mid - 1, push); + } + + if n == 0 { + return Some(Vec::new()); + } + + match (lower, upper) { + (Some(a), Some(b)) if a >= b => return None, + _ => {} + } + + let mut ans = Vec::with_capacity(n); + generate(lower, upper, n, &mut |v| ans.push(v)); + Some(ans) + } +} + +impl Display for FractionalIndex { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", bytes_to_hex(&self.0)) + } +} + +pub fn bytes_to_hex(bytes: &[u8]) -> String { + bytes.iter().fold(String::new(), |mut output, b| { + let _ = write!(output, "{b:02X}"); + output + }) +} + +#[cfg(test)] +mod tests { + use super::FractionalIndex; + + #[test] + fn new_between_handles_zero_shared_prefix() { + let lower = FractionalIndex::from_hex_string("0080"); + let upper = FractionalIndex::from_hex_string("008180"); + let index = FractionalIndex::new_between(&lower, &upper).unwrap(); + + assert!(lower < index); + assert!(index < upper); + assert_eq!(index.to_string(), "00817F80"); + } +} diff --git a/3rdparty/loro/crates/generic-btree/Cargo.toml b/3rdparty/loro/crates/generic-btree/Cargo.toml new file mode 100644 index 0000000000..e28744479d --- /dev/null +++ b/3rdparty/loro/crates/generic-btree/Cargo.toml @@ -0,0 +1,35 @@ +[package] +name = "generic-btree" +version = "0.10.7" +edition = "2021" +authors = ["zxch3n "] +description = "Generic BTree for versatile purposes" +homepage = "https://github.com/loro-dev/generic-btree" +documentation = "https://docs.rs/generic-btree" +readme = "README.md" +keywords = ["btree", "data-structure"] +license = "MIT" +repository = "https://github.com/loro-dev/generic-btree" + +# Vendored into the loro workspace (fork of crates.io generic-btree 0.10.7) so we +# can evolve the b-tree (e.g. deferred cache propagation). Redirected from +# crates.io via [patch.crates.io] in the root Cargo.toml. + +[features] +test = [] + +[lib] +name = "generic_btree" +path = "src/lib.rs" + +[dependencies] +arref = "0.1.0" +heapless = "0.9.1" +itertools = "0.11.0" +proc-macro2 = "1.0.67" +rustc-hash = "2.1.1" +thunderdome = { version = "0.6.2", package = "loro-thunderdome" } + +[dev-dependencies] +arbitrary = { version = "1", features = ["derive"] } +rand = "0.8.5" diff --git a/3rdparty/loro/crates/generic-btree/README.md b/3rdparty/loro/crates/generic-btree/README.md new file mode 100644 index 0000000000..593b12eaee --- /dev/null +++ b/3rdparty/loro/crates/generic-btree/README.md @@ -0,0 +1,12 @@ +# Generic B-Tree + +It’s a pure safe BTree that can be used to build your own special-purpose btree +data structure. It’s mainly developed to optimize the performance of Loro CRDT’s +components. + +It can be used to build: + +- Rope +- Run length encoding data structure +- RangeMap that uses range as its key +- BTreeSet & BTreeMap diff --git a/3rdparty/loro/crates/generic-btree/src/generic_impl/gap_buffer.rs b/3rdparty/loro/crates/generic-btree/src/generic_impl/gap_buffer.rs new file mode 100644 index 0000000000..b9ce6f040a --- /dev/null +++ b/3rdparty/loro/crates/generic-btree/src/generic_impl/gap_buffer.rs @@ -0,0 +1,293 @@ +use std::ops::{Range, RangeBounds}; + +use crate::rle::{CanRemove, HasLength, Mergeable, Sliceable, TryInsert}; + +#[cfg(not(test))] +pub const MAX_STRING_SIZE: usize = 128; +#[cfg(test)] +pub const MAX_STRING_SIZE: usize = 12; + +#[derive(Debug, Clone)] +pub(super) struct GapBuffer { + buffer: [u8; MAX_STRING_SIZE], + gap_start: u16, + gap_len: u16, +} + +impl GapBuffer { + pub fn new() -> Self { + Self { + buffer: [0; MAX_STRING_SIZE], + gap_start: 0, + gap_len: MAX_STRING_SIZE as u16, + } + } + + pub fn shift_at(&mut self, index: usize) { + if index > self.len() { + panic!("index {} out of range len={}", index, self.len()); + } + + let gap_start = self.gap_start as usize; + let gap_end = (self.gap_start + self.gap_len) as usize; + match index.cmp(&gap_start) { + std::cmp::Ordering::Equal => {} + std::cmp::Ordering::Less => { + let gap_move = gap_start - index; + self.buffer + .copy_within(index..gap_start, gap_end - gap_move); + self.gap_start -= gap_move as u16; + } + std::cmp::Ordering::Greater => { + let gap_move = index - gap_start; + let move_end = self.buffer.len().min(gap_end + gap_move); + self.buffer.copy_within(gap_end..move_end, gap_start); + self.gap_start += gap_move as u16; + } + } + } + + #[allow(unused)] + pub fn push(&mut self, value: u8) -> Result<(), ()> { + if self.gap_len == 0 { + return Err(()); + } + self.buffer[self.gap_start as usize] = value; + self.gap_start += 1; + self.gap_len -= 1; + Ok(()) + } + + #[inline(always)] + pub fn push_bytes(&mut self, bytes: &[u8]) -> Result<(), ()> { + self.insert_bytes(self.len(), bytes) + } + + pub fn insert_bytes(&mut self, index: usize, bytes: &[u8]) -> Result<(), ()> { + if (self.gap_len as usize) < bytes.len() { + return Err(()); + } + + self.shift_at(index); + self.buffer[index..index + bytes.len()].copy_from_slice(bytes); + self.gap_start += bytes.len() as u16; + self.gap_len -= bytes.len() as u16; + Ok(()) + } + + pub fn insert_bytes_pair( + &mut self, + index: usize, + (left, right): (&[u8], &[u8]), + ) -> Result<(), ()> { + let len = left.len() + right.len(); + if (self.gap_len as usize) < len { + return Err(()); + } + + self.shift_at(index); + self.buffer[index..index + left.len()].copy_from_slice(left); + self.buffer[index + left.len()..index + len].copy_from_slice(right); + self.gap_start += len as u16; + self.gap_len -= len as u16; + Ok(()) + } + + pub fn delete(&mut self, range: impl RangeBounds) { + let mut start = match range.start_bound() { + std::ops::Bound::Included(x) => *x, + std::ops::Bound::Excluded(x) => x + 1, + std::ops::Bound::Unbounded => 0, + }; + let mut end = match range.end_bound() { + std::ops::Bound::Included(x) => x + 1, + std::ops::Bound::Excluded(x) => *x, + std::ops::Bound::Unbounded => self.len(), + }; + + end = end.min(self.len()); + start = start.min(self.len()).min(end); + if start == end { + return; + } + + let len = end - start; + self.shift_at(end); + self.gap_start = start as u16; + self.gap_len += len as u16; + } + + #[inline] + pub fn capacity(&self) -> usize { + self.buffer.len() + } + + #[inline] + pub fn len(&self) -> usize { + self.buffer.len() - self.gap_len as usize + } + + pub fn as_bytes(&self) -> (&[u8], &[u8]) { + ( + &self.buffer[..self.gap_start as usize], + &self.buffer[(self.gap_start + self.gap_len) as usize..], + ) + } + + #[allow(unused)] + pub fn to_vec(&self) -> Vec { + let mut vec = Vec::with_capacity(self.len()); + let (left, right) = self.as_bytes(); + vec.extend_from_slice(left); + vec.extend_from_slice(right); + vec + } + + pub(crate) fn from_str(elem: &str) -> impl Iterator + '_ { + let mut i = 0; + let elem = elem.as_bytes(); + std::iter::from_fn(move || { + if i >= elem.len() { + return None; + } + + let mut gb = GapBuffer::new(); + gb.push_bytes(&elem[i..(i + MAX_STRING_SIZE).min(elem.len())]) + .unwrap(); + i += MAX_STRING_SIZE; + Some(gb) + }) + } +} + +impl HasLength for GapBuffer { + fn rle_len(&self) -> usize { + self.len() + } +} + +impl Sliceable for GapBuffer { + fn _slice(&self, range: Range) -> Self { + let mut gb = Self::new(); + let start = range.start; + let end = range.end; + + let (l, r) = self.as_bytes(); + if start < l.len() { + gb.push_bytes(&l[start..end.min(l.len())]).unwrap(); + } + if end > l.len() { + gb.push_bytes(&r[start.saturating_sub(l.len())..end.saturating_sub(l.len())]) + .unwrap(); + } + + debug_assert_eq!(gb.len(), end - start); + gb + } + + fn slice_(&mut self, range: impl RangeBounds) + where + Self: Sized, + { + let start = match range.start_bound() { + std::ops::Bound::Included(x) => *x, + std::ops::Bound::Excluded(x) => x + 1, + std::ops::Bound::Unbounded => 0, + }; + let end = match range.end_bound() { + std::ops::Bound::Included(x) => x + 1, + std::ops::Bound::Excluded(x) => *x, + std::ops::Bound::Unbounded => self.len(), + }; + + self.delete(end..); + self.delete(..start); + debug_assert_eq!(self.len(), end - start); + } + + fn split(&mut self, pos: usize) -> Self + where + Self: Sized, + { + self.shift_at(pos); + let right = self.as_bytes().1; + let mut r = Self::new(); + r.push_bytes(right).unwrap(); + self.gap_len = (self.capacity() - pos) as u16; + r + } +} + +impl Mergeable for GapBuffer { + fn can_merge(&self, rhs: &Self) -> bool { + self.len() + rhs.len() <= MAX_STRING_SIZE + } + + fn merge_right(&mut self, rhs: &Self) { + let pair = rhs.as_bytes(); + self.insert_bytes_pair(self.len(), pair).unwrap(); + } + + fn merge_left(&mut self, left: &Self) { + let pair = left.as_bytes(); + self.insert_bytes_pair(0, pair).unwrap(); + } +} + +impl TryInsert for GapBuffer { + fn try_insert(&mut self, pos: usize, elem: Self) -> Result<(), Self> + where + Self: Sized, + { + if self.len() + elem.len() > MAX_STRING_SIZE { + return Err(elem); + } + + let pair = elem.as_bytes(); + self.insert_bytes_pair(pos, pair).unwrap(); + Ok(()) + } +} + +impl CanRemove for GapBuffer { + fn can_remove(&self) -> bool { + self.len() == 0 + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn basic() { + let mut gb: GapBuffer = GapBuffer::new(); + gb.insert_bytes(0, &[3, 8]).unwrap(); + assert_eq!(gb.to_vec(), vec![3, 8]); + gb.insert_bytes(1, &[4, 5, 6]).unwrap(); + assert_eq!(gb.to_vec(), vec![3, 4, 5, 6, 8]); + assert_eq!(gb.len(), 5); + gb.insert_bytes(4, &[7]).unwrap(); + assert_eq!(gb.to_vec(), vec![3, 4, 5, 6, 7, 8]); + gb.insert_bytes(0, &[1, 2, 9, 9]).unwrap(); + assert_eq!(gb.to_vec(), vec![1, 2, 9, 9, 3, 4, 5, 6, 7, 8]); + gb.delete(2..4); + assert_eq!(gb.len(), 8); + let (left, right) = gb.as_bytes(); + assert_eq!(left, &[1, 2]); + assert_eq!(right, &[3, 4, 5, 6, 7, 8]); + assert_eq!(gb.to_vec(), vec![1, 2, 3, 4, 5, 6, 7, 8]) + } + + #[test] + fn slice() { + let mut gb = GapBuffer::new(); + gb.push_bytes(&[0, 1, 2, 3, 4, 5, 6, 7]).unwrap(); + gb.shift_at(5); + let b = gb.slice(2..5); + assert_eq!(b.to_vec(), vec![2, 3, 4]); + + gb.slice_(2..5); + assert_eq!(gb.to_vec(), vec![2, 3, 4]); + } +} diff --git a/3rdparty/loro/crates/generic-btree/src/generic_impl/len_finder.rs b/3rdparty/loro/crates/generic-btree/src/generic_impl/len_finder.rs new file mode 100644 index 0000000000..1fe0d6f813 --- /dev/null +++ b/3rdparty/loro/crates/generic-btree/src/generic_impl/len_finder.rs @@ -0,0 +1,91 @@ +use std::fmt::Debug; + +use thunderdome::Index; + +use crate::rle::HasLength; +use crate::{BTreeTrait, FindResult, Query}; + +/// A generic length finder +pub struct LengthFinder { + pub left: usize, + pub slot: u8, + pub parent: Option, +} + +impl LengthFinder { + #[inline(always)] + pub fn new() -> Self { + Self { + left: 0, + slot: 0, + parent: None, + } + } +} + +impl Default for LengthFinder { + #[inline(always)] + fn default() -> Self { + Self::new() + } +} + +pub trait UseLengthFinder { + fn get_len(cache: &B::Cache) -> usize; +} + +impl + UseLengthFinder> Query + for LengthFinder +{ + type QueryArg = usize; + + #[inline(always)] + fn init(target: &Self::QueryArg) -> Self { + Self { + left: *target, + slot: 0, + parent: None, + } + } + + #[inline(always)] + fn find_node( + &mut self, + _: &Self::QueryArg, + child_caches: &[crate::Child], + ) -> crate::FindResult { + let mut last_left = self.left; + let is_internal = child_caches.first().unwrap().is_internal(); + for (i, cache) in child_caches.iter().enumerate() { + let len = B::get_len(&cache.cache); + if self.left >= len { + last_left = self.left; + self.left -= len; + } else { + if is_internal { + self.parent = Some(cache.arena.unwrap()); + } else { + self.slot = i as u8; + } + return FindResult::new_found(i, self.left); + } + } + + self.left = last_left; + if is_internal { + self.parent = Some(child_caches.last().unwrap().arena.unwrap()); + } else { + self.slot = child_caches.len() as u8 - 1; + } + FindResult::new_missing(child_caches.len() - 1, last_left) + } + + #[inline(always)] + fn confirm_elem( + &mut self, + _: &Self::QueryArg, + elem: &::Elem, + ) -> (usize, bool) { + (self.left, self.left < elem.rle_len()) + } +} diff --git a/3rdparty/loro/crates/generic-btree/src/generic_impl/mod.rs b/3rdparty/loro/crates/generic-btree/src/generic_impl/mod.rs new file mode 100644 index 0000000000..8272773b4e --- /dev/null +++ b/3rdparty/loro/crates/generic-btree/src/generic_impl/mod.rs @@ -0,0 +1,7 @@ +mod gap_buffer; +mod len_finder; +mod ord; +mod rope; +pub use len_finder::{LengthFinder, UseLengthFinder}; +pub use ord::{OrdTreeMap, OrdTreeSet}; +pub use rope::Rope; diff --git a/3rdparty/loro/crates/generic-btree/src/generic_impl/ord.rs b/3rdparty/loro/crates/generic-btree/src/generic_impl/ord.rs new file mode 100644 index 0000000000..cd570fea08 --- /dev/null +++ b/3rdparty/loro/crates/generic-btree/src/generic_impl/ord.rs @@ -0,0 +1,408 @@ +use core::fmt::Debug; +use std::cmp::Ordering; +use std::ops::Range; + +use crate::rle::{CanRemove, HasLength, Mergeable, Sliceable, TryInsert}; +use crate::{BTree, BTreeTrait, FindResult, Query, SplitInfo}; + +#[derive(Debug)] +#[repr(transparent)] +struct OrdTrait { + _phantom: core::marker::PhantomData<(Key, Value)>, +} + +#[derive(Debug)] +pub struct OrdTreeMap { + tree: BTree>, + len: usize, +} + +#[derive(Debug)] +pub struct OrdTreeSet(OrdTreeMap); + +impl OrdTreeMap { + #[inline(always)] + pub fn new() -> Self { + Self { + tree: BTree::new(), + len: 0, + } + } + + #[inline(always)] + pub fn insert(&mut self, key: Key, value: Value) { + let Some(result) = self.tree.query::>(&key) else { + self.len += 1; + self.tree.push(Unmergeable((key, value))); + return; + }; + + if !result.found { + self.len += 1; + let tree = &mut self.tree; + let data = Unmergeable((key, value)); + let index = result.leaf(); + let leaf = tree.leaf_nodes.get_mut(index.0).unwrap(); + let parent = leaf.parent(); + + let mut is_full = false; + // Try to merge + if result.cursor.offset == 0 && data.can_merge(&leaf.elem) { + leaf.elem.merge_left(&data); + } else if result.cursor.offset == leaf.elem.rle_len() && leaf.elem.can_merge(&data) { + leaf.elem.merge_right(&data); + } else { + // Insert new leaf node + let child = tree.alloc_leaf_child(data, parent.unwrap_internal()); + let SplitInfo { + parent_idx: parent_index, + insert_slot: insert_index, + .. + } = tree.split_leaf_if_needed(result.cursor); + let parent = tree.in_nodes.get_mut(parent_index).unwrap(); + parent.children.insert(insert_index, child).unwrap(); + is_full = parent.is_full(); + } + + tree.recursive_update_cache(parent, false, None); + if is_full { + tree.split(parent); + } + } else { + let leaf = self.tree.get_elem_mut(result.leaf()).unwrap(); + leaf.0 .1 = value; + } + } + + #[inline(always)] + pub fn delete(&mut self, key: &Key) -> Option<(Key, Value)> { + let q = self.tree.query::>(key)?; + match self.tree.remove_leaf(q.cursor) { + Some(v) => { + self.len -= 1; + Some(v.0) + } + None => None, + } + } + + #[inline(always)] + pub fn iter(&self) -> impl Iterator { + self.tree.iter().map(|x| &x.0) + } + + #[inline(always)] + pub fn iter_key(&self) -> impl Iterator { + self.tree.iter().map(|x| &x.0 .0) + } + + #[inline(always)] + pub fn len(&self) -> usize { + self.len + } + + #[inline(always)] + pub fn is_empty(&self) -> bool { + self.len == 0 + } + + #[allow(unused)] + pub(crate) fn check(&self) { + self.tree.check() + } +} + +impl OrdTreeSet { + #[inline(always)] + pub fn new() -> Self { + Self(OrdTreeMap::new()) + } + + #[inline(always)] + pub fn insert(&mut self, key: Key) { + self.0.insert(key, ()); + } + + #[inline(always)] + pub fn delete(&mut self, key: &Key) -> bool { + self.0.delete(key).is_some() + } + + #[inline(always)] + pub fn iter(&self) -> impl Iterator { + self.0.iter_key() + } + + pub fn len(&self) -> usize { + self.0.len + } + + pub fn is_empty(&self) -> bool { + self.0.len == 0 + } + + #[allow(unused)] + fn check(&self) { + self.0.check() + } +} + +impl Default for OrdTreeSet { + #[inline(always)] + fn default() -> Self { + Self::new() + } +} + +impl Default + for OrdTreeMap +{ + #[inline(always)] + fn default() -> Self { + Self::new() + } +} + +impl Default for OrdTrait { + #[inline(always)] + fn default() -> Self { + Self { + _phantom: Default::default(), + } + } +} + +#[repr(transparent)] +#[derive(Debug, Clone)] +pub struct Unmergeable(T); + +impl HasLength for Unmergeable { + fn rle_len(&self) -> usize { + 1 + } +} + +impl Sliceable for Unmergeable { + fn _slice(&self, range: Range) -> Self { + if range.end - range.start != 1 { + panic!("Invalid range"); + } + + self.clone() + } +} + +impl Mergeable for Unmergeable { + fn can_merge(&self, _rhs: &Self) -> bool { + false + } + + fn merge_right(&mut self, _rhs: &Self) { + unreachable!() + } + + fn merge_left(&mut self, _left: &Self) { + unreachable!() + } +} + +impl TryInsert for Unmergeable { + fn try_insert(&mut self, _pos: usize, elem: Self) -> Result<(), Self> { + Err(elem) + } +} + +impl CanRemove for Unmergeable { + fn can_remove(&self) -> bool { + false + } +} + +impl BTreeTrait for OrdTrait { + type Elem = Unmergeable<(Key, Value)>; + type Cache = Option<(Key, Key)>; + type CacheDiff = (); + const USE_DIFF: bool = false; + + #[inline(always)] + fn calc_cache_internal(cache: &mut Self::Cache, caches: &[crate::Child]) { + if caches.is_empty() { + return; + } + + *cache = Some(( + caches[0].cache.as_ref().unwrap().0.clone(), + caches[caches.len() - 1].cache.as_ref().unwrap().1.clone(), + )); + } + + #[inline(always)] + fn apply_cache_diff(_: &mut Self::Cache, _: &Self::CacheDiff) { + unreachable!() + } + + #[inline(always)] + fn merge_cache_diff(_: &mut Self::CacheDiff, _: &Self::CacheDiff) {} + + #[inline(always)] + fn get_elem_cache(elem: &Self::Elem) -> Self::Cache { + Some((elem.0 .0.clone(), elem.0 .0.clone())) + } + + #[inline(always)] + fn new_cache_to_diff(_: &Self::Cache) -> Self::CacheDiff {} + + fn sub_cache(_: &Self::Cache, _: &Self::Cache) -> Self::CacheDiff {} +} + +impl Query> + for OrdTrait +{ + type QueryArg = Key; + + #[inline(always)] + fn init(_target: &Self::QueryArg) -> Self { + Self::default() + } + + #[inline] + fn find_node( + &mut self, + target: &Self::QueryArg, + child_caches: &[crate::Child>], + ) -> crate::FindResult { + let result = child_caches.binary_search_by(|x| { + let (min, max) = x.cache.as_ref().unwrap(); + if target < min { + core::cmp::Ordering::Greater + } else if target > max { + core::cmp::Ordering::Less + } else { + core::cmp::Ordering::Equal + } + }); + match result { + Ok(i) => FindResult::new_found(i, 0), + Err(i) => FindResult::new_missing( + i.min(child_caches.len() - 1), + if i == child_caches.len() { 1 } else { 0 }, + ), + } + } + + #[inline(always)] + fn confirm_elem( + &mut self, + q: &Self::QueryArg, + elem: & as BTreeTrait>::Elem, + ) -> (usize, bool) { + match q.cmp(&elem.0 .0) { + Ordering::Less => (0, false), + Ordering::Equal => (0, true), + Ordering::Greater => (1, false), + } + } +} + +#[cfg(test)] +mod test { + use std::cmp::Ordering; + + use rand::{Rng, SeedableRng}; + + use crate::HeapVec; + + use super::*; + + #[test] + fn test() { + let mut tree: OrdTreeSet = OrdTreeSet::new(); + let mut rng = rand::rngs::StdRng::seed_from_u64(123); + let mut data: HeapVec = (0..1000).map(|_| rng.gen()).collect(); + for &value in data.iter() { + tree.insert(value); + } + data.sort_unstable(); + assert_eq!(tree.iter().copied().collect::>(), data); + tree.check(); + } + + #[test] + fn test_delete() { + let mut tree: OrdTreeSet = OrdTreeSet::new(); + tree.insert(12); + tree.delete(&12); + assert_eq!(tree.len(), 0); + } + + #[test] + fn test_compare_pos() { + let mut tree: OrdTreeSet = OrdTreeSet::new(); + for i in 0..100 { + tree.insert(i); + } + + for i in 0..99 { + let a = tree.0.tree.query::>(&i).unwrap(); + assert_eq!( + tree.0.tree.compare_pos(a.cursor(), a.cursor()), + Ordering::Equal + ); + for j in i + 1..100 { + let b = tree.0.tree.query::>(&j).unwrap(); + assert_eq!( + tree.0.tree.compare_pos(a.cursor(), b.cursor()), + Ordering::Less + ); + assert_eq!( + tree.0.tree.compare_pos(b.cursor(), a.cursor()), + Ordering::Greater + ); + } + } + } + + mod move_event_test { + + use super::*; + + #[test] + fn test() { + let mut tree: OrdTreeMap = OrdTreeMap::new(); + let mut rng = rand::rngs::StdRng::seed_from_u64(123); + let mut data: HeapVec = (0..1000).map(|_| rng.gen()).collect(); + for &value in data.iter() { + tree.insert(value, 0); + } + for value in data.drain(0..100) { + tree.delete(&value); + } + for value in data.drain(0..800) { + tree.delete(&value); + } + tree.tree.check(); + for _ in (0..100).rev() { + tree.delete(&data.pop().unwrap()); + } + } + } + + #[test] + #[ignore] + fn depth_test() { + let mut tree: OrdTreeSet = OrdTreeSet::new(); + for i in 0..2_100_000 { + tree.insert(i as u64); + let m = (!i) + 1; + if (i & m) == i { + eprintln!( + "i={}, Depth={}, Avg Children={}", + i, + tree.0.tree.depth(), + tree.0.tree.internal_avg_children_num() + ); + } + } + tree.check(); + } +} diff --git a/3rdparty/loro/crates/generic-btree/src/generic_impl/rope.rs b/3rdparty/loro/crates/generic-btree/src/generic_impl/rope.rs new file mode 100644 index 0000000000..e1c0c71a36 --- /dev/null +++ b/3rdparty/loro/crates/generic-btree/src/generic_impl/rope.rs @@ -0,0 +1,3442 @@ +extern crate alloc; + +use core::ops::RangeBounds; +use std::assert_eq; +use std::fmt::Display; + +use crate::generic_impl::gap_buffer::MAX_STRING_SIZE; +use crate::rle::Sliceable; +use crate::{BTree, BTreeTrait, LeafIndex, LengthFinder, QueryResult}; + +use super::gap_buffer::GapBuffer; +use super::len_finder::UseLengthFinder; + +#[derive(Debug)] +struct RopeTrait; + +#[derive(Debug)] +struct Cursor { + pos: usize, + leaf: LeafIndex, +} + +// TODO: move Rope into a separate project +#[derive(Debug)] +pub struct Rope { + tree: BTree, + cursor: Option, +} + +impl UseLengthFinder for RopeTrait { + #[inline(always)] + fn get_len(cache: &::Cache) -> usize { + *cache as usize + } +} + +impl Rope { + #[inline(always)] + pub fn len(&self) -> usize { + self.tree.root_cache as usize + } + + #[inline(always)] + pub fn is_empty(&self) -> bool { + self.tree.root_cache == 0 + } + + pub fn insert(&mut self, index: usize, elem: &str) { + if index > self.len() { + panic!("index {} out of range len={}", index, self.len()); + } + + if self.is_empty() { + for chunk in GapBuffer::from_str(elem) { + self.tree.push(chunk); + } + return; + } + + if let Some(Cursor { pos, leaf }) = self.cursor { + if pos <= index { + let node = self.tree.leaf_nodes.get(leaf.0).unwrap(); + if index <= pos + node.elem.len() { + let mut success = true; + let offset = index - pos; + let valid = self + .tree + .update_leaf(leaf, |leaf| { + if leaf.len() + elem.len() < MAX_STRING_SIZE { + leaf.insert_bytes(offset, elem.as_bytes()).unwrap(); + (true, None, None) + } else { + let mut right = leaf.split(offset); + if leaf.len() + elem.len() < MAX_STRING_SIZE { + success = leaf.push_bytes(elem.as_bytes()).is_ok(); + } else { + success = right.insert_bytes(0, elem.as_bytes()).is_ok(); + } + + (true, Some(right), None) + } + }) + .0; + + if !valid { + self.cursor = None; + } + + if success { + return; + } + } + } + } + + let (q, f) = self.tree.query_with_finder_return::(&index); + self.cursor = q.and_then(|q| { + if q.offset() == 0 { + if f.slot == 0 || f.parent.is_none() { + None + } else { + let node = self.tree.in_nodes.get(f.parent.unwrap()).unwrap(); + let child = &node.children[f.slot as usize - 1]; + Some(Cursor { + pos: index - child.cache as usize, + leaf: child.arena.unwrap().into(), + }) + } + } else { + Some(Cursor { + pos: index - q.offset(), + leaf: q.leaf(), + }) + } + }); + + self.tree + .insert_many_by_cursor(q.map(|x| x.cursor), GapBuffer::from_str(elem)); + } + + pub fn delete_range(&mut self, range: impl RangeBounds) { + if self.is_empty() { + return; + } + + let start = match range.start_bound() { + core::ops::Bound::Included(x) => *x, + core::ops::Bound::Excluded(x) => *x + 1, + core::ops::Bound::Unbounded => 0, + }; + let end = match range.end_bound() { + core::ops::Bound::Included(&x) => x + 1, + core::ops::Bound::Excluded(&x) => x, + core::ops::Bound::Unbounded => self.len(), + }; + let end = end.min(self.len()); + let start = start.min(end); + if start == end { + return; + } + + if let Some(Cursor { pos, leaf }) = self.cursor { + if pos <= start { + let node = self.tree.leaf_nodes.get(leaf.0).unwrap(); + if end <= pos + node.elem.len() { + let start_offset = start - pos; + let end_offset = end - pos; + let valid = self + .tree + .update_leaf(leaf, |leaf| { + leaf.delete(start_offset..end_offset); + (true, None, None) + }) + .0; + + if !valid { + self.cursor = None; + } + + return; + } + } + } + + if end - start == 1 { + let q = self + .tree + .update_leaf_by_search::(&start, |leaf, pos| { + leaf.delete(pos.cursor.offset..pos.cursor.offset + 1); + Some((-1, None, None)) + }); + self.cursor = q.0.map(|q| Cursor { + pos: start - q.offset, + leaf: q.leaf, + }); + + return; + } + + self.cursor = None; + let from = self.tree.query::(&start); + let to = self.tree.query::(&end); + match (from, to) { + (Some(from), Some(to)) if from.cursor.leaf == to.cursor.leaf => { + let leaf = self.tree.leaf_nodes.get_mut(from.arena()).unwrap(); + if from.cursor.offset == 0 && to.cursor.offset == leaf.elem.len() { + // delete the whole leaf + self.tree.remove_leaf(from.cursor); + } else { + leaf.elem.delete(from.cursor.offset..to.cursor.offset); + self.tree.recursive_update_cache( + from.leaf().into(), + true, + Some(start as isize - end as isize), + ); + } + } + _ => { + crate::iter::Drain::new(&mut self.tree, from, to); + } + } + } + + fn iter(&self) -> impl Iterator { + let mut node_iter = self + .tree + .first_path() + .map(|first| crate::iter::Iter::new(&self.tree, first, self.tree.last_path().unwrap())); + std::iter::from_fn(move || match &mut node_iter { + Some(node_iter) => { + if let Some(node) = node_iter.next() { + Some(&node.1.elem) + } else { + None + } + } + None => None, + }) + } + + pub fn slice(&mut self, _range: impl RangeBounds) { + unimplemented!() + } + + pub fn new() -> Self { + Self { + tree: BTree::new(), + cursor: None, + } + } + + #[allow(unused)] + fn node_len(&self) -> usize { + self.tree.node_len() + } + + #[allow(unused)] + fn update_in_place(&mut self, pos: usize, new: &str) { + todo!() + } + + pub fn clear(&mut self) { + self.tree.clear(); + } + + #[allow(unused)] + pub fn check(&self) { + // dbg!(&self.tree); + self.tree.check() + } + + pub fn diagnose(&self) { + self.tree.diagnose_balance(); + } +} + +impl Default for Rope { + fn default() -> Self { + Self::new() + } +} + +impl Display for Rope { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut ans = Vec::with_capacity(self.len()); + for elem in self.iter() { + let (left, right) = elem.as_bytes(); + ans.extend_from_slice(left); + ans.extend_from_slice(right); + } + + f.write_str(std::str::from_utf8(ans.as_slice()).unwrap()) + } +} + +impl BTreeTrait for RopeTrait { + type Elem = GapBuffer; + type Cache = isize; + type CacheDiff = isize; + + #[inline(always)] + fn calc_cache_internal(cache: &mut Self::Cache, caches: &[crate::Child]) -> isize { + let new_cache = caches.iter().map(|x| x.cache).sum::(); + let diff = new_cache - *cache; + *cache = new_cache; + diff + } + + #[inline(always)] + fn apply_cache_diff(cache: &mut Self::Cache, diff: &Self::CacheDiff) { + *cache += *diff; + } + + #[inline(always)] + fn merge_cache_diff(diff1: &mut Self::CacheDiff, diff2: &Self::CacheDiff) { + *diff1 += diff2; + } + + #[inline(always)] + fn get_elem_cache(elem: &Self::Elem) -> Self::Cache { + elem.len() as isize + } + + #[inline(always)] + fn new_cache_to_diff(cache: &Self::Cache) -> Self::CacheDiff { + *cache + } + + fn sub_cache(cache_lhs: &Self::Cache, cache_rhs: &Self::Cache) -> Self::CacheDiff { + cache_lhs - cache_rhs + } +} + +#[allow(unused)] +fn test_prev_length(rope: &Rope, q: QueryResult) -> usize { + let mut count = 0; + rope.tree + .visit_previous_caches(q.cursor(), |cache| match cache { + crate::PreviousCache::NodeCache(cache) => { + count += *cache as usize; + } + crate::PreviousCache::PrevSiblingElem(p) => { + count += p.len(); + } + crate::PreviousCache::ThisElemAndOffset { offset, .. } => { + count += offset; + } + }); + count +} + +#[allow(unused)] +fn test_index(rope: &Rope) { + for index in 0..rope.len() { + let q = rope.tree.query::(&index).unwrap(); + let i = test_prev_length(rope, q); + assert_eq!(i, index); + } +} + +#[cfg(test)] +mod test { + + use Action::*; + + use crate::HeapVec; + + use super::*; + + #[test] + fn test() { + let mut rope = Rope::new(); + rope.insert(0, "123"); + assert_eq!(rope.len(), 3); + rope.insert(1, "x"); + test_index(&rope); + assert_eq!(rope.len(), 4); + rope.delete_range(2..4); + assert_eq!(&rope.to_string(), "1x"); + rope.delete_range(..1); + assert_eq!(&rope.to_string(), "x"); + rope.delete_range(..); + assert_eq!(&rope.to_string(), ""); + assert_eq!(rope.len(), 0); + } + + #[test] + fn test_delete_middle() { + let mut rope = Rope::new(); + rope.insert(0, "135"); + rope.delete_range(1..2); + assert_eq!(&rope.to_string(), "15"); + } + + #[test] + fn test_insert_repeatedly() { + let mut rope = Rope::new(); + rope.insert(0, "123"); + rope.insert(1, "x"); + rope.insert(2, "y"); + rope.insert(3, "z"); + test_index(&rope); + assert_eq!(&rope.to_string(), "1xyz23"); + } + + #[test] + #[ignore] + fn test_update() { + let mut rope = Rope::new(); + rope.insert(0, "123"); + rope.insert(3, "xyz"); + rope.update_in_place(1, "kkkk"); + assert_eq!(&rope.to_string(), "1kkkkz"); + } + + #[test] + fn test_clear() { + let mut rope = Rope::new(); + rope.insert(0, "123"); + assert_eq!(rope.len(), 3); + rope.clear(); + assert_eq!(rope.len(), 0); + assert_eq!(&rope.to_string(), ""); + rope.insert(0, "kkk"); + assert_eq!(&rope.to_string(), "kkk"); + } + + #[test] + fn test_insert_many() { + let mut rope = Rope::new(); + let s = "_12345678_".repeat(10); + let mut expected = String::new(); + for i in 0..100 { + expected.insert_str(i, &s); + rope.insert(i, &s); + assert_eq!(&rope.to_string(), &expected) + } + } + + #[test] + fn test_repeat_insert() { + let mut rope = Rope::new(); + rope.insert(0, "123"); + for _ in 0..10000 { + rope.insert(rope.len() / 2, "k"); + } + } + + #[test] + #[ignore] + fn test_update_1() { + let mut rope = Rope::new(); + for i in 0..100 { + rope.insert(i, &(i % 10).to_string()); + } + + rope.update_in_place(15, "kkkkk"); + assert_eq!(&rope.to_string()[10..20], "01234kkkkk"); + test_index(&rope); + } + + #[derive(Debug)] + enum Action { + Insert { pos: u8, content: u8 }, + Delete { pos: u8, len: u8 }, + } + + fn fuzz(data: HeapVec) { + let mut rope = Rope::new(); + let mut truth = String::new(); + for action in data { + match action { + Action::Insert { pos, content } => { + let pos = pos as usize % (truth.len() + 1); + let s = content.to_string(); + dbg!("INS", pos, &s); + dbg!(&rope); + truth.insert_str(pos, &s); + rope.insert(pos, &s); + dbg!(&rope); + rope.check(); + assert_eq!(rope.len(), truth.len()); + assert_eq!(rope.to_string(), truth, "{:#?}", &rope.tree); + } + Action::Delete { pos, len } => { + let pos = pos as usize % (truth.len() + 1); + let mut len = len as usize % 10; + len = len.min(truth.len() - pos); + dbg!("DEL", pos, len); + dbg!(&rope); + rope.delete_range(pos..(pos + len)); + dbg!(&rope); + truth.drain(pos..pos + len); + rope.check(); + assert_eq!(rope.len(), truth.len()); + assert_eq!(rope.to_string(), truth, "{:#?}", &rope.tree); + } + } + } + + assert_eq!(rope.to_string(), truth); + } + + #[test] + fn fuzz_0() { + fuzz(vec![ + Insert { + pos: 0, + content: 128, + }, + Insert { + pos: 0, + content: 249, + }, + Insert { + pos: 108, + content: 108, + }, + Delete { pos: 192, len: 193 }, + Insert { + pos: 106, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 100, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { pos: 0, content: 8 }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 111, + content: 127, + }, + Delete { pos: 255, len: 255 }, + Delete { pos: 255, len: 36 }, + Delete { pos: 255, len: 255 }, + Delete { pos: 255, len: 255 }, + Delete { pos: 255, len: 255 }, + Delete { pos: 135, len: 169 }, + Delete { pos: 255, len: 255 }, + Delete { pos: 255, len: 255 }, + Delete { pos: 255, len: 255 }, + Delete { pos: 255, len: 255 }, + ]) + } + + #[test] + fn fuzz_1() { + fuzz(vec![ + Insert { + pos: 157, + content: 108, + }, + Insert { + pos: 255, + content: 255, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 8, + content: 101, + }, + Insert { + pos: 111, + content: 127, + }, + Delete { pos: 255, len: 169 }, + ]) + } + + #[test] + fn fuzz_2() { + fuzz(vec![ + Insert { + pos: 0, + content: 128, + }, + Insert { + pos: 0, + content: 249, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 0, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 249, + }, + Insert { + pos: 135, + content: 255, + }, + Delete { pos: 255, len: 255 }, + Delete { pos: 169, len: 169 }, + ]) + } + + #[test] + fn fuzz_3() { + fuzz(vec![ + Insert { + pos: 111, + content: 140, + }, + Insert { + pos: 111, + content: 107, + }, + Insert { + pos: 35, + content: 102, + }, + Insert { + pos: 102, + content: 102, + }, + Insert { + pos: 102, + content: 102, + }, + Insert { + pos: 102, + content: 102, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 64, + content: 0, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 93, + content: 93, + }, + Insert { + pos: 93, + content: 93, + }, + Insert { + pos: 93, + content: 93, + }, + Insert { + pos: 93, + content: 93, + }, + Insert { + pos: 93, + content: 93, + }, + Insert { + pos: 93, + content: 93, + }, + Insert { + pos: 93, + content: 93, + }, + Insert { + pos: 93, + content: 93, + }, + Insert { + pos: 93, + content: 93, + }, + Insert { + pos: 93, + content: 93, + }, + Insert { + pos: 93, + content: 93, + }, + Insert { + pos: 93, + content: 93, + }, + Insert { + pos: 93, + content: 93, + }, + Insert { + pos: 93, + content: 93, + }, + Insert { + pos: 93, + content: 93, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 102, + content: 119, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 119, + content: 111, + }, + Insert { + pos: 102, + content: 102, + }, + Insert { + pos: 102, + content: 101, + }, + Insert { + pos: 36, + content: 146, + }, + Delete { pos: 74, len: 102 }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 17, + content: 17, + }, + Insert { + pos: 17, + content: 17, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 102, + content: 119, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 119, + content: 111, + }, + Insert { + pos: 102, + content: 102, + }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 3, + content: 73, + }, + Insert { + pos: 146, + content: 74, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 21, + content: 119, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 111, + content: 111, + }, + Insert { pos: 0, content: 8 }, + Insert { + pos: 102, + content: 102, + }, + Insert { + pos: 102, + content: 102, + }, + Insert { + pos: 102, + content: 3, + }, + Insert { + pos: 36, + content: 146, + }, + Insert { + pos: 119, + content: 119, + }, + Delete { pos: 111, len: 119 }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 102, + content: 102, + }, + Insert { + pos: 102, + content: 102, + }, + Insert { + pos: 73, + content: 36, + }, + Delete { pos: 74, len: 102 }, + Delete { pos: 255, len: 255 }, + Insert { + pos: 42, + content: 64, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 102, + content: 102, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 42, + content: 42, + }, + Insert { + pos: 42, + content: 42, + }, + Insert { + pos: 42, + content: 42, + }, + Insert { + pos: 0, + content: 15, + }, + Insert { + pos: 42, + content: 42, + }, + Insert { + pos: 42, + content: 42, + }, + Insert { + pos: 42, + content: 42, + }, + Insert { + pos: 42, + content: 42, + }, + Insert { + pos: 42, + content: 42, + }, + Insert { + pos: 42, + content: 42, + }, + Insert { + pos: 42, + content: 42, + }, + Insert { + pos: 42, + content: 42, + }, + Insert { + pos: 42, + content: 42, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 102, + content: 102, + }, + Insert { + pos: 102, + content: 102, + }, + Insert { + pos: 102, + content: 3, + }, + Insert { + pos: 36, + content: 146, + }, + Insert { + pos: 255, + content: 255, + }, + Insert { + pos: 42, + content: 42, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 42, + content: 42, + }, + Insert { + pos: 42, + content: 38, + }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 89, + content: 89, + }, + Insert { + pos: 89, + content: 89, + }, + Insert { + pos: 89, + content: 89, + }, + Insert { + pos: 89, + content: 89, + }, + Insert { + pos: 89, + content: 89, + }, + Insert { + pos: 89, + content: 89, + }, + Insert { + pos: 89, + content: 89, + }, + Insert { + pos: 89, + content: 89, + }, + Insert { + pos: 89, + content: 89, + }, + Insert { + pos: 89, + content: 89, + }, + Insert { + pos: 89, + content: 89, + }, + Insert { + pos: 89, + content: 89, + }, + Insert { + pos: 89, + content: 89, + }, + Insert { + pos: 89, + content: 89, + }, + Insert { + pos: 89, + content: 89, + }, + Insert { + pos: 89, + content: 89, + }, + Insert { + pos: 42, + content: 42, + }, + Insert { + pos: 42, + content: 42, + }, + Insert { + pos: 42, + content: 42, + }, + Insert { + pos: 42, + content: 42, + }, + Insert { + pos: 42, + content: 42, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 119, + content: 37, + }, + Insert { + pos: 101, + content: 102, + }, + Insert { pos: 0, content: 0 }, + Delete { pos: 193, len: 63 }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 0, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { pos: 0, content: 8 }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 102, + content: 102, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 119, + content: 119, + }, + Delete { pos: 199, len: 199 }, + Delete { pos: 199, len: 199 }, + Delete { pos: 199, len: 199 }, + Delete { pos: 199, len: 199 }, + Delete { pos: 199, len: 199 }, + Delete { pos: 199, len: 187 }, + Delete { pos: 187, len: 187 }, + Delete { pos: 187, len: 187 }, + Delete { pos: 187, len: 187 }, + Delete { pos: 187, len: 187 }, + Delete { pos: 187, len: 187 }, + Delete { pos: 187, len: 187 }, + Delete { pos: 187, len: 187 }, + Delete { pos: 187, len: 187 }, + Insert { + pos: 3, + content: 119, + }, + Insert { + pos: 102, + content: 102, + }, + Delete { pos: 163, len: 163 }, + Delete { pos: 163, len: 163 }, + Delete { pos: 163, len: 102 }, + Insert { + pos: 102, + content: 102, + }, + Insert { + pos: 108, + content: 249, + }, + Insert { + pos: 135, + content: 169, + }, + Delete { pos: 255, len: 255 }, + Delete { pos: 255, len: 255 }, + Delete { pos: 111, len: 255 }, + Insert { + pos: 111, + content: 111, + }, + Insert { + pos: 255, + content: 255, + }, + ]) + } + + #[test] + fn fuzz_4() { + fuzz(vec![ + Insert { + pos: 0, + content: 128, + }, + Insert { + pos: 0, + content: 249, + }, + Insert { pos: 8, content: 0 }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 0, + }, + Insert { + pos: 108, + content: 108, + }, + ]) + } + + #[test] + fn fuzz_5() { + fuzz(vec![ + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 0, + content: 123, + }, + Delete { pos: 108, len: 108 }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 12, + content: 0, + }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 127, + content: 135, + }, + Delete { pos: 255, len: 246 }, + Delete { pos: 246, len: 246 }, + Delete { pos: 246, len: 246 }, + Delete { pos: 246, len: 246 }, + Insert { + pos: 101, + content: 101, + }, + Insert { + pos: 101, + content: 101, + }, + Delete { pos: 255, len: 255 }, + Delete { pos: 169, len: 169 }, + ]) + } + + #[test] + fn fuzz_6() { + fuzz(vec![ + Insert { + pos: 0, + content: 128, + }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 0, + content: 249, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 171, + content: 171, + }, + Delete { pos: 171, len: 0 }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 171, + }, + Delete { pos: 187, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 171, + content: 171, + }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 110, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 171, + }, + Delete { pos: 187, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Insert { + pos: 8, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 50, + content: 108, + }, + Delete { pos: 108, len: 108 }, + Insert { + pos: 108, + content: 87, + }, + Insert { + pos: 249, + content: 1, + }, + Delete { pos: 169, len: 235 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 163, len: 171 }, + Delete { pos: 171, len: 171 }, + Insert { pos: 8, content: 0 }, + Insert { pos: 0, content: 0 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 41, len: 164 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 2, + }, + Insert { + pos: 254, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 0, + content: 123, + }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Insert { + pos: 238, + content: 238, + }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 238, + content: 238, + }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 86, + content: 86, + }, + Insert { + pos: 123, + content: 2, + }, + Insert { + pos: 254, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 0, + content: 238, + }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Insert { + pos: 238, + content: 123, + }, + Delete { pos: 123, len: 123 }, + Insert { + pos: 86, + content: 254, + }, + Insert { + pos: 33, + content: 238, + }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Insert { + pos: 123, + content: 2, + }, + Insert { pos: 0, content: 0 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 123, len: 123 }, + Insert { + pos: 0, + content: 121, + }, + Insert { + pos: 26, + content: 0, + }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 123, + content: 123, + }, + Delete { pos: 238, len: 254 }, + Insert { + pos: 144, + content: 238, + }, + Delete { pos: 91, len: 238 }, + Delete { pos: 238, len: 238 }, + Insert { + pos: 123, + content: 238, + }, + Delete { pos: 238, len: 238 }, + Delete { pos: 0, len: 51 }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 102, + content: 102, + }, + Insert { + pos: 102, + content: 102, + }, + Insert { + pos: 123, + content: 123, + }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 123 }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 86, + }, + Delete { pos: 101, len: 144 }, + Delete { pos: 238, len: 91 }, + Delete { pos: 238, len: 238 }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 108, + content: 108, + }, + Insert { pos: 3, content: 0 }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 171, + content: 63, + }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 108, + content: 108, + }, + Insert { pos: 0, content: 0 }, + Delete { pos: 235, len: 235 }, + Insert { + pos: 108, + content: 108, + }, + Insert { pos: 8, content: 0 }, + Insert { + pos: 127, + content: 135, + }, + Delete { pos: 171, len: 171 }, + Delete { pos: 0, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Insert { + pos: 0, + content: 171, + }, + Delete { pos: 1, len: 126 }, + Delete { pos: 235, len: 154 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Insert { + pos: 84, + content: 84, + }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Insert { pos: 0, content: 0 }, + Delete { pos: 91, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 171, len: 171 }, + Insert { + pos: 249, + content: 1, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 108, + content: 108, + }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Insert { pos: 0, content: 8 }, + Insert { + pos: 108, + content: 32, + }, + Insert { pos: 0, content: 0 }, + Delete { pos: 235, len: 108 }, + Insert { + pos: 108, + content: 108, + }, + Delete { pos: 255, len: 6 }, + Insert { + pos: 135, + content: 169, + }, + Delete { pos: 171, len: 171 }, + Insert { + pos: 171, + content: 171, + }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Insert { + pos: 171, + content: 171, + }, + Insert { + pos: 126, + content: 111, + }, + Delete { pos: 154, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Insert { + pos: 84, + content: 171, + }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 235, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 87, + content: 0, + }, + Delete { pos: 1, len: 111 }, + Delete { pos: 238, len: 238 }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 121, + content: 86, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 86, + content: 254, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 86, + content: 0, + }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 254, len: 193 }, + Delete { pos: 63, len: 64 }, + Insert { pos: 0, content: 0 }, + Delete { pos: 235, len: 235 }, + Insert { + pos: 108, + content: 108, + }, + Insert { pos: 0, content: 8 }, + Insert { + pos: 111, + content: 127, + }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 0 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 8, len: 0 }, + Delete { pos: 249, len: 1 }, + Delete { pos: 169, len: 235 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Insert { + pos: 8, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 50, + content: 108, + }, + Delete { pos: 108, len: 108 }, + Insert { + pos: 108, + content: 8, + }, + Insert { pos: 0, content: 0 }, + Delete { pos: 169, len: 235 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Insert { pos: 8, content: 0 }, + Insert { pos: 0, content: 0 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 41, len: 164 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Delete { pos: 235, len: 235 }, + Insert { + pos: 108, + content: 108, + }, + Insert { pos: 8, content: 0 }, + Insert { + pos: 171, + content: 171, + }, + Insert { pos: 8, content: 0 }, + Insert { + pos: 127, + content: 135, + }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 41, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 41 }, + Insert { + pos: 171, + content: 171, + }, + Delete { pos: 171, len: 171 }, + Delete { pos: 165, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 170 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 235, len: 235 }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 0, + content: 108, + }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Insert { + pos: 171, + content: 171, + }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Delete { pos: 235, len: 235 }, + Insert { + pos: 108, + content: 108, + }, + Insert { pos: 8, content: 0 }, + Insert { + pos: 127, + content: 135, + }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Insert { + pos: 123, + content: 2, + }, + Insert { + pos: 254, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Delete { pos: 255, len: 255 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 121, + content: 86, + }, + Insert { pos: 0, content: 0 }, + Delete { pos: 238, len: 238 }, + Insert { + pos: 123, + content: 123, + }, + Delete { pos: 255, len: 255 }, + Delete { pos: 8, len: 238 }, + Delete { pos: 238, len: 238 }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Delete { pos: 255, len: 255 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Insert { + pos: 238, + content: 238, + }, + Insert { pos: 0, content: 0 }, + Delete { pos: 91, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 18 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 121, + content: 86, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 86, + content: 254, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 0, + content: 123, + }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 91, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 123 }, + Delete { pos: 238, len: 238 }, + Insert { + pos: 121, + content: 86, + }, + Insert { + pos: 123, + content: 123, + }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 86, + content: 86, + }, + Insert { + pos: 202, + content: 238, + }, + Delete { pos: 238, len: 238 }, + Insert { + pos: 123, + content: 2, + }, + Insert { + pos: 254, + content: 123, + }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 123, + content: 123, + }, + Delete { pos: 238, len: 238 }, + Delete { pos: 255, len: 101 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Insert { + pos: 238, + content: 123, + }, + Delete { pos: 123, len: 238 }, + Delete { pos: 238, len: 238 }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 102, + content: 102, + }, + Insert { + pos: 102, + content: 102, + }, + Insert { + pos: 102, + content: 123, + }, + Insert { + pos: 238, + content: 238, + }, + Delete { pos: 238, len: 238 }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Delete { pos: 255, len: 255 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 1, len: 0 }, + Insert { pos: 0, content: 7 }, + Insert { + pos: 108, + content: 108, + }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 108, + content: 108, + }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 108 }, + Insert { + pos: 108, + content: 108, + }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 235, + content: 235, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 111, + content: 111, + }, + Delete { pos: 154, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Insert { + pos: 171, + content: 8, + }, + Delete { pos: 171, len: 249 }, + Insert { + pos: 135, + content: 169, + }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Insert { + pos: 87, + content: 84, + }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 11, len: 238 }, + Insert { pos: 0, content: 0 }, + Delete { pos: 41, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 171, len: 0 }, + Insert { + pos: 108, + content: 108, + }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 0, + content: 108, + }, + Delete { pos: 63, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 157, len: 157 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 108, len: 0 }, + Insert { + pos: 108, + content: 108, + }, + Insert { pos: 0, content: 0 }, + Delete { pos: 235, len: 235 }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 0, + content: 248, + }, + Delete { pos: 154, len: 127 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 0 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 8, len: 0 }, + Delete { pos: 249, len: 1 }, + Delete { pos: 169, len: 235 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Insert { + pos: 84, + content: 84, + }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Insert { pos: 0, content: 8 }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 49, + }, + Delete { pos: 235, len: 108 }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 0, + content: 249, + }, + Insert { + pos: 135, + content: 169, + }, + Delete { pos: 238, len: 123 }, + Insert { pos: 2, content: 0 }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 121, + content: 86, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 1, + }, + Delete { pos: 238, len: 238 }, + Insert { + pos: 238, + content: 238, + }, + Delete { pos: 238, len: 238 }, + Insert { + pos: 193, + content: 192, + }, + Delete { pos: 63, len: 127 }, + Insert { + pos: 0, + content: 235, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 87, + content: 0, + }, + Delete { pos: 1, len: 111 }, + Delete { pos: 235, len: 154 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 0, len: 171 }, + Delete { pos: 171, len: 171 }, + Insert { + pos: 127, + content: 135, + }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Delete { pos: 235, len: 235 }, + Insert { + pos: 108, + content: 108, + }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 127, + content: 135, + }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 172 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 0 }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 0, + content: 171, + }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 108 }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 235, + content: 235, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 111, + content: 111, + }, + Delete { pos: 171, len: 0 }, + Insert { + pos: 48, + content: 111, + }, + Delete { pos: 154, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Insert { + pos: 84, + content: 84, + }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 235 }, + Delete { pos: 254, len: 86 }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { pos: 0, content: 8 }, + Insert { + pos: 108, + content: 171, + }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 20 }, + Delete { pos: 171, len: 108 }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 235, + content: 235, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 111, + content: 111, + }, + Delete { pos: 154, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 123, len: 123 }, + Insert { + pos: 86, + content: 86, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 238, + }, + Delete { pos: 238, len: 238 }, + Insert { + pos: 123, + content: 36, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 254, + content: 255, + }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 123 }, + Insert { pos: 0, content: 0 }, + Insert { pos: 0, content: 0 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 123, len: 123 }, + Insert { + pos: 238, + content: 238, + }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Insert { + pos: 238, + content: 238, + }, + Insert { pos: 0, content: 0 }, + Delete { pos: 91, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Insert { pos: 0, content: 0 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 238, + content: 238, + }, + Insert { + pos: 108, + content: 108, + }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Insert { + pos: 123, + content: 2, + }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 238, + content: 238, + }, + Insert { + pos: 0, + content: 238, + }, + Delete { pos: 238, len: 238 }, + Delete { pos: 0, len: 249 }, + Insert { + pos: 135, + content: 255, + }, + Delete { pos: 255, len: 255 }, + Delete { pos: 144, len: 255 }, + Delete { pos: 169, len: 169 }, + ]) + } + + #[test] + fn ben() { + use arbitrary::Arbitrary; + #[derive(Arbitrary, Debug, Clone, Copy)] + enum Action { + Insert { pos: u8, content: u8 }, + Delete { pos: u8, len: u8 }, + } + + use rand::{Rng, SeedableRng}; + let mut rng = rand::rngs::StdRng::seed_from_u64(123); + let mut expected = String::new(); + let unstructured: Vec = (0..10_000).map(|_| rng.gen()).collect(); + let mut gen = arbitrary::Unstructured::new(&unstructured); + let actions: [Action; 1_000] = gen.arbitrary().unwrap(); + let mut rope = Rope::new(); + for action in actions.iter() { + match *action { + Action::Insert { pos, content } => { + let pos = pos as usize % (rope.len() + 1); + let s = content.to_string(); + expected.insert_str(pos, &s); + rope.insert(pos, &s); + assert_eq!(expected.len(), rope.len()); + } + Action::Delete { pos, len } => { + let pos = pos as usize % (rope.len() + 1); + let mut len = len as usize % 10; + len = len.min(rope.len() - pos); + expected.drain(pos..pos + len); + rope.delete_range(pos..(pos + len)); + assert_eq!(expected.len(), rope.len()); + } + } + } + assert_eq!(rope.to_string(), expected); + } + + #[test] + fn fuzz_7() { + fuzz(vec![ + Insert { + pos: 111, + content: 111, + }, + Insert { pos: 0, content: 0 }, + Insert { pos: 0, content: 0 }, + Insert { pos: 0, content: 0 }, + Insert { pos: 0, content: 0 }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 255, + }, + Delete { pos: 255, len: 0 }, + Insert { + pos: 140, + content: 140, + }, + Insert { + pos: 102, + content: 101, + }, + Insert { + pos: 36, + content: 146, + }, + Insert { + pos: 102, + content: 119, + }, + Insert { + pos: 118, + content: 119, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 21, + content: 0, + }, + Insert { + pos: 140, + content: 140, + }, + Insert { + pos: 107, + content: 19, + }, + Insert { + pos: 102, + content: 47, + }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 0, + content: 102, + }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 0, + content: 102, + }, + Insert { + pos: 102, + content: 64, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 123, + content: 123, + }, + Delete { pos: 255, len: 136 }, + Delete { pos: 119, len: 111 }, + Insert { + pos: 102, + content: 102, + }, + Insert { + pos: 3, + content: 73, + }, + Insert { + pos: 146, + content: 74, + }, + Delete { pos: 255, len: 255 }, + Delete { pos: 0, len: 102 }, + Insert { + pos: 102, + content: 102, + }, + Insert { + pos: 102, + content: 102, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { pos: 0, content: 0 }, + Delete { pos: 255, len: 255 }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 0, + content: 255, + }, + Delete { pos: 111, len: 108 }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 102, + content: 102, + }, + Insert { + pos: 83, + content: 108, + }, + Insert { + pos: 111, + content: 111, + }, + Insert { + pos: 119, + content: 21, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 55, + content: 119, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 121, + content: 86, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Delete { pos: 130, len: 130 }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 96, + content: 102, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 156, + content: 111, + }, + Insert { + pos: 123, + content: 37, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 37, + content: 121, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 121, + content: 86, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 123, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Delete { pos: 239, len: 239 }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 0, + }, + Delete { pos: 255, len: 255 }, + Delete { pos: 255, len: 255 }, + Delete { pos: 255, len: 255 }, + Delete { pos: 255, len: 255 }, + Delete { pos: 255, len: 255 }, + Delete { pos: 255, len: 255 }, + Delete { pos: 255, len: 255 }, + Delete { pos: 125, len: 125 }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 118, + content: 118, + }, + Insert { + pos: 255, + content: 255, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 102, + content: 102, + }, + Delete { pos: 209, len: 255 }, + Delete { pos: 255, len: 255 }, + ]) + } + + #[test] + fn from_str() { + for i in 0..100000 { + let s = i.to_string(); + let mut g = GapBuffer::from_str(&s); + assert_eq!(s.len(), g.next().unwrap().len()); + } + } + + #[test] + fn from_iter() { + let mut v = vec![]; + for i in 0..100000 { + v.push(i.to_string()); + } + + let rope = Rope { + tree: v + .iter() + .flat_map(|x| GapBuffer::from_str(x.as_str())) + .collect(), + cursor: None, + }; + + let s = v.join(""); + assert_eq!(rope.to_string(), s); + assert_eq!(rope.len(), s.len()); + rope.tree.check(); + } + + #[test] + fn drain() { + let mut rope = Rope::new(); + for i in 0..100000 { + rope.insert(0, &i.to_string()); + } + + while !rope.is_empty() { + let leaf = rope.tree.first_leaf(); + rope.tree.update_leaf(leaf.unwrap(), |elem| { + elem.slice_(1..1); + (true, None, None) + }); + } + } + + #[test] + fn fuzz_empty() { + fuzz(vec![]) + } +} diff --git a/3rdparty/loro/crates/generic-btree/src/iter.rs b/3rdparty/loro/crates/generic-btree/src/iter.rs new file mode 100644 index 0000000000..e8314a93de --- /dev/null +++ b/3rdparty/loro/crates/generic-btree/src/iter.rs @@ -0,0 +1,322 @@ +use crate::{ + delete_range, rle::HasLength, ArenaIndex, BTree, BTreeTrait, Cursor, LeafNode, NodePath, + QueryResult, +}; + +/// iterate node (not element) from the start path to the **inclusive** end path +pub(super) struct Iter<'a, B: BTreeTrait> { + tree: &'a BTree, + inclusive_end: NodePath, + path: NodePath, + done: bool, +} + +struct TempStore { + start_path: NodePath, + end_path: NodePath, + leaf_before_drain_range: Option, + leaf_after_drain_range: Option, +} + +pub struct Drain<'a, B: BTreeTrait> { + tree: &'a mut BTree, + current_path: NodePath, + done: bool, + end_cursor: Option, + store: Option>, +} + +impl<'a, B: BTreeTrait> Drain<'a, B> { + pub fn new( + tree: &'a mut BTree, + start_result: Option, + end_result: Option, + ) -> Self { + if start_result.is_none() || end_result.is_none() { + return Self::none(tree); + } + + let start_result = start_result.unwrap(); + let end_result = end_result.unwrap(); + let end_result = tree.split_leaf_if_needed(end_result.cursor).new_pos; + let Some(start_result) = tree.split_leaf_if_needed(start_result.cursor).new_pos else { + // if start from the right most leaf, the range is empty + return Self::none(tree); + }; + let start_path = tree.get_path(start_result.leaf.into()); + let end_path = tree.get_path( + end_result + .map(|x| x.leaf.into()) + .unwrap_or_else(|| tree.last_leaf().unwrap().into()), + ); + let leaf_before_drain_range = { + let node_idx = start_path.last().unwrap().arena; + if start_result.offset == 0 { + tree.prev_same_level_in_node(node_idx) + } else { + Some(node_idx) + } + }; + let leaf_after_drain_range = { + let node_idx = end_path.last().unwrap().arena; + if let Some(end) = end_result { + let len = tree.leaf_nodes.get(end.leaf.0).unwrap().elem.rle_len(); + if len == end.offset { + tree.next_same_level_in_node(node_idx) + } else { + Some(node_idx) + } + } else { + None + } + }; + Self { + current_path: tree.get_path(start_result.leaf.into()), + tree, + done: false, + end_cursor: end_result, + store: Some(Box::new(TempStore { + start_path, + end_path, + leaf_before_drain_range, + leaf_after_drain_range, + })), + } + } + + fn none(tree: &'a mut BTree) -> Drain { + Self { + current_path: Default::default(), + done: true, + end_cursor: None, + tree, + store: None, + } + } +} + +impl<'a, B: BTreeTrait> Iterator for Drain<'a, B> { + type Item = B::Elem; + + fn next(&mut self) -> Option { + if self.done { + return None; + } + + // end iteration if pointing to the end leaf + if let Some(end_cursor) = self.end_cursor { + if end_cursor.leaf.0 == self.current_path.last().unwrap().arena.unwrap_leaf() { + return None; + } + } + + let idx = *self.current_path.last().unwrap(); + if !self.tree.next_sibling(&mut self.current_path) { + self.done = true; + } + + // NOTE: we removed the node here, the tree is in an invalid state + let node = self + .tree + .leaf_nodes + .remove(idx.arena.unwrap_leaf()) + .unwrap(); + Some(node.elem) + } +} + +impl<'a, B: BTreeTrait> Drain<'a, B> { + fn ensure_finished(&mut self) { + while self.next().is_some() {} + } +} + +impl<'a, B: BTreeTrait> Drop for Drain<'a, B> { + fn drop(&mut self) { + self.ensure_finished(); + let TempStore { + start_path, + end_path, + leaf_before_drain_range, + leaf_after_drain_range, + } = *self.store.take().unwrap(); + // the deepest internal node level + let mut level = start_path.len() - 2; + let mut deleted = Vec::new(); + + // The deepest internal node level, need to filter deleted children + // to ensure is_empty() has correct result + self.tree.filter_deleted_children(start_path[level].arena); + self.tree.filter_deleted_children(end_path[level].arena); + while start_path[level].arena != end_path[level].arena { + let start_node = self.tree.get_internal(start_path[level].arena); + let end_node = self.tree.get_internal(end_path[level].arena); + let del_start = if start_node.is_empty() { + start_path[level].arr + } else { + start_path[level].arr + 1 + }; + let del_end = if end_node.is_empty() { + end_path[level].arr + 1 + } else { + end_path[level].arr + }; + + // remove del_start.. in start_node's parent + // remove ..del_end in end_node's parent + let start_arena = start_path[level - 1].arena; + let end_arena = end_path[level - 1].arena; + if start_arena == end_arena { + // parent is the same, delete start..end + let parent = self.tree.get_internal_mut(start_arena); + for x in &parent.children[del_start as usize..del_end as usize] { + deleted.push(x.arena); + } + + delete_range(&mut parent.children, del_start as usize..del_end as usize); + self.tree + .update_children_parent_slot_from(start_arena, del_start as usize); + } else { + // parent is different + { + // delete start.. + let start_parent = self.tree.get_internal_mut(start_arena); + for x in &start_parent.children[del_start as usize..] { + deleted.push(x.arena); + } + delete_range(&mut start_parent.children, del_start as usize..); + } + { + // delete ..end + let end_parent = self.tree.get_internal_mut(end_arena); + for x in &end_parent.children[..del_end as usize] { + deleted.push(x.arena); + } + delete_range(&mut end_parent.children, ..del_end as usize); + self.tree.update_children_parent_slot_from(end_arena, 0); + } + } + + level -= 1 + // this loop will abort before overflow, because level=0 is guaranteed to be the same + } + + while level >= 1 { + let (child, parent) = self + .tree + .get2_mut(start_path[level].arena, start_path[level - 1].arena); + if child.is_empty() { + assert_eq!( + parent.children[start_path[level].arr as usize].arena, + start_path[level].arena + ); + deleted.push(parent.children.remove(start_path[level].arr as usize).arena); + self.tree.update_children_parent_slot_from( + start_path[level - 1].arena, + start_path[level].arr as usize, + ); + } else { + break; + } + level -= 1; + } + + // release memory + for x in deleted { + self.tree.purge(x); + } + + if let Some(after) = leaf_after_drain_range { + self.tree.recursive_update_cache( + after, + leaf_after_drain_range == leaf_before_drain_range, + None, + ); + } + + // otherwise the path is invalid (e.g. the tree is empty) + if let Some(before) = leaf_before_drain_range { + if leaf_before_drain_range == leaf_after_drain_range { + self.tree.recursive_update_cache(before, B::USE_DIFF, None); + } else { + self.tree.recursive_update_cache(before, false, None); + if let Some(after) = leaf_after_drain_range { + self.tree.recursive_update_cache(after, false, None); + } + } + seal(self.tree, before); + } else { + self.tree.update_root_cache(); + self.tree.try_reduce_levels(); + } + } +} + +fn seal(tree: &mut BTree, leaf: ArenaIndex) { + handle_lack_on_path_to_leaf(tree, leaf); + if let Some(sibling) = tree.next_same_level_in_node(leaf) { + handle_lack_on_path_to_leaf(tree, sibling); + } + tree.try_reduce_levels(); +} + +fn handle_lack_on_path_to_leaf(tree: &mut BTree, leaf: ArenaIndex) { + let mut last_lack_count = 0; + let mut lack_count; + loop { + lack_count = 0; + let path = tree.get_path(leaf); + for i in 1..path.len() - 1 { + let Some(node) = tree.in_nodes.get(path[i].arena.unwrap_internal()) else { + unreachable!() + }; + let is_lack = node.is_lack(); + if is_lack { + let lack_info = tree.handle_lack_single_layer(path[i].arena); + if lack_info.parent_lack.is_some() { + lack_count += 1; + } + } + } + // parent may be lack after some children is merged + if lack_count == 0 || lack_count == last_lack_count { + break; + } + + last_lack_count = lack_count; + } +} + +impl<'a, B: BTreeTrait> Iter<'a, B> { + pub fn new(tree: &'a BTree, start: NodePath, inclusive_end: NodePath) -> Self { + Self { + tree, + inclusive_end, + path: start, + done: false, + } + } +} + +impl<'a, B: BTreeTrait> Iterator for Iter<'a, B> { + type Item = (NodePath, &'a LeafNode); + + fn next(&mut self) -> Option { + if self.done { + return None; + } + + if self.inclusive_end.last() == self.path.last() { + self.done = true; + } + + let last = *self.path.last().unwrap(); + let path = self.path.clone(); + if !self.tree.next_sibling(&mut self.path) { + self.done = true; + } + + let node = self.tree.leaf_nodes.get(last.arena.unwrap_leaf()).unwrap(); + Some((path, node)) + } +} diff --git a/3rdparty/loro/crates/generic-btree/src/lib.rs b/3rdparty/loro/crates/generic-btree/src/lib.rs new file mode 100644 index 0000000000..66c05303f4 --- /dev/null +++ b/3rdparty/loro/crates/generic-btree/src/lib.rs @@ -0,0 +1,3016 @@ +#![doc = include_str!("../README.md")] +#![forbid(unsafe_code)] + +use core::{fmt::Debug, ops::Range}; +use std::collections::{BTreeSet, VecDeque}; +use std::ops::AddAssign; +use std::{cmp::Ordering, mem::take, ops::RangeBounds}; + +pub(crate) use heapless::Vec as HeaplessVec; +use itertools::Itertools; +use rle::{CanRemove, TryInsert}; +use rustc_hash::{FxHashMap, FxHashSet}; +use thunderdome::Arena; +use thunderdome::Index as RawArenaIndex; + +pub use generic_impl::*; + +use crate::rle::{HasLength, Mergeable, Sliceable}; + +mod generic_impl; +pub mod iter; + +pub mod rle; + +pub type HeapVec = Vec; + +const MAX_CHILDREN_NUM: usize = 12; + +/// `Elem` should has length. `offset` in search result should always >= `Elem.rle_len()` +pub trait BTreeTrait { + /// Sometime an [Elem] with length of 0, but it's not empty. + /// + /// The empty [Elem]s are the ones that can be safely ignored. + type Elem: Debug + HasLength + Sliceable + Mergeable + TryInsert + CanRemove; + type Cache: Debug + Default + Clone + Eq; + type CacheDiff: Debug + Default + CanRemove; + // Whether we should use cache diff by default + const USE_DIFF: bool = true; + + /// If diff.is_some, return value should be some too + fn calc_cache_internal(cache: &mut Self::Cache, caches: &[Child]) -> Self::CacheDiff; + fn apply_cache_diff(cache: &mut Self::Cache, diff: &Self::CacheDiff); + fn merge_cache_diff(diff1: &mut Self::CacheDiff, diff2: &Self::CacheDiff); + fn get_elem_cache(elem: &Self::Elem) -> Self::Cache; + fn new_cache_to_diff(cache: &Self::Cache) -> Self::CacheDiff; + fn sub_cache(cache_lhs: &Self::Cache, cache_rhs: &Self::Cache) -> Self::CacheDiff; +} + +pub trait Query { + type QueryArg: Clone; + + fn init(target: &Self::QueryArg) -> Self; + + fn find_node(&mut self, target: &Self::QueryArg, child_caches: &[Child]) -> FindResult; + + /// Confirm the search result and returns (offset, found) + /// + /// If elem is not target, `found=false` + fn confirm_elem(&mut self, q: &Self::QueryArg, elem: &B::Elem) -> (usize, bool); +} + +pub struct BTree { + /// internal nodes + in_nodes: Arena>, + /// leaf nodes + leaf_nodes: Arena>, + /// root is always a internal node + /// TODO: we may use a constant as root index + root: ArenaIndex, + root_cache: B::Cache, +} + +impl> Clone for BTree { + fn clone(&self) -> Self { + Self { + in_nodes: self.in_nodes.clone(), + leaf_nodes: self.leaf_nodes.clone(), + root: self.root, + root_cache: self.root_cache.clone(), + } + } +} + +pub struct FindResult { + pub index: usize, + pub offset: usize, + pub found: bool, +} + +impl FindResult { + pub fn new_found(index: usize, offset: usize) -> Self { + Self { + index, + offset, + found: true, + } + } + + pub fn new_missing(index: usize, offset: usize) -> Self { + Self { + index, + offset, + found: false, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct Idx { + pub arena: ArenaIndex, + pub arr: u8, +} + +impl Idx { + pub fn new(arena: ArenaIndex, arr: u8) -> Self { + Self { arena, arr } + } +} + +type NodePath = HeaplessVec; + +#[derive(Debug, Clone, PartialEq, Eq, Copy, Hash)] +pub struct Cursor { + pub leaf: LeafIndex, + pub offset: usize, +} + +#[derive(Debug, Clone, PartialEq, Eq, Copy)] +pub struct QueryResult { + pub cursor: Cursor, + pub found: bool, +} + +/// Exposed arena index +/// +/// Only exposed arena index of leaf node. +/// +/// +#[repr(transparent)] +#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, PartialOrd, Ord)] +pub struct LeafIndex(RawArenaIndex); + +impl LeafIndex { + pub fn inner(&self) -> RawArenaIndex { + self.0 + } +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)] +pub enum ArenaIndex { + Leaf(RawArenaIndex), + Internal(RawArenaIndex), +} + +impl ArenaIndex { + fn unwrap(self) -> RawArenaIndex { + match self { + ArenaIndex::Leaf(x) => x, + ArenaIndex::Internal(x) => x, + } + } + + pub fn unwrap_leaf(self) -> RawArenaIndex { + match self { + ArenaIndex::Leaf(x) => x, + ArenaIndex::Internal(_) => panic!("unwrap_leaf on internal node"), + } + } + + pub fn unwrap_internal(self) -> RawArenaIndex { + match self { + ArenaIndex::Leaf(_) => panic!("unwrap_internal on leaf node"), + ArenaIndex::Internal(x) => x, + } + } +} + +impl From for ArenaIndex { + fn from(value: LeafIndex) -> Self { + Self::Leaf(value.0) + } +} + +impl From for LeafIndex { + fn from(value: RawArenaIndex) -> Self { + Self(value) + } +} + +/// A slice of element +/// +/// - `start` is Some(start_offset) when it's first element of the given range. +/// - `end` is Some(end_offset) when it's last element of the given range. +#[derive(Debug)] +pub struct ElemSlice<'a, Elem> { + cursor: Cursor, + pub elem: &'a Elem, + pub start: Option, + pub end: Option, +} + +impl<'a, Elem> ElemSlice<'a, Elem> { + pub fn cursor(&self) -> &Cursor { + &self.cursor + } +} + +impl QueryResult { + pub fn elem<'b, Elem: Debug, B: BTreeTrait>( + &self, + tree: &'b BTree, + ) -> Option<&'b Elem> { + tree.leaf_nodes.get(self.cursor().leaf.0).map(|x| &x.elem) + } + + #[inline(always)] + pub fn cursor(&self) -> Cursor { + self.cursor + } + + #[inline(always)] + pub fn leaf(&self) -> LeafIndex { + self.cursor().leaf + } + + #[inline(always)] + pub fn offset(&self) -> usize { + self.cursor().offset + } + + #[inline(always)] + pub fn found(&self) -> bool { + self.found + } + + #[inline(always)] + pub fn arena(&self) -> RawArenaIndex { + self.cursor.leaf.0 + } +} + +#[derive(Debug, Clone)] +pub struct LeafNode { + elem: Elem, + parent: RawArenaIndex, +} + +impl LeafNode { + pub fn parent(&self) -> ArenaIndex { + ArenaIndex::Internal(self.parent) + } + + pub fn elem(&self) -> &T { + &self.elem + } +} + +impl LeafNode { + fn split(&mut self, offset: usize) -> Self { + let new_elem = self.elem.split(offset); + Self { + elem: new_elem, + parent: self.parent, + } + } +} + +pub struct Node { + parent: Option, + parent_slot: u8, + children: HeaplessVec, MAX_CHILDREN_NUM>, +} + +#[repr(transparent)] +#[derive(Debug, Default, Clone)] +pub struct SplittedLeaves { + pub arr: HeaplessVec, +} + +impl SplittedLeaves { + #[inline] + fn push_option(&mut self, leaf: Option) { + if let Some(leaf) = leaf { + self.arr.push(leaf.unwrap().into()).unwrap(); + } + } + + #[inline] + fn push(&mut self, leaf: ArenaIndex) { + self.arr.push(leaf.unwrap().into()).unwrap(); + } +} + +impl> Debug for BTree { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + fn fmt_node>( + tree: &BTree, + node_idx: ArenaIndex, + f: &mut core::fmt::Formatter<'_>, + indent_size: usize, + ) -> core::fmt::Result { + match node_idx { + ArenaIndex::Leaf(_) => {} + ArenaIndex::Internal(_) => { + let node = tree.get_internal(node_idx); + for child in node.children.iter() { + indent(f, indent_size)?; + if child.is_internal() { + let child_node = tree.get_internal(child.arena); + f.write_fmt(format_args!( + "{} Arena({:?}) Cache: {:?}\n", + child_node.parent_slot, &child.arena, &child.cache + ))?; + fmt_node::(tree, child.arena, f, indent_size + 1)?; + } else { + let node = tree.get_leaf(child.arena); + f.write_fmt(format_args!( + "Leaf({:?}) Arena({:?}) Parent({:?}) Cache: {:?}\n", + &node.elem, child.arena, node.parent, &child.cache + ))?; + } + } + } + } + + Ok(()) + } + + fn indent(f: &mut core::fmt::Formatter<'_>, indent: usize) -> core::fmt::Result { + for _ in 0..indent { + f.write_str(" ")?; + } + Ok(()) + } + + f.write_str("BTree\n")?; + indent(f, 1)?; + f.write_fmt(format_args!( + "Root Arena({:?}) Cache: {:?}\n", + &self.root, &self.root_cache + ))?; + fmt_node::(self, self.root, f, 1) + } +} + +impl> Debug for Node { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("Node") + .field("children", &self.children) + .finish() + } +} + +impl> Debug for Child { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("Child") + .field("index", &self.arena) + .field("cache", &self.cache) + .finish() + } +} + +impl> Clone for Node { + fn clone(&self) -> Self { + Self { + parent: self.parent, + parent_slot: self.parent_slot, + children: self.children.clone(), + } + } +} + +pub struct Child { + pub arena: ArenaIndex, + pub cache: B::Cache, +} + +impl Child { + #[inline] + fn is_internal(&self) -> bool { + matches!(self.arena, ArenaIndex::Internal(_)) + } + + #[inline] + #[allow(unused)] + fn is_leaf(&self) -> bool { + matches!(self.arena, ArenaIndex::Leaf(_)) + } +} + +impl Clone for Child { + fn clone(&self) -> Self { + Self { + arena: self.arena, + cache: self.cache.clone(), + } + } +} + +impl Child { + pub fn cache(&self) -> &B::Cache { + &self.cache + } + + fn new(arena: ArenaIndex, cache: B::Cache) -> Self { + Self { arena, cache } + } +} + +impl Node { + #[inline(always)] + pub fn new() -> Self { + Self { + parent: None, + parent_slot: u8::MAX, + children: HeaplessVec::new(), + } + } + + #[inline(always)] + pub fn is_full(&self) -> bool { + self.children.len() >= MAX_CHILDREN_NUM + } + + #[inline(always)] + pub fn is_lack(&self) -> bool { + self.children.len() < MAX_CHILDREN_NUM / 2 + } + + #[inline(always)] + pub fn len(&self) -> usize { + self.children.len() + } + + #[inline(always)] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + #[inline(always)] + pub fn is_child_leaf(&self) -> bool { + if self.children.is_empty() { + return true; + } + + self.children[0].is_leaf() + } + + /// if diff is not provided, the cache will be calculated from scratch + #[inline(always)] + fn calc_cache(&self, cache: &mut B::Cache, diff: Option) -> B::CacheDiff { + match diff { + Some(inner) => { + B::apply_cache_diff(cache, &inner); + inner + } + None => B::calc_cache_internal(cache, &self.children), + } + } +} + +impl Default for Node { + fn default() -> Self { + Self::new() + } +} + +type LeafDirtyMap = FxHashMap; + +/// Whether the parent node is lack of children +#[repr(transparent)] +struct LackInfo { + /// if Some, the parent node is lack + parent_lack: Option, +} + +impl BTree { + pub fn new() -> Self { + let mut arena = Arena::new(); + let root = arena.insert(Node::new()); + Self { + in_nodes: arena, + leaf_nodes: Arena::new(), + root: ArenaIndex::Internal(root), + root_cache: B::Cache::default(), + } + } + + /// Get the number of nodes in the tree. + /// It includes all the internal nodes and the leaf nodes. + #[inline(always)] + pub fn node_len(&self) -> usize { + self.in_nodes.len() + self.leaf_nodes.len() + } + + /// Insert new element to the tree. + /// + /// Returns (insert_pos, splitted_leaves) + pub fn insert(&mut self, q: &Q::QueryArg, data: B::Elem) -> (Cursor, SplittedLeaves) + where + Q: Query, + { + let Some(result) = self.query::(q) else { + return (self.push(data), Default::default()); + }; + + self.insert_by_path(result.cursor, data) + } + + pub fn insert_by_path(&mut self, cursor: Cursor, data: B::Elem) -> (Cursor, SplittedLeaves) { + let index = cursor.leaf; + let leaf = self.leaf_nodes.get_mut(index.0).unwrap(); + let mut parent_idx = leaf.parent(); + let cache_diff = if B::USE_DIFF { + Some(B::new_cache_to_diff(&B::get_elem_cache(&data))) + } else { + None + }; + + let mut is_full = false; + let mut splitted: SplittedLeaves = Default::default(); + let ans = match leaf.elem.try_insert(cursor.offset, data) { + Ok(_) => cursor, + Err(data) => { + // Try to merge + if cursor.offset == 0 && data.can_merge(&leaf.elem) { + leaf.elem.merge_left(&data); + Cursor { + leaf: index, + offset: 0, + } + } else if cursor.offset == leaf.elem.rle_len() && leaf.elem.can_merge(&data) { + let offset = leaf.elem.rle_len(); + leaf.elem.merge_right(&data); + Cursor { + leaf: index, + offset, + } + } else { + // Insert new leaf node + let SplitInfo { + parent_idx: parent_index, + insert_slot: insert_index, + new_leaf, + .. + } = self.split_leaf_if_needed(cursor); + parent_idx = ArenaIndex::Internal(parent_index); + let child = self.alloc_leaf_child(data, parent_index); + let ans = child.arena; + splitted.push_option(new_leaf); + let parent = self.in_nodes.get_mut(parent_index).unwrap(); + parent.children.insert(insert_index, child).unwrap(); + is_full = parent.is_full(); + Cursor { + leaf: ans.unwrap().into(), + offset: 0, + } + } + } + }; + + self.recursive_update_cache(cursor.leaf.into(), B::USE_DIFF, cache_diff); + if is_full { + self.split(parent_idx); + } + + (ans, splitted) + } + + fn alloc_leaf_child( + &mut self, + data: ::Elem, + parent_index: RawArenaIndex, + ) -> Child { + let elem_cache = B::get_elem_cache(&data); + let new_leaf_index = self.alloc_new_leaf(LeafNode { + elem: data, + parent: parent_index, + }); + Child { + arena: new_leaf_index, + cache: elem_cache, + } + } + + /// Split a leaf node at offset if it's not the start/end of the leaf node. + /// + /// This method should be called when inserting at target pos. + fn split_leaf_if_needed(&mut self, pos: Cursor) -> SplitInfo { + let leaf = self.leaf_nodes.get_mut(pos.leaf.0).unwrap(); + let parent_idx = leaf.parent; + let parent = self.in_nodes.get_mut(leaf.parent).unwrap(); + let mut new_pos = Some(pos); + let mut rt_new_leaf = None; + let leaf_slot = parent + .children + .iter() + .position(|x| x.arena.unwrap() == pos.leaf.0) + .unwrap(); + let left_neighbour = if leaf_slot == 0 { + None + } else { + Some(parent.children[leaf_slot - 1].arena.unwrap().into()) + }; + let insert_pos = if pos.offset == 0 { + leaf_slot + } else if pos.offset == leaf.elem.rle_len() { + if leaf_slot + 1 < parent.children.len() { + new_pos = Some(Cursor { + leaf: parent.children[leaf_slot + 1].arena.unwrap().into(), + offset: 0, + }); + } else { + new_pos = self.next_elem(pos); + } + leaf_slot + 1 + } else { + assert!( + pos.offset < leaf.elem.rle_len(), + "elem.rle_len={} but pos.offset={} Elem:{:?}", + leaf.elem.rle_len(), + pos.offset, + &leaf.elem + ); + + if parent.children.len() + 1 >= MAX_CHILDREN_NUM { + self.split(ArenaIndex::Internal(parent_idx)); + // parent may be changed because of splitting + return self.split_leaf_if_needed(pos); + } + + let new_leaf = leaf.split(pos.offset); + let left_cache = B::get_elem_cache(&leaf.elem); + let cache = B::get_elem_cache(&new_leaf.elem); + // alloc new leaf node + let leaf_arena_index = { + let arena_index = self.leaf_nodes.insert(new_leaf); + ArenaIndex::Leaf(arena_index) + }; + rt_new_leaf = Some(leaf_arena_index); + new_pos = Some(Cursor { + leaf: leaf_arena_index.unwrap().into(), + offset: 0, + }); + parent.children[leaf_slot].cache = left_cache; + parent + .children + .insert( + leaf_slot + 1, + Child { + arena: leaf_arena_index, + cache, + }, + ) + .unwrap(); + + leaf_slot + 1 + }; + + SplitInfo { + left_neighbour, + new_pos, + parent_idx, + insert_slot: insert_pos, + new_leaf: rt_new_leaf, + } + } + + fn alloc_new_leaf(&mut self, leaf: LeafNode) -> ArenaIndex { + let arena_index = self.leaf_nodes.insert(leaf); + ArenaIndex::Leaf(arena_index) + } + + /// Insert many elements into the tree at once + /// + /// It will invoke [`BTreeTrait::insert_batch`] + pub fn insert_many_by_cursor( + &mut self, + cursor: Option, + mut data_iter: impl Iterator, + ) { + let Some(first) = data_iter.next() else { + return; + }; + + let Some(second) = data_iter.next() else { + if let Some(c) = cursor { + self.insert_by_path(c, first); + return; + } else { + self.push(first); + return; + } + }; + + let mut data = Vec::with_capacity(data_iter.size_hint().0 + 2); + data.push(first); + data.push(second); + for elem in data_iter { + data.push(elem); + } + + merge_adj(&mut data); + if data.len() == 1 { + if let Some(c) = cursor { + self.insert_by_path(c, data.pop().unwrap()); + return; + } else { + self.push(data.pop().unwrap()); + return; + } + } + + if cursor.is_none() && self.is_empty() { + assert!(self.is_empty()); + let (new_root, _) = self.create_subtrees_from_elem(data); + self.in_nodes.remove(self.root.unwrap()).unwrap(); + self.root = new_root; + return; + } + + // dbg!(cursor, &data); + // dbg!(&self); + let cursor = cursor.expect("Cursor must be provided when tree is not empty"); + let SplitInfo { + new_pos, + left_neighbour, + .. + } = self.split_leaf_if_needed(cursor); + let mut inserted = 0; + if let Some(left) = left_neighbour { + let left_node = self.leaf_nodes.get_mut(left.0).unwrap(); + let mut i = 0; + while i < data.len() && left_node.elem.can_merge(&data[i]) { + left_node.elem.merge_right(&data[i]); + i += 1; + } + + self.recursive_update_cache(left.into(), B::USE_DIFF, None); + inserted = i; + } + + let mut pos = new_pos.unwrap_or(cursor); + // TODO: PERF this can be optimized further + for item in data.drain(inserted..).rev() { + let (p, _) = self.insert_by_path(pos, item); + pos = p + } + } + + /// The returned height starts from 0. Leaf level is 0. + /// + /// Returns (newly created subtree's root, height) + fn create_subtrees_from_elem(&mut self, data: Vec) -> (ArenaIndex, usize) { + let mut height = 0; + let mut nodes = Vec::with_capacity(data.len() / MAX_CHILDREN_NUM + 1); + for elem in data.into_iter().chunks(MAX_CHILDREN_NUM).into_iter() { + let parent_index = self.in_nodes.insert(Node { + parent: None, + parent_slot: 0, + children: Default::default(), + }); + + nodes.push(parent_index); + let parent = self.in_nodes.get_mut(parent_index).unwrap(); + for (i, elem) in elem.enumerate() { + let leaf = { + // alloc new leaf child + let elem_cache = B::get_elem_cache(&elem); + let new_leaf_index = { + let leaf = LeafNode { + elem, + parent: parent_index, + }; + let arena_index = self.leaf_nodes.insert(leaf); + ArenaIndex::Leaf(arena_index) + }; + Child { + arena: new_leaf_index, + cache: elem_cache, + } + }; + parent.children[i] = leaf; + } + } + + while nodes.len() > 1 { + let mut new_nodes = Vec::with_capacity(nodes.len() / MAX_CHILDREN_NUM + 1); + for chunk in nodes.into_iter().chunks(MAX_CHILDREN_NUM).into_iter() { + let parent_index = self.in_nodes.insert(Node { + parent: None, + parent_slot: 0, + children: Default::default(), + }); + + new_nodes.push(parent_index); + for (i, child_idx) in chunk.enumerate() { + let (parent, child) = self.in_nodes.get2_mut(parent_index, child_idx); + let parent = parent.unwrap(); + let child = child.unwrap(); + let mut cache = B::Cache::default(); + B::calc_cache_internal(&mut cache, &child.children); + parent.children[i] = Child { + arena: ArenaIndex::Internal(child_idx), + cache, + }; + child.parent = Some(ArenaIndex::Internal(parent_index)); + child.parent_slot = i as u8; + } + } + nodes = new_nodes; + height += 1; + } + + (ArenaIndex::Internal(nodes[0]), height) + } + + /// Shift by offset 1. + /// + /// It will not stay on empty spans but scan forward + pub fn shift_path_by_one_offset(&self, mut path: Cursor) -> Option + where + B::Elem: rle::HasLength, + { + let leaf = self.leaf_nodes.get(path.leaf.0).unwrap(); + if path.offset + 1 < leaf.elem.rle_len() { + path.offset += 1; + return Some(path); + } + + let mut parent_idx = leaf.parent; + let mut parent = self.in_nodes.get(leaf.parent).unwrap(); + let mut elem_slot_index = Self::get_leaf_slot(path.leaf.0, parent); + path.offset += 1; + loop { + if elem_slot_index == parent.children.len() { + if let Some(next) = self.next_same_level_in_node(ArenaIndex::Internal(parent_idx)) { + elem_slot_index = 0; + path.offset = 0; + parent_idx = next.unwrap_internal(); + parent = self.in_nodes.get(parent_idx).unwrap(); + } else { + return None; + } + } + + let elem = &parent.children[elem_slot_index]; + let leaf = self.leaf_nodes.get(elem.arena.unwrap()).unwrap(); + // skip empty span + if leaf.elem.rle_len() <= path.offset { + path.offset -= leaf.elem.rle_len(); + elem_slot_index += 1; + } else { + path.leaf = elem.arena.unwrap_leaf().into(); + break; + } + } + + Some(path) + } + + fn get_leaf_slot(leaf_arena_index: RawArenaIndex, parent: &Node) -> usize { + parent + .children + .iter() + .position(|x| x.arena.unwrap_leaf() == leaf_arena_index) + .unwrap() + } + + /// Query the tree by custom query type + /// + /// Return None if the tree is empty + pub fn query(&self, query: &Q::QueryArg) -> Option + where + Q: Query, + { + self.query_with_finder_return::(query).0 + } + + pub fn query_with_finder_return(&self, query: &Q::QueryArg) -> (Option, Q) + where + Q: Query, + { + let mut finder = Q::init(query); + if self.is_empty() { + return (None, finder); + } + + let mut node = self.in_nodes.get(self.root.unwrap()).unwrap(); + let mut index; + let mut found = true; + loop { + let result = finder.find_node(query, &node.children); + debug_assert!(!node.children.is_empty()); + let i = result.index; + found = found && result.found; + index = node.children[i].arena; + match index { + ArenaIndex::Internal(index) => { + node = self.in_nodes.get(index).unwrap(); + } + ArenaIndex::Leaf(_) => { + let (offset, leaf_found) = finder.confirm_elem( + query, + &self.leaf_nodes.get(index.unwrap_leaf()).unwrap().elem, + ); + return ( + Some(QueryResult { + cursor: Cursor { + leaf: index.unwrap_leaf().into(), + offset, + }, + found: found && leaf_found, + }), + finder, + ); + } + } + } + } + + pub fn get_elem_mut(&mut self, leaf: LeafIndex) -> Option<&mut B::Elem> { + let node = self.leaf_nodes.get_mut(leaf.0)?; + Some(&mut node.elem) + } + + pub fn get_elem(&self, leaf: LeafIndex) -> Option<&::Elem> { + self.leaf_nodes.get(leaf.0).map(|x| &x.elem) + } + + /// Remove leaf node from the tree + /// + /// If it's already removed, this method will return None + pub fn remove_leaf(&mut self, path: Cursor) -> Option { + let leaf = self.leaf_nodes.get_mut(path.leaf.0)?; + let parent_idx = leaf.parent(); + let parent = self.in_nodes.get_mut(leaf.parent).unwrap(); + let index = Self::get_leaf_slot(path.leaf.0, parent); + let child = parent.children.remove(index); + let is_lack = parent.is_lack(); + let is_empty = parent.is_empty(); + debug_assert_eq!(child.arena.unwrap(), path.leaf.0); + let elem = self.leaf_nodes.remove(child.arena.unwrap()).unwrap().elem; + + self.recursive_update_cache(parent_idx, B::USE_DIFF, None); + if is_empty { + self.remove_internal_node(parent_idx.unwrap()); + } else if is_lack { + self.handle_lack_recursively(parent_idx); + } + + Some(elem) + } + + fn remove_internal_node(&mut self, node: RawArenaIndex) { + if node == self.root.unwrap() { + return; + } + + let node = self.in_nodes.remove(node).unwrap(); + if let Some(parent_idx) = node.parent { + let parent = self.in_nodes.get_mut(parent_idx.unwrap_internal()).unwrap(); + parent.children.remove(node.parent_slot as usize); + let is_lack = parent.is_lack(); + let is_empty = parent.is_empty(); + self.update_children_parent_slot_from(parent_idx, node.parent_slot as usize); + if is_empty { + self.remove_internal_node(parent_idx.unwrap_internal()); + } else if is_lack { + self.handle_lack_recursively(parent_idx); + } + } else { + // ignore remove root + unreachable!() + } + } + + /// Update the elements in place. + /// + /// If the range.start or range.end is in the middle of a leaf node, the leaf node + /// will be splitted into two leaf nodes. The new leaf nodes will be returned. + /// + /// F should returns `Some(cache_diff)` if cache needs to be updated. Otherwise, returns None. + /// + /// If the given range has zero length, f will still be called, and the slice will + /// have same `start` and `end` field + /// + /// TODO: need better test coverage + pub fn update(&mut self, range: Range, f: &mut F) -> SplittedLeaves + where + F: FnMut(&mut B::Elem) -> Option, + { + let mut splitted = SplittedLeaves::default(); + let start = range.start; + let SplitInfo { + new_pos: end, + new_leaf, + .. + } = self.split_leaf_if_needed(range.end); + splitted.push_option(new_leaf); + let SplitInfo { + new_pos: start, + new_leaf, + .. + } = self.split_leaf_if_needed(start); + splitted.push_option(new_leaf); + let Some(start) = start else { + return splitted; + }; + let start_leaf = start.leaf; + let mut path = self.get_path(start_leaf.into()); + let mut dirty_map: LeafDirtyMap = FxHashMap::default(); + let mut to_remove = Vec::default(); + + loop { + let current_leaf = path.last().unwrap(); + if let Some(end) = end { + if current_leaf.arena.unwrap_leaf() == end.leaf.0 { + break; + } + } + + let node = self + .leaf_nodes + .get_mut(current_leaf.arena.unwrap_leaf()) + .unwrap(); + let cache_diff = f(&mut node.elem); + if node.elem.can_remove() { + to_remove.push(current_leaf.arena); + } + + if let Some(diff) = cache_diff { + add_leaf_dirty_map(current_leaf.arena, &mut dirty_map, diff); + } + + if !self.next_sibling(&mut path) { + break; + } + } + + if !dirty_map.is_empty() { + self.update_dirty_cache_map(dirty_map); + } else { + self.in_nodes + .get(self.root.unwrap_internal()) + .unwrap() + .calc_cache(&mut self.root_cache, None); + } + + for leaf in to_remove { + self.remove_leaf(Cursor { + leaf: leaf.unwrap().into(), + offset: 0, + }); + } + splitted + } + + /// Prefer begin of the next leaf node than end of the current leaf node + /// + /// When path.offset == leaf.rle_len(), this method will return + /// the next leaf node with offset 0 + #[allow(unused)] + pub fn prefer_right(&self, path: Cursor) -> Option { + if path.offset == 0 { + return Some(path); + } + + let leaf = self.leaf_nodes.get(path.leaf.0).unwrap(); + if path.offset == leaf.elem.rle_len() { + self.next_elem(path) + } else { + Some(path) + } + } + + /// Prefer end of the previous leaf node than begin of the current leaf node + /// + /// When path.offset == 0, this method will return + /// the previous leaf node with offset leaf.rle_len() + #[allow(unused)] + pub fn prefer_left(&self, path: Cursor) -> Option { + if path.offset != 0 { + return Some(path); + } + + let elem = self.prev_elem(path); + if let Some(elem) = elem { + let leaf = self.leaf_nodes.get(elem.leaf.0).unwrap(); + Some(Cursor { + leaf: elem.leaf, + offset: leaf.elem.rle_len(), + }) + } else { + None + } + } + + /// Update leaf node's elements. + /// + /// `f` returns Option<(cache_diff, new_insert_1, new_insert2)> + /// + /// - If returned value is `None`, the cache will not be updated. + /// - If leaf_node.can_remove(), it will be removed from the tree. + /// + /// Returns (path, splitted_leaves), if is is still valid after this method. (If the leaf node is removed, the path will be None) + pub fn update_leaf_by_search>( + &mut self, + q: &Q::QueryArg, + f: impl FnOnce( + &mut B::Elem, + QueryResult, + ) -> Option<(B::CacheDiff, Option, Option)>, + ) -> (Option, SplittedLeaves) { + if self.is_empty() { + panic!("update_leaf_by_search called on empty tree"); + } + + let mut splitted = SplittedLeaves::default(); + let mut finder = Q::init(q); + let mut path = NodePath::default(); + let mut node_idx = self.root; + let mut child_arr_pos = 0; + while let ArenaIndex::Internal(node_idx_inner) = node_idx { + path.push(Idx { + arena: ArenaIndex::Internal(node_idx_inner), + arr: child_arr_pos, + }) + .unwrap(); + let node = self.in_nodes.get(node_idx_inner).unwrap(); + let result = finder.find_node(q, &node.children); + child_arr_pos = result.index as u8; + node_idx = node.children[result.index].arena; + } + + let leaf = self.get_leaf_mut(node_idx); + let (offset, found) = finder.confirm_elem(q, &leaf.elem); + let ans = QueryResult { + cursor: Cursor { + leaf: node_idx.unwrap_leaf().into(), + offset, + }, + found, + }; + let Some((diff, new_insert_1, new_insert_2)) = f(&mut leaf.elem, ans) else { + return (Some(ans.cursor), splitted); + }; + + if new_insert_2.is_some() { + unimplemented!() + } + + // Delete + if leaf.elem.can_remove() { + // handle deletion + // leaf node should be deleted + assert!(new_insert_1.is_none()); + assert!(new_insert_2.is_none()); + self.leaf_nodes.remove(node_idx.unwrap()).unwrap(); + let mut is_first = true; + let mut is_child_lack = false; + let mut child_idx = node_idx; + + // iterate from leaf to root, child to parent + while let Some(Idx { + arena: parent_idx, + arr: parent_arr_pos, + }) = path.pop() + { + let parent = self.get_internal_mut(parent_idx); + if is_first { + parent.children.remove(child_arr_pos as usize); + is_first = false; + } else { + B::apply_cache_diff(&mut parent.children[child_arr_pos as usize].cache, &diff); + } + + let is_lack = parent.is_lack(); + + if is_child_lack { + self.handle_lack_single_layer(child_idx); + } + + is_child_lack = is_lack; + child_idx = parent_idx; + child_arr_pos = parent_arr_pos; + } + + B::apply_cache_diff(&mut self.root_cache, &diff); + + if is_child_lack { + let root = self.get_internal_mut(self.root); + if root.children.len() == 1 && !root.is_child_leaf() { + self.try_reduce_levels(); + } + } + + return (None, splitted); + } + + let mut new_cache_and_child = None; + if let Some(new_insert_1) = new_insert_1 { + let cache = B::get_elem_cache(&leaf.elem); + let child = self.alloc_leaf_child(new_insert_1, path.last().unwrap().arena.unwrap()); + splitted.push(child.arena); + new_cache_and_child = Some((cache, child)); + } + + while let Some(Idx { + arena: parent_idx, + arr: parent_arr_pos, + }) = path.pop() + { + let parent = self.get_internal_mut(parent_idx); + match take(&mut new_cache_and_child) { + Some((cache, child)) => { + parent.children[child_arr_pos as usize].cache = cache; + parent + .children + .insert(child_arr_pos as usize + 1, child) + .unwrap(); + let is_full = parent.is_full(); + if !parent.is_child_leaf() { + self.update_children_parent_slot_from( + parent_idx, + child_arr_pos as usize + 1, + ); + } + if is_full { + let (_, _, this_cache, right_child) = self.split_node(parent_idx, None); + new_cache_and_child = Some((this_cache, right_child)); + } + } + None => { + B::apply_cache_diff(&mut parent.children[child_arr_pos as usize].cache, &diff); + } + } + + child_arr_pos = parent_arr_pos; + } + + if let Some((cache, child)) = new_cache_and_child { + self.split_root(cache, child); + } else { + B::apply_cache_diff(&mut self.root_cache, &diff); + } + + (Some(ans.cursor), splitted) + } + + /// Update leaf node's elements, return true if cache need to be updated + /// + /// `f` returns (is_cache_updated, cache_diff, new_insert_1, new_insert2) + /// + /// - If leaf_node.can_remove(), it will be removed from the tree. + /// + /// Returns true if the node_idx is still valid. (If the leaf node is removed, it will return false). + pub fn update_leaf( + &mut self, + node_idx: LeafIndex, + f: impl FnOnce(&mut B::Elem) -> (bool, Option, Option), + ) -> (bool, SplittedLeaves) { + let mut splitted = SplittedLeaves::default(); + let node = self.leaf_nodes.get_mut(node_idx.0).unwrap(); + let mut parent_idx = node.parent(); + let (need_update_cache, mut new_insert_1, mut new_insert_2) = f(&mut node.elem); + { + // Normalize returned values + // + // If the node can be removed, then both new_insert_1 & new_insert_2 should be None + // The priority is node.elem > new_insert_1 > new_insert_2 + // + // And new_insert_1 and new_insert_2 should not match `can_remove` condition + if let Some(ref new_1) = new_insert_1 { + if new_1.can_remove() { + new_insert_1 = new_insert_2.take(); + if let Some(ref new_1) = new_insert_1 { + if new_1.can_remove() { + new_insert_1 = None; + } + } + } + } + + if let Some(ref new_2) = new_insert_2 { + if new_2.can_remove() { + new_insert_2 = None; + } else if new_insert_1.is_none() { + std::mem::swap(&mut new_insert_1, &mut new_insert_2); + } + } + + if node.elem.can_remove() { + if let Some(new_1) = new_insert_1 { + node.elem = new_1; + new_insert_1 = new_insert_2.take(); + } + } + } + + let deleted = node.elem.can_remove(); + + if need_update_cache { + self.recursive_update_cache(node_idx.into(), B::USE_DIFF, None); + } + + if deleted { + debug_assert!(new_insert_1.is_none()); + debug_assert!(new_insert_2.is_none()); + self.leaf_nodes.remove(node_idx.0).unwrap(); + let parent = self.in_nodes.get_mut(parent_idx.unwrap()).unwrap(); + let slot = Self::get_leaf_slot(node_idx.0, parent); + parent.children.remove(slot); + let is_lack = parent.is_lack(); + if is_lack { + self.handle_lack_recursively(parent_idx); + } + + (false, splitted) + } else if new_insert_1.is_none() { + debug_assert!(new_insert_2.is_none()); + return (true, splitted); + } else { + if new_insert_1.is_some() && new_insert_2.is_none() { + // try merge new insert to next element + let parent = self.in_nodes.get_mut(parent_idx.unwrap()).unwrap(); + let slot = Self::get_leaf_slot(node_idx.0, parent); + if slot + 1 < parent.children.len() { + let next_idx = parent.children[slot + 1].arena.unwrap().into(); + let next = self.get_elem_mut(next_idx).unwrap(); + let new = new_insert_1.as_ref().unwrap(); + if new.can_merge(next) { + next.merge_left(new); + self.recursive_update_cache(next_idx.into(), B::USE_DIFF, None); + splitted.push(next_idx.into()); + return (true, splitted); + } + } + } + + let count = if new_insert_1.is_some() { 1 } else { 0 } + + if new_insert_2.is_some() { 1 } else { 0 }; + let parent = self.in_nodes.get_mut(parent_idx.unwrap()).unwrap(); + let parent = if parent.children.len() + count >= MAX_CHILDREN_NUM { + self.split(parent_idx); + let node = self.leaf_nodes.get(node_idx.0).unwrap(); + parent_idx = node.parent(); + self.in_nodes.get_mut(parent_idx.unwrap()).unwrap() + } else { + parent + }; + + let new: HeaplessVec<_, 2> = new_insert_1 + .into_iter() + .chain(new_insert_2) + .map(|elem| { + // Allocate new leaf node + let parent_index = parent_idx.unwrap(); + let elem_cache = B::get_elem_cache(&elem); + let new_leaf_index = { + let leaf = LeafNode { + elem, + parent: parent_index, + }; + let arena_index = self.leaf_nodes.insert(leaf); + ArenaIndex::Leaf(arena_index) + }; + Child { + arena: new_leaf_index, + cache: elem_cache, + } + }) + .collect(); + let slot = Self::get_leaf_slot(node_idx.0, parent); + for (i, v) in new.into_iter().enumerate() { + splitted.push(v.arena); + parent.children.insert(slot + 1 + i, v).unwrap(); + } + + assert!(!parent.is_full()); + self.recursive_update_cache(parent_idx, B::USE_DIFF, None); + (true, splitted) + } + } + + /// Update the given leaves with the given function in the given range. + /// + /// - The range descibes the range inside the leaf node. + /// - There can be multiple ranges in the same leaf node. + /// - The cahce will be recalculated for each affected node + /// - It doesn't guarantee the applying order + /// + /// Currently, the time complexity is O(m^2) for each leaf node, + /// where m is the number of ranges inside the same leaf node. + /// If we have a really large m, this function need to be optimized. + pub fn update_leaves_with_arg_in_ranges( + &mut self, + mut args: Vec<(LeafIndex, Range, A)>, + mut f: impl FnMut(&mut B::Elem, &A), + ) -> Vec { + args.sort_by_key(|x| x.0); + let mut new_leaves = Vec::new(); + let mut dirty_map: LeafDirtyMap = Default::default(); + let mut new_elems_at_cursor: FxHashMap> = Default::default(); + for (leaf, group) in &args.into_iter().group_by(|x| x.0) { + // This loop doesn't change the shape of the tree. It only changes each leaf element. + // A leaf element may be splitted into several parts. The first part stay in the tree, + // while the rest of them are inserted into `new_elem_at_cursor`, which will be inserted + // into the tree later. + let leaf_node = self.leaf_nodes.get_mut(leaf.0).unwrap(); + let len = leaf_node.elem().rle_len(); + let mut split_at = BTreeSet::new(); + // PERF we can avoid this alloc and `group_by` + let group: Vec<_> = group.into_iter().collect(); + for (_, range, _) in group.iter() { + split_at.insert(range.start); + split_at.insert(range.end); + } + + split_at.remove(&0); + split_at.remove(&len); + + // leaf_node.elem is the first elem + let old_cache = B::get_elem_cache(&leaf_node.elem); + if split_at.is_empty() { + // doesn't need to split + for (_, range, a) in group.iter() { + assert_eq!(range.start, 0); + assert_eq!(range.end, len); + f(&mut leaf_node.elem, a); + } + } else { + let mut new_elems = Vec::new(); + + let first_split = split_at.first().copied().unwrap(); + + // handle first element + let mut elem = leaf_node.elem.split(first_split); + for (_, r, a) in group.iter() { + if r.start == 0 { + f(&mut leaf_node.elem, a); + } + } + + // handle elements in the middle + let mut last_index = first_split; + for &index in split_at.iter().skip(1) { + let next_elem = elem.split(index - last_index); + let cur_range = last_index..index; + for (_, r, a) in group.iter() { + if r.start <= cur_range.start && cur_range.end <= r.end { + f(&mut elem, a); + } + } + + new_elems.push(elem); + elem = next_elem; + last_index = index; + } + + // handle the last element + for (_, r, a) in group.iter() { + if r.end == len { + f(&mut elem, a); + } + } + + new_elems.push(elem); + new_elems_at_cursor.insert( + Cursor { + leaf, + offset: leaf_node.elem().rle_len(), + }, + new_elems, + ); + } + + let new_cache = B::get_elem_cache(&leaf_node.elem); + let diff = B::sub_cache(&new_cache, &old_cache); + dirty_map.insert(leaf.into(), diff); + } + + // update cache + self.update_dirty_cache_map(dirty_map); + + // PERF we can use batch insert to optimize this + // insert the new leaf nodes + for (mut cursor, elems) in new_elems_at_cursor { + for elem in elems.into_iter() { + // PERF can use insert many to optimize when it's supported + let result = self.insert_by_path(cursor, elem); + let len = self.get_elem(result.0.leaf).unwrap().rle_len(); + new_leaves.push(result.0.leaf); + debug_assert_eq!(result.1.arr.len(), 0); + cursor = Cursor { + leaf: result.0.leaf, + offset: len, + }; + } + } + + new_leaves + } + + fn update_root_cache(&mut self) { + self.in_nodes + .get(self.root.unwrap_internal()) + .unwrap() + .calc_cache(&mut self.root_cache, None); + } + + fn update_dirty_cache_map(&mut self, mut diff_map: LeafDirtyMap) { + // diff_map only contains leaf nodes when this function is called + let mut visit_set: FxHashSet = diff_map.keys().copied().collect(); + while !visit_set.is_empty() { + for child_idx in take(&mut visit_set) { + let (parent_idx, cache_diff) = match child_idx { + ArenaIndex::Leaf(leaf_idx) => { + let node = self.leaf_nodes.get(leaf_idx).unwrap(); + let parent_idx = node.parent; + let parent = self.in_nodes.get_mut(parent_idx).unwrap(); + let cache_diff = diff_map.remove(&child_idx).unwrap(); + for child in parent.children.iter_mut() { + if child.arena == child_idx { + B::apply_cache_diff(&mut child.cache, &cache_diff); + break; + } + } + + (ArenaIndex::Internal(parent_idx), cache_diff) + } + ArenaIndex::Internal(_) => { + let node = self.in_nodes.get(child_idx.unwrap_internal()).unwrap(); + let Some(parent_idx) = node.parent else { + continue; + }; + let (child, parent) = self.get2_mut(child_idx, parent_idx); + let cache_diff = child.calc_cache( + &mut parent.children[child.parent_slot as usize].cache, + diff_map.remove(&child_idx), + ); + + (parent_idx, cache_diff) + } + }; + + visit_set.insert(parent_idx); + if let Some(e) = diff_map.get_mut(&parent_idx) { + B::merge_cache_diff(e, &cache_diff); + } else { + diff_map.insert(parent_idx, cache_diff); + } + } + } + + self.in_nodes + .get(self.root.unwrap_internal()) + .unwrap() + .calc_cache(&mut self.root_cache, None); + } + + /// Removed deleted children. `deleted` means they are removed from the arena. + fn filter_deleted_children(&mut self, internal_node: ArenaIndex) { + let node = self + .in_nodes + .get_mut(internal_node.unwrap_internal()) + .unwrap(); + // PERF: I hate this pattern... + let mut children = take(&mut node.children); + children.retain(|x| match x.arena { + ArenaIndex::Leaf(leaf) => self.leaf_nodes.contains(leaf), + ArenaIndex::Internal(index) => self.in_nodes.contains(index), + }); + let node = self + .in_nodes + .get_mut(internal_node.unwrap_internal()) + .unwrap(); + node.children = children; + } + + pub fn iter(&self) -> impl Iterator + '_ { + let mut path = self.first_path().unwrap_or_default(); + path.pop(); + let idx = path.last().copied().unwrap_or(Idx::new(self.root, 0)); + debug_assert!(matches!(idx.arena, ArenaIndex::Internal(_))); + let node = self.get_internal(idx.arena); + let mut iter = node.children.iter(); + core::iter::from_fn(move || loop { + if path.is_empty() { + return None; + } + + match iter.next() { + None => { + if !self.next_sibling(&mut path) { + return None; + } + + let idx = *path.last().unwrap(); + debug_assert!(matches!(idx.arena, ArenaIndex::Internal(_))); + let node = self.get_internal(idx.arena); + iter = node.children.iter(); + } + Some(elem) => { + let leaf = self.leaf_nodes.get(elem.arena.unwrap_leaf()).unwrap(); + return Some(&leaf.elem); + } + } + }) + } + + pub fn drain(&mut self, range: Range) -> iter::Drain { + iter::Drain::new(self, Some(range.start), Some(range.end)) + } + + pub fn drain_by_query>(&mut self, range: Range) -> iter::Drain { + let start = self.query::(&range.start); + let end = self.query::(&range.end); + iter::Drain::new(self, start, end) + } + + fn first_path(&self) -> Option { + let mut index = self.root; + let mut node = self.in_nodes.get(index.unwrap_internal()).unwrap(); + if node.is_empty() { + return None; + } + + let mut path = NodePath::new(); + loop { + path.push(Idx::new(index, 0)).unwrap(); + match index { + ArenaIndex::Leaf(_) => { + break; + } + ArenaIndex::Internal(_) => { + index = node.children[0].arena; + if let ArenaIndex::Internal(i) = index { + node = self.in_nodes.get(i).unwrap(); + }; + } + } + } + + Some(path) + } + + fn last_path(&self) -> Option { + let mut path = NodePath::new(); + let mut index = self.root; + let mut node = self.in_nodes.get(index.unwrap_internal()).unwrap(); + let mut pos_in_parent = 0; + if node.is_empty() { + return None; + } + + loop { + path.push(Idx::new(index, pos_in_parent)).unwrap(); + match index { + ArenaIndex::Leaf(_) => { + break; + } + ArenaIndex::Internal(_) => { + pos_in_parent = node.children.len() as u8 - 1; + index = node.children[node.children.len() - 1].arena; + if let ArenaIndex::Internal(i) = index { + node = self.in_nodes.get(i).unwrap(); + } + } + } + } + + Some(path) + } + + pub fn first_leaf(&self) -> Option { + let mut index = self.root; + let mut node = self.in_nodes.get(index.unwrap_internal()).unwrap(); + loop { + index = node.children.first()?.arena; + match index { + ArenaIndex::Leaf(leaf) => { + return Some(leaf.into()); + } + ArenaIndex::Internal(index) => { + node = self.in_nodes.get(index).unwrap(); + } + } + } + } + + pub fn last_leaf(&self) -> Option { + let mut index = self.root; + let mut node = self.in_nodes.get(index.unwrap_internal()).unwrap(); + loop { + index = node.children.last()?.arena; + match index { + ArenaIndex::Leaf(leaf) => { + return Some(leaf.into()); + } + ArenaIndex::Internal(index) => { + node = self.in_nodes.get(index).unwrap(); + } + } + } + } + + pub fn range(&self, range: Range) -> Option> + where + Q: Query, + { + if self.is_empty() { + return None; + } + + Some(self.query::(&range.start).unwrap()..self.query::(&range.end).unwrap()) + } + + pub fn iter_range( + &self, + range: impl RangeBounds, + ) -> impl Iterator> + '_ { + let start = match range.start_bound() { + std::ops::Bound::Included(start) => *start, + std::ops::Bound::Excluded(_) => unreachable!(), + std::ops::Bound::Unbounded => self.start_cursor().unwrap(), + }; + let (inclusive, end) = match range.end_bound() { + std::ops::Bound::Included(end) => (true, *end), + std::ops::Bound::Excluded(end) => (false, *end), + std::ops::Bound::Unbounded => (true, self.end_cursor().unwrap()), + }; + self._iter_range(start, end, inclusive) + } + + fn _iter_range( + &self, + start: Cursor, + end: Cursor, + inclusive_end: bool, + ) -> impl Iterator> + '_ { + let node_iter = iter::Iter::new( + self, + self.get_path(start.leaf.into()), + self.get_path(end.leaf.into()), + ); + node_iter.filter_map(move |(path, node)| { + let leaf = LeafIndex(path.last().unwrap().arena.unwrap_leaf()); + if end.leaf == leaf && end.offset == 0 && !inclusive_end { + return None; + } + + Some(ElemSlice { + cursor: Cursor { leaf, offset: 0 }, + elem: &node.elem, + start: if start.leaf == leaf { + Some(start.offset) + } else { + None + }, + end: if end.leaf == leaf { + Some(end.offset) + } else { + None + }, + }) + }) + } + + pub fn start_cursor(&self) -> Option { + Some(Cursor { + leaf: self.first_leaf()?, + offset: 0, + }) + } + + pub fn end_cursor(&self) -> Option { + let leaf = self.last_leaf()?; + let node = self.get_leaf(leaf.into()); + Some(Cursor { + leaf, + offset: node.elem.rle_len(), + }) + } + + /// Split the internal node at path into two nodes recursively upwards. + /// + // at call site the cache at path can be out-of-date. + // the cache will be up-to-date after this method + fn split(&mut self, node_idx: ArenaIndex) { + self.split_at(node_idx, None) + } + + fn split_at(&mut self, node_idx: ArenaIndex, at: Option) { + let (node_parent, node_parent_slot, this_cache, right_child) = + self.split_node(node_idx, at); + + self.inner_insert_node( + node_parent, + node_parent_slot as usize, + this_cache, + right_child, + ); + // don't need to recursive update cache + } + + fn split_node( + &mut self, + node_idx: ArenaIndex, + at: Option, + ) -> (Option, u8, ::Cache, Child) { + let node = self.in_nodes.get_mut(node_idx.unwrap_internal()).unwrap(); + let node_parent = node.parent; + let node_parent_slot = node.parent_slot; + let right: Node = Node { + parent: node.parent, + parent_slot: u8::MAX, + children: HeaplessVec::new(), + }; + + // split + let split = at.unwrap_or(node.children.len() / 2); + let right_children = HeaplessVec::from_slice(&node.children[split..]).unwrap(); + delete_range(&mut node.children, split..); + + // update cache + let mut right_cache = B::Cache::default(); + let right_arena_idx = self.in_nodes.insert(right); + let this_cache = { + let node = self.get_internal_mut(node_idx); + let mut cache = Default::default(); + node.calc_cache(&mut cache, None); + cache + }; + + // update children's parent info + for (i, child) in right_children.iter().enumerate() { + if matches!(child.arena, ArenaIndex::Internal(_)) { + let child = self.get_internal_mut(child.arena); + child.parent = Some(ArenaIndex::Internal(right_arena_idx)); + child.parent_slot = i as u8; + } else { + self.get_leaf_mut(child.arena).parent = right_arena_idx; + } + } + + let right = self.in_nodes.get_mut(right_arena_idx).unwrap(); + right.children = right_children; + // update parent cache + right.calc_cache(&mut right_cache, None); + let right_child = Child { + arena: ArenaIndex::Internal(right_arena_idx), + cache: right_cache, + }; + (node_parent, node_parent_slot, this_cache, right_child) + } + + // call site should ensure the cache is up-to-date after this method + fn inner_insert_node( + &mut self, + parent_idx: Option, + index: usize, + new_cache: B::Cache, + node: Child, + ) { + if let Some(parent_idx) = parent_idx { + let parent = self.get_internal_mut(parent_idx); + parent.children[index].cache = new_cache; + parent.children.insert(index + 1, node).unwrap(); + let is_full = parent.is_full(); + self.update_children_parent_slot_from(parent_idx, index + 1); + if is_full { + self.split(parent_idx); + } + } else { + self.split_root(new_cache, node); + } + } + + /// Update the `parent_slot` fields in `children[index..]` + fn update_children_parent_slot_from(&mut self, parent_idx: ArenaIndex, index: usize) { + let parent = self.get_internal_mut(parent_idx); + if parent.children.len() <= index || parent.is_child_leaf() { + return; + } + + // PERF: Is there a way to avoid `take` like this? + let children = take(&mut parent.children); + for (i, child) in children[index..].iter().enumerate() { + let idx = index + i; + let child = self.get_internal_mut(child.arena); + child.parent_slot = idx as u8; + } + let parent = self.get_internal_mut(parent_idx); + parent.children = children; + } + + /// right's cache should be up-to-date + fn split_root(&mut self, new_cache: B::Cache, right: Child) { + let root_idx = self.root; + // set right parent + let right_node = &mut self.get_internal_mut(right.arena); + right_node.parent_slot = 1; + right_node.parent = Some(root_idx); + let root = self.get_internal_mut(self.root); + // let left be root + let mut left_node: Node = core::mem::replace( + root, + Node { + parent: None, + parent_slot: 0, + children: Default::default(), + }, + ); + left_node.parent_slot = 0; + // set left parent + left_node.parent = Some(root_idx); + + // push left and right to root.children + root.children = Default::default(); + let left_children = left_node.children.clone(); + let left_arena = self.in_nodes.insert(left_node); + let left = Child::new(ArenaIndex::Internal(left_arena), new_cache); + let mut cache = std::mem::take(&mut self.root_cache); + let root = self.get_internal_mut(self.root); + root.children.push(left).unwrap(); + root.children.push(right).unwrap(); + + // update new root cache + root.calc_cache(&mut cache, None); + + for (i, child) in left_children.iter().enumerate() { + if child.is_internal() { + let node = self.get_internal_mut(child.arena); + node.parent = Some(ArenaIndex::Internal(left_arena)); + node.parent_slot = i as u8; + } else { + self.get_leaf_mut(child.arena).parent = left_arena; + } + } + + self.root_cache = cache; + } + + #[inline] + pub fn get_internal_mut(&mut self, index: ArenaIndex) -> &mut Node { + self.in_nodes.get_mut(index.unwrap_internal()).unwrap() + } + + #[inline] + pub fn get_leaf_mut(&mut self, index: ArenaIndex) -> &mut LeafNode { + self.leaf_nodes.get_mut(index.unwrap_leaf()).unwrap() + } + + #[inline] + fn get2_mut(&mut self, a: ArenaIndex, b: ArenaIndex) -> (&mut Node, &mut Node) { + let (a, b) = self + .in_nodes + .get2_mut(a.unwrap_internal(), b.unwrap_internal()); + (a.unwrap(), b.unwrap()) + } + + /// # Panic + /// + /// If the given index is not valid or deleted + #[inline] + pub fn get_internal(&self, index: ArenaIndex) -> &Node { + self.in_nodes.get(index.unwrap_internal()).unwrap() + } + + #[inline] + pub fn get_leaf(&self, index: ArenaIndex) -> &LeafNode { + self.leaf_nodes.get(index.unwrap_leaf()).unwrap() + } + + /// The given node is lack of children. + /// We should merge it into its neighbor or borrow from its neighbor. + /// + /// Given a random neighbor is neither full or lack, it's guaranteed + /// that we can either merge into or borrow from it without breaking + /// the balance rule. + /// + /// - The caches in parent's subtree should be up-to-date when calling this. + /// - The caches in the parent node will be updated + fn handle_lack_recursively(&mut self, node_idx: ArenaIndex) { + let mut lack_info = self.handle_lack_single_layer(node_idx); + while let Some(parent) = lack_info.parent_lack { + lack_info = self.handle_lack_single_layer(parent); + } + } + + /// The given node is lack of children. This method doesn't handle parent's lack. + /// + /// - The caches in parent's subtree should be up-to-date when calling this. + /// - The caches in the parent node will be updated + fn handle_lack_single_layer(&mut self, node_idx: ArenaIndex) -> LackInfo { + if self.root == node_idx { + self.try_reduce_levels(); + return LackInfo { parent_lack: None }; + } + + let node = self.get_internal(node_idx); + let parent_idx = node.parent.unwrap(); + let parent = self.get_internal(parent_idx); + debug_assert_eq!(parent.children[node.parent_slot as usize].arena, node_idx); + if node.children.is_empty() { + let slot = node.parent_slot as usize; + self.get_internal_mut(parent_idx).children.remove(slot); + self.in_nodes.remove(node_idx.unwrap_internal()); + self.update_children_parent_slot_from(parent_idx, slot); + return LackInfo { + parent_lack: Some(parent_idx), + }; + } + let ans = match self.pair_neighbor(node_idx) { + Some((a_idx, b_idx)) => { + let parent = self.get_internal_mut(parent_idx); + let mut a_cache = std::mem::take(&mut parent.children[a_idx.arr as usize].cache); + let mut b_cache = std::mem::take(&mut parent.children[b_idx.arr as usize].cache); + let mut re_parent = FxHashMap::default(); + + let (a, b) = self + .in_nodes + .get2_mut(a_idx.arena.unwrap_internal(), b_idx.arena.unwrap_internal()); + let a = a.unwrap(); + let b = b.unwrap(); + let ans = if a.len() + b.len() >= MAX_CHILDREN_NUM { + // move partially + if a.len() < b.len() { + // move part of b's children to a + let move_len = (b.len() - a.len()) / 2; + for child in &b.children[..move_len] { + re_parent.insert(child.arena, (a_idx.arena, a.children.len())); + a.children.push(child.clone()).unwrap(); + } + delete_range(&mut b.children, ..move_len); + for (i, child) in b.children.iter().enumerate() { + re_parent.insert(child.arena, (b_idx.arena, i)); + } + } else { + // move part of a's children to b + let move_len = (a.len() - b.len()) / 2; + for (i, child) in b.children.iter().enumerate() { + re_parent.insert(child.arena, (b_idx.arena, i + move_len)); + } + let mut b_children = + HeaplessVec::from_slice(&a.children[a.children.len() - move_len..]) + .unwrap(); + for child in take(&mut b.children) { + b_children.push(child).unwrap(); + } + b.children = b_children; + for (i, child) in b.children.iter().enumerate() { + re_parent.insert(child.arena, (b_idx.arena, i)); + } + let len = a.children.len(); + delete_range(&mut a.children, len - move_len..); + } + a.calc_cache(&mut a_cache, None); + b.calc_cache(&mut b_cache, None); + let parent = self.get_internal_mut(parent_idx); + parent.children[a_idx.arr as usize].cache = a_cache; + parent.children[b_idx.arr as usize].cache = b_cache; + LackInfo { + parent_lack: if parent.is_lack() { + Some(parent_idx) + } else { + None + }, + } + } else { + // merge + let is_parent_lack = if node_idx == a_idx.arena { + // merge b to a, delete b + for (i, child) in b.children.iter().enumerate() { + re_parent.insert(child.arena, (a_idx.arena, a.children.len() + i)); + } + + for child in take(&mut b.children) { + a.children.push(child).unwrap(); + } + + a.calc_cache(&mut a_cache, None); + let parent = self.get_internal_mut(parent_idx); + parent.children[a_idx.arr as usize].cache = a_cache; + parent.children.remove(b_idx.arr as usize); + let is_lack = parent.is_lack(); + self.purge(b_idx.arena); + self.update_children_parent_slot_from(parent_idx, b_idx.arr as usize); + is_lack + } else { + // merge a to b, delete a + for (i, child) in a.children.iter().enumerate() { + re_parent.insert(child.arena, (b_idx.arena, i)); + } + for (i, child) in b.children.iter().enumerate() { + re_parent.insert(child.arena, (b_idx.arena, i + a.children.len())); + } + + for child in take(&mut b.children) { + a.children.push(child).unwrap(); + } + + b.children = take(&mut a.children); + b.calc_cache(&mut b_cache, None); + let parent = self.get_internal_mut(parent_idx); + parent.children[b_idx.arr as usize].cache = b_cache; + parent.children.remove(a_idx.arr as usize); + let is_lack = parent.is_lack(); + self.purge(a_idx.arena); + self.update_children_parent_slot_from(parent_idx, a_idx.arr as usize); + is_lack + }; + + LackInfo { + parent_lack: if is_parent_lack { + Some(parent_idx) + } else { + None + }, + } + }; + + // FIXME: make this work + if cfg!(debug_assertions) { + // let (a, b) = self + // .in_nodes + // .get2_mut(a_idx.arena.unwrap_internal(), b_idx.arena.unwrap_internal()); + // if let Some(a) = a { + // assert!(!a.is_lack() && !a.is_full()); + // } + // if let Some(b) = b { + // assert!(!b.is_lack() && !b.is_full()); + // } + } + + for (child, (parent, slot)) in re_parent { + match child { + ArenaIndex::Leaf(_) => { + let child = self.get_leaf_mut(child); + child.parent = parent.unwrap_internal(); + } + ArenaIndex::Internal(_) => { + let child = self.get_internal_mut(child); + child.parent = Some(parent); + child.parent_slot = slot as u8; + } + } + } + ans + } + None => LackInfo { + parent_lack: Some(parent_idx), + }, + }; + ans + } + + fn try_reduce_levels(&mut self) { + let mut reduced = false; + while self.get_internal(self.root).children.len() == 1 { + let root = self.get_internal(self.root); + if root.is_child_leaf() { + break; + } + + let child_arena = root.children[0].arena; + let child = self.in_nodes.remove(child_arena.unwrap_internal()).unwrap(); + let root = self.get_internal_mut(self.root); + let _ = core::mem::replace(root, child); + reduced = true; + // root cache should be the same as child cache because there is only one child + } + if reduced { + let root_idx = self.root; + let root = self.get_internal_mut(self.root); + root.parent = None; + root.parent_slot = u8::MAX; + self.reset_children_parent_pointer(root_idx); + } + } + + fn reset_children_parent_pointer(&mut self, parent_idx: ArenaIndex) { + let parent = self.in_nodes.get(parent_idx.unwrap_internal()).unwrap(); + let children = parent.children.clone(); + for child in children { + match child.arena { + ArenaIndex::Leaf(_) => { + let child = self.get_leaf_mut(child.arena); + child.parent = parent_idx.unwrap_internal(); + } + ArenaIndex::Internal(_) => { + let child = self.get_internal_mut(child.arena); + child.parent = Some(parent_idx); + } + } + } + } + + fn pair_neighbor(&self, this: ArenaIndex) -> Option<(Idx, Idx)> { + let node = self.get_internal(this); + let arr = node.parent_slot as usize; + let parent = self.get_internal(node.parent.unwrap()); + + if arr == 0 { + parent + .children + .get(1) + .map(|x| (Idx::new(this, arr as u8), Idx::new(x.arena, 1))) + } else { + parent + .children + .get(arr - 1) + .map(|x| (Idx::new(x.arena, arr as u8 - 1), Idx::new(this, arr as u8))) + } + } + + /// Sometimes we cannot use diff because no only the given node is changed, but also its siblings. + /// For example, after delete a range of nodes, we cannot use the diff from child to infer the diff of parent. + pub fn recursive_update_cache( + &mut self, + mut node_idx: ArenaIndex, + can_use_diff: bool, + cache_diff: Option, + ) { + if let ArenaIndex::Leaf(index) = node_idx { + let leaf = self.leaf_nodes.get(index).unwrap(); + let cache = B::get_elem_cache(&leaf.elem); + node_idx = leaf.parent(); + let node = self.get_internal_mut(node_idx); + node.children + .iter_mut() + .find(|x| x.arena.unwrap_leaf() == index) + .unwrap() + .cache = cache; + } + + if can_use_diff { + if let Some(diff) = cache_diff { + return self.recursive_update_cache_with_diff(node_idx, diff); + } + } + + let mut this_idx = node_idx; + let mut node = self.get_internal_mut(node_idx); + let mut this_arr = node.parent_slot; + if can_use_diff { + if node.parent.is_some() { + let parent_idx = node.parent.unwrap(); + let (parent, this) = self.get2_mut(parent_idx, this_idx); + let diff = + this.calc_cache(&mut parent.children[this_arr as usize].cache, cache_diff); + return self.recursive_update_cache_with_diff(parent_idx, diff); + } + } else { + while node.parent.is_some() { + let parent_idx = node.parent.unwrap(); + let (parent, this) = self.get2_mut(parent_idx, this_idx); + this.calc_cache(&mut parent.children[this_arr as usize].cache, None); + this_idx = parent_idx; + this_arr = parent.parent_slot; + node = parent; + } + } + + let mut root_cache = std::mem::take(&mut self.root_cache); + let root = self.root_mut(); + root.calc_cache( + &mut root_cache, + if can_use_diff { cache_diff } else { None }, + ); + self.root_cache = root_cache; + } + + fn recursive_update_cache_with_diff(&mut self, node_idx: ArenaIndex, diff: B::CacheDiff) { + let mut node = self.get_internal_mut(node_idx); + let mut this_arr = node.parent_slot; + while node.parent.is_some() { + let parent_idx = node.parent.unwrap(); + let parent = self.get_internal_mut(parent_idx); + B::apply_cache_diff(&mut parent.children[this_arr as usize].cache, &diff); + this_arr = parent.parent_slot; + node = parent; + } + + B::apply_cache_diff(&mut self.root_cache, &diff); + } + + fn purge(&mut self, index: ArenaIndex) { + let mut stack = vec![index]; + while let Some(x) = stack.pop() { + if let ArenaIndex::Leaf(index) = x { + self.leaf_nodes.remove(index); + + continue; + } + + let Some(node) = self.in_nodes.remove(x.unwrap()) else { + continue; + }; + + for x in node.children.iter() { + stack.push(x.arena); + } + } + } + + /// find the next sibling at the same level + /// + /// return false if there is no next sibling + #[must_use] + fn next_sibling(&self, path: &mut [Idx]) -> bool { + if path.len() <= 1 { + return false; + } + + let depth = path.len(); + let parent_idx = path[depth - 2]; + let this_idx = path[depth - 1]; + let parent = self.get_internal(parent_idx.arena); + match parent.children.get(this_idx.arr as usize + 1) { + Some(next) => { + path[depth - 1] = Idx::new(next.arena, this_idx.arr + 1); + } + None => { + if !self.next_sibling(&mut path[..depth - 1]) { + return false; + } + + let parent = self.get_internal(path[depth - 2].arena); + path[depth - 1] = Idx::new(parent.children[0].arena, 0); + } + } + + true + } + + fn next_same_level_in_node(&self, node_idx: ArenaIndex) -> Option { + match node_idx { + ArenaIndex::Leaf(_) => { + let leaf_idx = node_idx.unwrap_leaf(); + let leaf1 = self.leaf_nodes.get(leaf_idx).unwrap(); + let parent1 = self.get_internal(leaf1.parent()); + let (leaf, parent, index) = + (leaf1, parent1, Self::get_leaf_slot(leaf_idx, parent1)); + if index + 1 < parent.children.len() { + Some(parent.children[index + 1].arena) + } else if let Some(parent_next) = self.next_same_level_in_node(leaf.parent()) { + let parent_next = self.get_internal(parent_next); + Some(parent_next.children.first().unwrap().arena) + } else { + None + } + } + ArenaIndex::Internal(_) => { + let node = self.get_internal(node_idx); + let parent = self.get_internal(node.parent?); + if let Some(next) = parent.children.get(node.parent_slot as usize + 1) { + Some(next.arena) + } else if let Some(parent_next) = self.next_same_level_in_node(node.parent?) { + let parent_next = self.get_internal(parent_next); + parent_next.children.first().map(|x| x.arena) + } else { + None + } + } + } + } + + fn prev_same_level_in_node(&self, node_idx: ArenaIndex) -> Option { + match node_idx { + ArenaIndex::Leaf(leaf_idx) => { + let leaf = self.leaf_nodes.get(leaf_idx).unwrap(); + let parent = self.get_internal(leaf.parent()); + let index = Self::get_leaf_slot(leaf_idx, parent); + if index > 0 { + Some(parent.children[index - 1].arena) + } else if let Some(parent_next) = self.prev_same_level_in_node(leaf.parent()) { + let parent_next = self.get_internal(parent_next); + Some(parent_next.children.last().unwrap().arena) + } else { + None + } + } + ArenaIndex::Internal(_) => { + let node = self.get_internal(node_idx); + let parent = self.get_internal(node.parent?); + if node.parent_slot > 0 { + let Some(next) = parent.children.get(node.parent_slot as usize - 1) else { + unreachable!() + }; + Some(next.arena) + } else if let Some(parent_prev) = self.prev_same_level_in_node(node.parent?) { + let parent_prev = self.get_internal(parent_prev); + parent_prev.children.last().map(|x| x.arena) + } else { + None + } + } + } + } + + /// find the next element in the tree + pub fn next_elem(&self, path: Cursor) -> Option { + self.next_same_level_in_node(path.leaf.into()) + .map(|x| Cursor { + leaf: x.unwrap_leaf().into(), + offset: 0, + }) + } + + pub fn prev_elem(&self, path: Cursor) -> Option { + self.prev_same_level_in_node(path.leaf.into()) + .map(|x| Cursor { + leaf: x.unwrap_leaf().into(), + offset: 0, + }) + } + + #[inline(always)] + pub fn root_cache(&self) -> &B::Cache { + &self.root_cache + } + + /// This method will release the memory back to OS. + /// Currently, it's just `*self = Self::new()` + #[inline(always)] + pub fn clear(&mut self) { + *self = Self::new(); + } + + #[inline(always)] + fn root_mut(&mut self) -> &mut Node { + self.get_internal_mut(self.root) + } + + #[inline(always)] + pub fn is_empty(&self) -> bool { + self.get_internal(self.root).is_empty() + } + + fn get_path(&self, idx: ArenaIndex) -> NodePath { + let mut path = NodePath::new(); + let mut node_idx = idx; + while node_idx != self.root { + match node_idx { + ArenaIndex::Leaf(inner_node_idx) => { + let node = self.leaf_nodes.get(inner_node_idx).unwrap(); + let parent = self.in_nodes.get(node.parent).unwrap(); + let index = Self::get_leaf_slot(inner_node_idx, parent); + path.push(Idx::new(node_idx, index as u8)).unwrap(); + node_idx = ArenaIndex::Internal(node.parent); + } + ArenaIndex::Internal(_) => { + let node = self.get_internal(node_idx); + path.push(Idx::new(node_idx, node.parent_slot)).unwrap(); + node_idx = node.parent.unwrap(); + } + } + } + path.push(Idx::new(self.root, 0)).unwrap(); + path.reverse(); + path + } + + pub fn push(&mut self, elem: B::Elem) -> Cursor { + let mut is_full = false; + let mut parent_idx = self.root; + let mut update_cache_idx = parent_idx; + let cache = B::get_elem_cache(&elem); + let ans = if self.is_empty() { + let data = self.alloc_leaf_child(elem, parent_idx.unwrap()); + let parent = self.in_nodes.get_mut(parent_idx.unwrap()).unwrap(); + let ans = data.arena; + parent.children.push(data).unwrap(); + Cursor { + leaf: ans.unwrap().into(), + offset: 0, + } + } else { + let leaf_idx = self.last_leaf().unwrap(); + let leaf = self.leaf_nodes.get_mut(leaf_idx.0).unwrap(); + parent_idx = leaf.parent(); + if leaf.elem.can_merge(&elem) { + update_cache_idx = leaf_idx.into(); + let offset = leaf.elem.rle_len(); + leaf.elem.merge_right(&elem); + Cursor { + leaf: leaf_idx, + offset, + } + } else { + let data = self.alloc_leaf_child(elem, parent_idx.unwrap()); + let parent = self.in_nodes.get_mut(parent_idx.unwrap()).unwrap(); + let ans = data.arena; + update_cache_idx = parent_idx; + parent.children.push(data).unwrap(); + is_full = parent.is_full(); + Cursor { + leaf: ans.unwrap().into(), + offset: 0, + } + } + }; + + self.recursive_update_cache( + update_cache_idx, + B::USE_DIFF, + if B::USE_DIFF { + Some(B::new_cache_to_diff(&cache)) + } else { + None + }, + ); + if is_full { + self.split(parent_idx); + } + + ans + } + + pub fn prepend(&mut self, elem: B::Elem) -> Cursor { + let Some(leaf_idx) = self.first_leaf() else { + let parent_idx = self.root; + let data = self.alloc_leaf_child(elem, parent_idx.unwrap()); + let parent = self.in_nodes.get_mut(parent_idx.unwrap()).unwrap(); + let ans = data.arena; + parent.children.push(data).unwrap(); + return Cursor { + leaf: ans.unwrap().into(), + offset: 0, + }; + }; + let leaf = self.leaf_nodes.get_mut(leaf_idx.0).unwrap(); + let parent_idx = leaf.parent(); + let mut is_full = false; + let ans = if elem.can_merge(&leaf.elem) { + leaf.elem.merge_left(&elem); + Cursor { + leaf: leaf_idx, + offset: 0, + } + } else { + let parent_idx = leaf.parent; + let data = self.alloc_leaf_child(elem, parent_idx); + let parent = self.in_nodes.get_mut(parent_idx).unwrap(); + let ans = data.arena; + parent.children.insert(0, data).unwrap(); + is_full = parent.is_full(); + Cursor { + leaf: ans.unwrap().into(), + offset: 0, + } + }; + + self.recursive_update_cache(leaf_idx.into(), B::USE_DIFF, None); + if is_full { + self.split(parent_idx); + } + + ans + } + + /// compare the position of a and b + pub fn compare_pos(&self, a: Cursor, b: Cursor) -> Ordering { + if a.leaf == b.leaf { + return a.offset.cmp(&b.offset); + } + + let leaf_a = self.leaf_nodes.get(a.leaf.0).unwrap(); + let leaf_b = self.leaf_nodes.get(b.leaf.0).unwrap(); + let mut node_a = self.get_internal(leaf_a.parent()); + if leaf_a.parent == leaf_b.parent { + for child in node_a.children.iter() { + if child.arena.unwrap() == a.leaf.0 { + return Ordering::Less; + } + if child.arena.unwrap() == b.leaf.0 { + return Ordering::Greater; + } + } + } + + let mut node_b = self.get_internal(leaf_b.parent()); + while node_a.parent != node_b.parent { + node_a = self.get_internal(node_a.parent.unwrap()); + node_b = self.get_internal(node_b.parent.unwrap()); + } + + node_a.parent_slot.cmp(&node_b.parent_slot) + } + + /// Iterate the caches of previous nodes/elements. + /// This method will visit as less caches as possible. + /// For example, if all nodes in a subtree need to be visited, we will only visit the root cache. + /// + /// f: (node_cache, previous_sibling_elem, (this_elem, offset)) + pub fn visit_previous_caches(&self, cursor: Cursor, mut f: F) + where + F: FnMut(PreviousCache<'_, B>), + { + // the last index of path points to the leaf element + let path = self.get_path(cursor.leaf.into()); + let mut path_index = 0; + let mut child_index = 0; + let mut node = self.get_internal(path[path_index].arena); + 'outer: loop { + if path_index + 1 >= path.len() { + break; + } + + while child_index == path.get(path_index + 1).map(|x| x.arr).unwrap() { + path_index += 1; + if path_index + 1 < path.len() { + node = self.get_internal(path[path_index].arena); + child_index = 0; + } else { + break 'outer; + } + } + + f(PreviousCache::NodeCache( + &node.children[child_index as usize].cache, + )); + child_index += 1; + } + + let node = self.leaf_nodes.get(cursor.leaf.0).unwrap(); + f(PreviousCache::ThisElemAndOffset { + elem: &node.elem, + offset: cursor.offset, + }); + } + + pub fn diagnose_balance(&self) { + let mut size_counter: FxHashMap = Default::default(); + for (_, node) in self.in_nodes.iter() { + *size_counter.entry(node.children.len()).or_default() += 1; + } + dbg!(size_counter); + + let mut size_counter: FxHashMap = Default::default(); + for (_, node) in self.leaf_nodes.iter() { + *size_counter.entry(node.elem.rle_len()).or_default() += 1; + } + dbg!(size_counter); + } + + /// Iterate over the leaf elements in the tree if the filter returns true for all its ancestors' caches, including its own cache. + pub fn iter_with_filter<'a, R: Default + Copy + AddAssign + 'a>( + &'a self, + mut f: impl FnMut(&B::Cache) -> (bool, R) + 'a, + ) -> impl Iterator + '_ { + let mut queue = VecDeque::new(); + queue.push_back((self.root, R::default())); + std::iter::from_fn(move || { + while let Some((node_idx, mut r)) = queue.pop_front() { + match node_idx { + ArenaIndex::Leaf(leaf) => { + let node = self.leaf_nodes.get(leaf).unwrap(); + return Some((r, &node.elem)); + } + ArenaIndex::Internal(idx) => { + let node = self.in_nodes.get(idx).unwrap(); + for child in node.children.iter() { + let (drill, new_r) = f(&child.cache); + if drill { + queue.push_back((child.arena, r)); + } + r += new_r; + } + } + } + } + + None + }) + } + + /// This method allows users to update the caches and the elements with a filter. + /// + /// If `f` returns true for a node, it will drill down into the subtree whose root is the node. + /// + /// It's the caller's responsibility to ensure the invariance of caches being up to date. + pub fn update_cache_and_elem_with_filter<'a>( + &'a mut self, + mut f: impl FnMut(&mut B::Cache) -> bool + 'a, + mut g: impl FnMut(&mut B::Elem) + 'a, + ) { + let mut stack = vec![self.root]; + while let Some(node_idx) = stack.pop() { + match node_idx { + ArenaIndex::Leaf(leaf) => { + let node = self.leaf_nodes.get_mut(leaf).unwrap(); + g(&mut node.elem); + } + ArenaIndex::Internal(idx) => { + let node = self.in_nodes.get_mut(idx).unwrap(); + for child in node.children.iter_mut() { + if f(&mut child.cache) { + stack.push(child.arena); + } + } + } + } + } + } + + pub fn depth(&self) -> usize { + let mut depth = 0; + let mut index = self.root; + let mut node = self.in_nodes.get(index.unwrap_internal()).unwrap(); + loop { + depth += 1; + index = node.children.first().unwrap().arena; + match index { + ArenaIndex::Leaf(_) => return depth, + ArenaIndex::Internal(index) => { + node = self.in_nodes.get(index).unwrap(); + } + } + } + } + + pub fn internal_avg_children_num(&self) -> f64 { + let mut sum = 0; + for (_, node) in self.in_nodes.iter() { + sum += node.children.len(); + } + sum as f64 / self.in_nodes.len() as f64 + } +} + +fn merge_adj(data: &mut Vec) { + // Merge adjacent elements + let mut i = 0; + let last = data.len() - 1; + let mut to_delete_start = 0; + let mut del_len = 0; + while i < last { + if data[i].can_merge(&data[i + 1]) { + let (a, b) = arref::mut_twice(data.as_mut_slice(), i, i + 1).unwrap(); + a.merge_right(b); + if del_len == 0 { + to_delete_start = i + 1; + } + + data.swap(i + 1, to_delete_start + del_len); + del_len += 1; + i += 1; + } + i += 1; + } + + if del_len > 0 { + data.drain(to_delete_start..to_delete_start + del_len); + } +} + +pub enum PreviousCache<'a, B: BTreeTrait> { + NodeCache(&'a B::Cache), + PrevSiblingElem(&'a B::Elem), + ThisElemAndOffset { elem: &'a B::Elem, offset: usize }, +} + +#[inline(always)] +fn add_leaf_dirty_map(leaf: ArenaIndex, dirty_map: &mut LeafDirtyMap, leaf_diff: T) { + dirty_map.insert(leaf, leaf_diff); +} + +impl BTree { + pub fn check(&self) { + // check cache + let mut leaf_level = None; + for (index, node) in self.in_nodes.iter() { + if index != self.root.unwrap() { + assert!(!node.is_empty()); + } + + for (i, child_info) in node.children.iter().enumerate() { + if matches!(child_info.arena, ArenaIndex::Internal(_)) { + assert!(!node.is_child_leaf()); + let child = self.get_internal(child_info.arena); + let mut cache = Default::default(); + child.calc_cache(&mut cache, None); + assert_eq!(child.parent_slot, i as u8); + assert_eq!(child.parent, Some(ArenaIndex::Internal(index))); + assert_eq!( + cache, child_info.cache, + "index={:?} child_index={:?}", + index, child_info.arena + ); + } + } + + if let Some(parent) = node.parent { + let parent = self.get_internal(parent); + assert_eq!( + parent.children[node.parent_slot as usize].arena, + ArenaIndex::Internal(index) + ); + self.get_path(ArenaIndex::Internal(index)); + } else { + assert_eq!(index, self.root.unwrap_internal()) + } + + // if index != self.root.unwrap() { + // assert!(!node.is_lack(), "len={}\n", node.len()); + // } + // + // assert!(!node.is_full(), "len={}", node.len()); + } + + let root = self.get_internal(self.root); + let mut root_cache = Default::default(); + root.calc_cache(&mut root_cache, None); + assert_eq!(&self.root_cache, &root_cache); + + for (leaf_index, leaf_node) in self.leaf_nodes.iter() { + let mut length = 1; + let mut node_idx = leaf_node.parent; + while node_idx != self.root.unwrap() { + let node = self.get_internal(ArenaIndex::Internal(node_idx)); + length += 1; + node_idx = node.parent.unwrap().unwrap(); + } + match leaf_level { + Some(expected) => { + if length != expected { + dbg!(leaf_index, leaf_node); + assert_eq!(length, expected); + } + } + None => { + leaf_level = Some(length); + } + } + + let cache = B::get_elem_cache(&leaf_node.elem); + let parent = self.get_internal(leaf_node.parent()); + assert_eq!( + parent + .children + .iter() + .find(|x| x.arena.unwrap_leaf() == leaf_index) + .unwrap() + .cache, + cache + ); + self.get_path(ArenaIndex::Leaf(leaf_index)); + } + } +} + +impl> FromIterator for BTree { + fn from_iter>(iter: I) -> Self { + let mut tree = Self::new(); + let iter = iter.into_iter(); + let min_size = iter.size_hint().0; + tree.leaf_nodes.reserve(min_size); + let max_child_size = MAX_CHILDREN_NUM - 2; + + struct TempInternalNode { + children: HeaplessVec, MAX_CHILDREN_NUM>, + cache: B::Cache, + arena_index: RawArenaIndex, + } + + let parent_num = (min_size + max_child_size - 1) / max_child_size; + let mut internal_nodes: Vec> = Vec::with_capacity(parent_num); + let index = tree.in_nodes.insert(Default::default()); + internal_nodes.push(TempInternalNode { + children: Default::default(), + cache: Default::default(), + arena_index: index, + }); + + // create all leaf nodes and their parents + for elem in iter { + let parent = match internal_nodes.last_mut() { + Some(last) if last.children.len() < max_child_size => last, + Some(last) => { + // calculate cache + B::calc_cache_internal(&mut last.cache, &last.children); + let index = tree.in_nodes.insert(Default::default()); + internal_nodes.push(TempInternalNode { + children: Default::default(), + cache: Default::default(), + arena_index: index, + }); + internal_nodes.last_mut().unwrap() + } + _ => unreachable!(), + }; + + let leaf = LeafNode { + elem: elem.into(), + parent: parent.arena_index, + }; + + let cache = B::get_elem_cache(&leaf.elem); + let leaf_index = tree.leaf_nodes.insert(leaf); + parent + .children + .push(Child { + arena: ArenaIndex::Leaf(leaf_index), + cache, + }) + .unwrap(); + } + + // recursively create the internal nodes in higher level, until we reach root + while internal_nodes.len() > 1 { + let parent_num = (internal_nodes.len() + max_child_size - 1) / max_child_size; + let children = std::mem::replace(&mut internal_nodes, Vec::with_capacity(parent_num)); + let index = tree.in_nodes.insert(Default::default()); + internal_nodes.push(TempInternalNode { + children: Default::default(), + cache: Default::default(), + arena_index: index, + }); + + let mut parent_slot = 0; + // eprintln!( + // "children.len={} max_child_size={}", + // children.len(), + // max_child_size + // ); + for mut child in children { + let parent = match internal_nodes.last_mut() { + Some(last) if last.children.len() < max_child_size => last, + Some(last) => { + // calculate cache + B::calc_cache_internal(&mut last.cache, &last.children); + let index = tree.in_nodes.insert(Default::default()); + internal_nodes.push(TempInternalNode { + children: Default::default(), + cache: Default::default(), + arena_index: index, + }); + internal_nodes.last_mut().unwrap() + } + _ => unreachable!(), + }; + + B::calc_cache_internal(&mut child.cache, &child.children); + let child_node = tree.in_nodes.get_mut(child.arena_index).unwrap(); + child_node.children = child.children; + child_node.parent = Some(ArenaIndex::Internal(parent.arena_index)); + child_node.parent_slot = parent_slot; + parent_slot = (parent_slot + 1) % (max_child_size as u8); + parent + .children + .push(Child { + arena: ArenaIndex::Internal(child.arena_index), + cache: child.cache, + }) + .unwrap(); + } + + debug_assert_eq!(parent_num, internal_nodes.len()); + } + + debug_assert_eq!(internal_nodes.len(), 1); + let node = internal_nodes.remove(0); + B::calc_cache_internal(&mut tree.root_cache, &node.children); + tree.in_nodes.remove(tree.root.unwrap()); + tree.root = ArenaIndex::Internal(node.arena_index); + let root = tree.root.unwrap(); + tree.in_nodes.get_mut(root).unwrap().children = node.children; + tree + } +} + +struct SplitInfo { + new_pos: Option, + left_neighbour: Option, + parent_idx: RawArenaIndex, + insert_slot: usize, + new_leaf: Option, +} + +impl Default for BTree { + fn default() -> Self { + Self::new() + } +} + +fn delete_range( + arr: &mut heapless::Vec, + range: impl RangeBounds, +) { + let start = match range.start_bound() { + std::ops::Bound::Included(x) => *x, + std::ops::Bound::Excluded(x) => x + 1, + std::ops::Bound::Unbounded => 0, + }; + let end = match range.end_bound() { + std::ops::Bound::Included(x) => x + 1, + std::ops::Bound::Excluded(x) => *x, + std::ops::Bound::Unbounded => arr.len(), + }; + + if start == end { + return; + } + + if end - start == 1 { + arr.remove(start); + return; + } + + let mut ans = heapless::Vec::from_slice(&arr[..start]).unwrap(); + ans.extend_from_slice(&arr[end..]).unwrap(); + *arr = ans; +} diff --git a/3rdparty/loro/crates/generic-btree/src/rle.rs b/3rdparty/loro/crates/generic-btree/src/rle.rs new file mode 100644 index 0000000000..52ec0a2f15 --- /dev/null +++ b/3rdparty/loro/crates/generic-btree/src/rle.rs @@ -0,0 +1,141 @@ +use core::ops::RangeBounds; +use std::ops::Range; + +/// For better performance, it's advised to impl split +pub trait Sliceable: HasLength + Sized { + #[must_use] + fn _slice(&self, range: Range) -> Self; + + #[must_use] + #[inline(always)] + fn slice(&self, range: impl RangeBounds) -> Self { + let start = match range.start_bound() { + std::ops::Bound::Included(x) => *x, + std::ops::Bound::Excluded(x) => x + 1, + std::ops::Bound::Unbounded => 0, + }; + + let end = match range.end_bound() { + std::ops::Bound::Included(x) => x + 1, + std::ops::Bound::Excluded(x) => *x, + std::ops::Bound::Unbounded => self.rle_len(), + }; + + self._slice(start..end) + } + + /// slice in-place + #[inline(always)] + fn slice_(&mut self, range: impl RangeBounds) { + *self = self.slice(range); + } + + #[must_use] + fn split(&mut self, pos: usize) -> Self { + let right = self.slice(pos..); + self.slice_(..pos); + right + } + + /// Update the slice in the given range. + /// This method may split `self` into two or three parts. + /// If so, it will make `self` the leftmost part and return the next split parts. + /// + /// # Example + /// + /// If `self.rle_len() == 10`, `self.update(1..5)` will split self into three parts and update the middle part. + /// It returns the middle and the right part. + fn update_with_split( + &mut self, + range: impl RangeBounds, + f: impl FnOnce(&mut Self), + ) -> (Option, Option) { + let start = match range.start_bound() { + std::ops::Bound::Included(x) => *x, + std::ops::Bound::Excluded(x) => x + 1, + std::ops::Bound::Unbounded => 0, + }; + + let end = match range.end_bound() { + std::ops::Bound::Included(x) => x + 1, + std::ops::Bound::Excluded(x) => *x, + std::ops::Bound::Unbounded => self.rle_len(), + }; + + if start >= end { + return (None, None); + } + + match (start == 0, end == self.rle_len()) { + (true, true) => { + f(self); + (None, None) + } + (true, false) => { + let right = self.split(end); + f(self); + (Some(right), None) + } + (false, true) => { + let mut right = self.split(start); + f(&mut right); + (Some(right), None) + } + (false, false) => { + let right = self.split(end); + let mut middle = self.split(start); + f(&mut middle); + (Some(middle), Some(right)) + } + } + } +} + +pub trait Mergeable { + /// Whether self can merge rhs with self on the left. + /// + /// Note: This is not symmetric. + fn can_merge(&self, rhs: &Self) -> bool; + fn merge_right(&mut self, rhs: &Self); + fn merge_left(&mut self, left: &Self); +} + +pub trait HasLength { + fn rle_len(&self) -> usize; +} + +pub trait TryInsert { + fn try_insert(&mut self, pos: usize, elem: Self) -> Result<(), Self> + where + Self: Sized; +} + +pub trait CanRemove { + fn can_remove(&self) -> bool; +} + +impl CanRemove for () { + fn can_remove(&self) -> bool { + true + } +} +impl CanRemove for usize { + fn can_remove(&self) -> bool { + *self == 0 + } +} +impl CanRemove for isize { + fn can_remove(&self) -> bool { + *self == 0 + } +} +impl CanRemove for u32 { + fn can_remove(&self) -> bool { + *self == 0 + } +} +impl CanRemove for i32 { + fn can_remove(&self) -> bool { + *self == 0 + } +} diff --git a/3rdparty/loro/crates/kv-store/Cargo.toml b/3rdparty/loro/crates/kv-store/Cargo.toml new file mode 100644 index 0000000000..faddce7ff0 --- /dev/null +++ b/3rdparty/loro/crates/kv-store/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "loro-kv-store" +version = "1.13.7" +edition = "2021" +license = "MIT" +description = "Key-value store component for Loro, a high-performance CRDTs framework" +homepage = "https://loro.dev" +repository = "https://github.com/loro-dev/loro/" +authors = ["Liang Zhao", "Zixuan Chen"] + + +[target.'cfg(all(target_arch = "wasm32", target_os = "unknown"))'.dependencies] +quick_cache = { version = "0.6.2", default-features = false } + +[target.'cfg(not(all(target_arch = "wasm32", target_os = "unknown")))'.dependencies] +quick_cache = { version = "0.6.2" } + +[dependencies] +loro-common = { path = "../loro-common", version = "1.13.1" } +bytes = { workspace = true } +rustc-hash = { workspace = true } +once_cell = { workspace = true } +lz4_flex = { version = "0.11" } +xxhash-rust = { workspace = true } +ensure-cov = { workspace = true } +tracing = { workspace = true } + +[dev-dependencies] +rand = "0.8.5" +ctor = "0.2" diff --git a/3rdparty/loro/crates/kv-store/LICENSE b/3rdparty/loro/crates/kv-store/LICENSE new file mode 100644 index 0000000000..bf4183a502 --- /dev/null +++ b/3rdparty/loro/crates/kv-store/LICENSE @@ -0,0 +1,21 @@ +MIT License Copyright (c) 2024 Zixuan Chen + +Permission is hereby granted, free of +charge, to any person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the Software without +restriction, including without limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to the +following conditions: + +The above copyright notice and this permission notice +(including the next paragraph) shall be included in all copies or substantial +portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO +EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/3rdparty/loro/crates/kv-store/src/block.rs b/3rdparty/loro/crates/kv-store/src/block.rs new file mode 100644 index 0000000000..beaf970f29 --- /dev/null +++ b/3rdparty/loro/crates/kv-store/src/block.rs @@ -0,0 +1,869 @@ +use std::{ + fmt::Debug, + io::Write, + ops::{Bound, Range}, + sync::Arc, +}; + +use bytes::{Buf, Bytes}; +use loro_common::{LoroError, LoroResult}; +use once_cell::sync::OnceCell; + +use crate::{ + compress::{compress, decompress, CompressionType}, + iter::KvIterator, + sstable::{get_common_prefix_len_and_strip, SIZE_OF_U32, XXH_SEED}, +}; + +use super::sstable::{SIZE_OF_U16, SIZE_OF_U8}; + +const MAX_NORMAL_BLOCK_DATA_LEN: usize = u16::MAX as usize; +const MAX_NORMAL_BLOCK_ENTRIES: usize = u16::MAX as usize; + +#[derive(Debug, Clone)] +pub struct LargeValueBlock { + // without checksum + pub value_bytes: Bytes, + pub encoded_bytes: OnceCell<(Bytes, CompressionType)>, + pub key: Bytes, +} + +impl LargeValueBlock { + /// ┌──────────────────────────┐ + /// │Large Block │ + /// │┌ ─ ─ ─ ┬ ─ ─ ─ ─ ─ ─ ─ ─ │ + /// │ value Block Checksum ││ + /// ││ bytes │ u32 │ + /// │ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┘│ + /// └──────────────────────────┘ + fn encode(&self, w: &mut Vec, mut compression_type: CompressionType) -> CompressionType { + if let Some((bytes, encoded_compression_type)) = self.encoded_bytes.get() { + if encoded_compression_type == &compression_type { + w.extend_from_slice(bytes); + return compression_type; + } + } + + let origin_len = w.len(); + compress(w, &self.value_bytes, compression_type); + if !compression_type.is_none() && w.len() - origin_len > self.value_bytes.len() { + w.truncate(origin_len); + compress(w, &self.value_bytes, CompressionType::None); + ensure_cov::notify_cov("kv_store::block::LargeValueBlock::encode::compress_fallback"); + compression_type = CompressionType::None; + } + let checksum = xxhash_rust::xxh32::xxh32(&w[origin_len..], XXH_SEED); + w.write_all(&checksum.to_le_bytes()).unwrap(); + compression_type + } + + fn decode(bytes: Bytes, key: Bytes, compression_type: CompressionType) -> LoroResult { + let mut value_bytes = vec![]; + decompress( + &mut value_bytes, + bytes.slice(..bytes.len() - SIZE_OF_U32), + compression_type, + )?; + Ok(LargeValueBlock { + value_bytes: Bytes::from(value_bytes), + encoded_bytes: OnceCell::with_value((bytes, compression_type)), + key, + }) + } +} + +#[derive(Debug, Clone)] +pub struct NormalBlock { + pub data: Bytes, + pub encoded_data: OnceCell<(Bytes, CompressionType)>, + pub first_key: Bytes, + pub offsets: Vec, +} + +impl NormalBlock { + /// ┌────────────────────────────────────────────────────────────────────────────────────────┐ + /// │Block │ + /// │┌ ─ ─ ─ ─ ─ ─ ─ ┬ ─ ─ ─┌ ─ ─ ─ ─ ─ ─ ─ ┬ ─ ─ ─ ─┌ ─ ─ ─┌ ─ ─ ─ ┬ ─ ─ ─ ─┌ ─ ─ ─ ─ ─ ─ ─ │ + /// │ Key Value Chunk ... │Key Value Chunk offset │ ... │ offset kv len │Block Checksum││ + /// ││ bytes │ │ bytes │ u16 │ │ u16 │ u16 │ u32 │ + /// │ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┘─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┘─ ─ ─ ┘─ ─ ─ ─ ─ ─ ─ ─ ┘─ ─ ─ ─ ─ ─ ─ ┘│ + /// └────────────────────────────────────────────────────────────────────────────────────────┘ + /// + /// The block body may be compressed then we calculate its checksum (the checksum is not compressed). + fn encode(&self, w: &mut Vec, mut compression_type: CompressionType) -> CompressionType { + if let Some((encoded_data, encoded_compression_type)) = self.encoded_data.get() { + if encoded_compression_type == &compression_type { + w.extend_from_slice(encoded_data); + return compression_type; + } + } + + let origin_len = w.len(); + let mut buf = self.data.to_vec(); + for offset in &self.offsets { + buf.extend_from_slice(&offset.to_le_bytes()); + } + buf.extend_from_slice(&(self.offsets.len() as u16).to_le_bytes()); + compress(w, &buf, compression_type); + if !compression_type.is_none() && w.len() - origin_len > buf.len() { + w.truncate(origin_len); + compress(w, &buf, CompressionType::None); + ensure_cov::notify_cov("kv_store::block::NormalBlock::encode::compress_fallback"); + compression_type = CompressionType::None; + } + let checksum = xxhash_rust::xxh32::xxh32(&w[origin_len..], XXH_SEED); + w.extend_from_slice(&checksum.to_le_bytes()); + compression_type + } + + fn decode( + raw_block_and_check: Bytes, + first_key: Bytes, + compression_type: CompressionType, + ) -> LoroResult { + if raw_block_and_check.len() < SIZE_OF_U32 { + return Err(LoroError::DecodeError("Invalid bytes".into())); + } + + let buf = raw_block_and_check.slice(..raw_block_and_check.len() - SIZE_OF_U32); + let mut data = vec![]; + decompress(&mut data, buf, compression_type)?; + if data.len() < SIZE_OF_U16 { + return Err(LoroError::DecodeError("Invalid bytes".into())); + } + + let offsets_len = (&data[data.len() - SIZE_OF_U16..]).get_u16_le() as usize; + if offsets_len == 0 { + return Err(LoroError::DecodeError("Invalid bytes".into())); + } + + let offsets_bytes_len = SIZE_OF_U16 + .checked_mul(offsets_len + 1) + .ok_or_else(|| LoroError::DecodeError("Invalid bytes".into()))?; + if data.len() < offsets_bytes_len { + return Err(LoroError::DecodeError("Invalid bytes".into())); + } + + let data_end = data.len() - offsets_bytes_len; + if data_end > u16::MAX as usize { + return Err(LoroError::DecodeError("Invalid bytes".into())); + } + + let offsets = &data[data_end..data.len() - SIZE_OF_U16]; + let offsets: Vec = offsets + .chunks(SIZE_OF_U16) + .map(|mut chunk| chunk.get_u16_le()) + .collect(); + Self::validate_decoded_data(&data[..data_end], &offsets, &first_key)?; + Ok(NormalBlock { + data: Bytes::copy_from_slice(&data[..data_end]), + encoded_data: OnceCell::with_value((raw_block_and_check, compression_type)), + offsets, + first_key, + }) + } + + fn validate_decoded_data(data: &[u8], offsets: &[u16], first_key: &[u8]) -> LoroResult<()> { + if offsets.first().copied() != Some(0) { + return Err(LoroError::DecodeError("Invalid bytes".into())); + } + + let mut prev_key: Option> = None; + let mut prev_offset = 0usize; + for (idx, offset) in offsets.iter().map(|x| *x as usize).enumerate() { + let offset_end = offsets + .get(idx + 1) + .map_or(data.len(), |next| *next as usize); + if offset < prev_offset || offset > offset_end || offset_end > data.len() { + return Err(LoroError::DecodeError("Invalid bytes".into())); + } + + let key = if idx == 0 { + first_key.to_vec() + } else { + let header_end = offset + .checked_add(SIZE_OF_U8 + SIZE_OF_U16) + .ok_or_else(|| LoroError::DecodeError("Invalid bytes".into()))?; + if header_end > offset_end { + return Err(LoroError::DecodeError("Invalid bytes".into())); + } + + let common_prefix_len = data[offset] as usize; + if common_prefix_len > first_key.len() { + return Err(LoroError::DecodeError("Invalid bytes".into())); + } + + let key_suffix_len = + u16::from_le_bytes(data[offset + SIZE_OF_U8..header_end].try_into().unwrap()) + as usize; + let key_end = header_end + .checked_add(key_suffix_len) + .ok_or_else(|| LoroError::DecodeError("Invalid bytes".into()))?; + if key_end > offset_end { + return Err(LoroError::DecodeError("Invalid bytes".into())); + } + + let mut key = Vec::with_capacity(common_prefix_len + key_suffix_len); + key.extend_from_slice(&first_key[..common_prefix_len]); + key.extend_from_slice(&data[header_end..key_end]); + key + }; + + if key.is_empty() + || prev_key + .as_ref() + .is_some_and(|prev_key| prev_key.as_slice() >= key.as_slice()) + { + return Err(LoroError::DecodeError("Invalid bytes".into())); + } + + prev_offset = offset; + prev_key = Some(key); + } + + Ok(()) + } +} + +#[derive(Debug, Clone)] +pub enum Block { + Normal(NormalBlock), + Large(LargeValueBlock), +} + +impl Block { + pub fn is_large(&self) -> bool { + matches!(self, Block::Large(_)) + } + + pub fn data(&self) -> Bytes { + match self { + Block::Normal(block) => block.data.clone(), + Block::Large(block) => block.value_bytes.clone(), + } + } + + pub fn first_key(&self) -> Bytes { + match self { + Block::Normal(block) => block.first_key.clone(), + Block::Large(block) => block.key.clone(), + } + } + + pub fn last_key(&self) -> Bytes { + match self { + Block::Normal(block) => { + if block.offsets.len() == 1 { + return block.first_key.clone(); + } + + let offset = *block.offsets.last().unwrap() as usize; + let mut bytes = &block.data[offset..]; + let common_prefix_len = bytes.get_u8() as usize; + let key_suffix_len = bytes.get_u16_le() as usize; + let mut last_key = Vec::with_capacity(common_prefix_len + key_suffix_len); + last_key.extend_from_slice(&block.first_key[..common_prefix_len]); + last_key.extend_from_slice(&bytes[..key_suffix_len]); + last_key.into() + } + Block::Large(block) => block.key.clone(), + } + } + + pub fn encode(&self, w: &mut Vec, compression_type: CompressionType) -> CompressionType { + match self { + Block::Normal(block) => block.encode(w, compression_type), + Block::Large(block) => block.encode(w, compression_type), + } + } + + pub(crate) fn try_decode( + raw_block_and_check: Bytes, + is_large: bool, + key: Bytes, + compression_type: CompressionType, + ) -> LoroResult { + if key.is_empty() { + return Err(LoroError::DecodeError("Invalid bytes".into())); + } + + if is_large { + return LargeValueBlock::decode(raw_block_and_check, key, compression_type) + .map(Block::Large); + } + NormalBlock::decode(raw_block_and_check, key, compression_type).map(Block::Normal) + } + + pub fn decode( + raw_block_and_check: Bytes, + is_large: bool, + key: Bytes, + compression_type: CompressionType, + ) -> Self { + // The caller is responsible for validating SSTable integrity before lazy block reads. + Self::try_decode(raw_block_and_check, is_large, key, compression_type) + .expect("validated SSTable block should decode") + } + + pub fn len(&self) -> usize { + match self { + Block::Normal(block) => block.offsets.len(), + Block::Large(_) => 1, + } + } + + pub fn is_empty(&self) -> bool { + match self { + Block::Normal(block) => block.offsets.is_empty(), + Block::Large(_) => false, + } + } +} + +#[derive(Debug)] +pub struct BlockBuilder { + data: Vec, + offsets: Vec, + block_size: usize, + // for key compression + first_key: Bytes, + is_large: bool, +} + +impl BlockBuilder { + pub fn new(block_size: usize) -> Self { + Self { + data: Vec::new(), + offsets: Vec::new(), + block_size, + first_key: Bytes::new(), + is_large: false, + } + } + + pub fn estimated_size(&self) -> usize { + if self.is_large { + self.data.len() + } else { + // key-value pairs number + SIZE_OF_U16 + + // offsets + self.offsets.len() * SIZE_OF_U16 + + // key-value pairs data + self.data.len() + + // checksum + SIZE_OF_U32 + } + } + + pub fn is_empty(&self) -> bool { + !self.is_large && self.offsets.is_empty() + } + + /// Add a key-value pair to the block. + /// Returns true if the key-value pair is added successfully, false the block is full. + /// + /// ┌─────────────────────────────────────────────────────┐ + /// │ Key Value Chunk │ + /// │┌ ─ ─ ─ ─ ─ ─ ─ ─ ┬ ─ ─ ─ ─ ─ ─ ─┌ ─ ─ ─ ─ ─┬ ─ ─ ─ ┐│ + /// │ common prefix len key suffix len│key suffix│ value ││ + /// ││ u8 │ u16 │ bytes │ bytes ││ + /// │ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┘─ ─ ─ ─ ─ ┘ ─ ─ ─ ┘│ + /// └─────────────────────────────────────────────────────┘ + /// + pub fn add(&mut self, key: &[u8], value: &[u8]) -> bool { + if key.is_empty() { + return false; + } + + debug_assert!(!key.is_empty(), "key cannot be empty"); + if self.first_key.is_empty() { + if value.len() > self.block_size || value.len() > MAX_NORMAL_BLOCK_DATA_LEN { + self.data.extend_from_slice(value); + self.is_large = true; + self.first_key = Bytes::copy_from_slice(key); + return true; + } + + self.first_key = Bytes::copy_from_slice(key); + self.offsets.push(self.data.len() as u16); + self.data.extend_from_slice(value); + return true; + } + + if self.offsets.len() >= MAX_NORMAL_BLOCK_ENTRIES { + return false; + } + + let (common, suffix) = get_common_prefix_len_and_strip(key, &self.first_key); + let key_len = suffix.len(); + let Some(next_data_len) = self + .data + .len() + .checked_add(SIZE_OF_U8 + SIZE_OF_U16) + .and_then(|len| len.checked_add(key_len)) + .and_then(|len| len.checked_add(value.len())) + else { + return false; + }; + if next_data_len > MAX_NORMAL_BLOCK_DATA_LEN { + return false; + } + + // whether the block is full + let Some(estimated_size) = self + .estimated_size() + .checked_add(key_len) + .and_then(|len| len.checked_add(value.len())) + .and_then(|len| len.checked_add(SIZE_OF_U8 + SIZE_OF_U16)) + else { + return false; + }; + if estimated_size > self.block_size { + return false; + } + + self.offsets.push(self.data.len() as u16); + self.data.push(common); + self.data.extend_from_slice(&(key_len as u16).to_le_bytes()); + self.data.extend_from_slice(suffix); + self.data.extend_from_slice(value); + true + } + + pub fn build(self) -> Block { + if self.is_large { + return Block::Large(LargeValueBlock { + value_bytes: Bytes::from(self.data), + key: self.first_key, + encoded_bytes: OnceCell::new(), + }); + } + debug_assert!(!self.offsets.is_empty(), "block is empty"); + Block::Normal(NormalBlock { + data: Bytes::from(self.data), + offsets: self.offsets, + first_key: self.first_key, + encoded_data: OnceCell::new(), + }) + } +} + +/// Block iterator +/// +/// If the key is empty, it means the iterator is invalid. +#[derive(Clone)] +pub struct BlockIter { + block: Arc, + next_key: Bytes, + next_value_range: Range, + prev_key: Bytes, + prev_value_range: Range, + next_idx: usize, + prev_idx: isize, + first_key: Bytes, +} + +impl Debug for BlockIter { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("BlockIter") + .field("is_large", &self.block.is_large()) + .field("next_key", &self.next_key) + .field("next_value_range", &self.next_value_range) + .field("prev_key", &self.prev_key) + .field("prev_value_range", &self.prev_value_range) + .field("next_idx", &self.next_idx) + .field("prev_idx", &self.prev_idx) + .field("first_key", &Bytes::copy_from_slice(&self.first_key)) + .finish() + } +} + +impl BlockIter { + pub fn new(block: Arc) -> Self { + let prev_idx = block.len() as isize - 1; + let mut iter = Self { + first_key: block.first_key(), + block, + next_key: Bytes::new(), + next_value_range: 0..0, + prev_key: Bytes::new(), + prev_value_range: 0..0, + next_idx: 0, + prev_idx, + }; + iter.seek_to_idx(0); + iter.back_to_idx(prev_idx); + iter + } + + pub fn new_seek_to_key(block: Arc, key: &[u8]) -> Self { + let prev_idx = block.len() as isize - 1; + let mut iter = Self { + first_key: block.first_key(), + block, + next_key: Bytes::new(), + next_value_range: 0..0, + prev_key: Bytes::new(), + prev_value_range: 0..0, + next_idx: 0, + prev_idx, + }; + iter.seek_to_key(key); + iter.back_to_idx(prev_idx); + iter + } + + pub fn new_back_to_key(block: Arc, key: &[u8]) -> Self { + let prev_idx = block.len() as isize - 1; + let mut iter = Self { + first_key: block.first_key(), + block, + next_key: Bytes::new(), + next_value_range: 0..0, + prev_key: Bytes::new(), + prev_value_range: 0..0, + next_idx: 0, + prev_idx, + }; + iter.seek_to_idx(0); + iter.back_to_key(key); + iter + } + + pub fn new_scan(block: Arc, start: Bound<&[u8]>, end: Bound<&[u8]>) -> Self { + let mut iter = match start { + Bound::Included(key) => Self::new_seek_to_key(block, key), + Bound::Excluded(key) => { + let mut iter = Self::new_seek_to_key(block, key); + while iter.has_next() && iter.peek_next_curr_key().unwrap() == key { + iter.next(); + } + iter + } + Bound::Unbounded => Self::new(block), + }; + match end { + Bound::Included(key) => { + iter.back_to_key(key); + } + Bound::Excluded(key) => { + iter.back_to_key(key); + while iter.has_next_back() && iter.peek_back_curr_key().unwrap() == key { + iter.next_back(); + } + } + Bound::Unbounded => {} + } + iter + } + + pub fn peek_next_curr_key(&self) -> Option { + if self.has_next() { + Some(Bytes::copy_from_slice(&self.next_key)) + } else { + None + } + } + + pub fn peek_next_curr_value(&self) -> Option { + if self.has_next() { + Some(self.block.data().slice(self.next_value_range.clone())) + } else { + None + } + } + + pub fn has_next(&self) -> bool { + !self.next_key.is_empty() && self.next_idx as isize <= self.prev_idx + } + + pub fn peek_back_curr_key(&self) -> Option { + if self.has_next_back() { + Some(Bytes::copy_from_slice(&self.prev_key)) + } else { + None + } + } + + pub fn peek_back_curr_value(&self) -> Option { + if self.has_next_back() { + Some(self.block.data().slice(self.prev_value_range.clone())) + } else { + None + } + } + + pub fn has_next_back(&self) -> bool { + !self.prev_key.is_empty() && self.next_idx as isize <= self.prev_idx + } + + pub fn next(&mut self) { + self.next_idx += 1; + if self.next_idx as isize > self.prev_idx { + self.next_key.clear(); + self.next_value_range = 0..0; + return; + } + self.seek_to_idx(self.next_idx); + } + + pub fn next_back(&mut self) { + self.prev_idx -= 1; + if self.prev_idx < 0 || self.prev_idx < (self.next_idx as isize) { + self.prev_key.clear(); + self.prev_value_range = 0..0; + return; + } + self.back_to_idx(self.prev_idx); + } + + pub fn seek_to_key(&mut self, key: &[u8]) { + match self.block.as_ref() { + Block::Normal(block) => { + let mut left = 0; + let mut right = block.offsets.len(); + while left < right { + let mid = left + (right - left) / 2; + self.seek_to_idx(mid); + debug_assert!(self.has_next()); + if self.next_key == key { + return; + } + if self.next_key < key { + left = mid + 1; + } else { + right = mid; + } + } + self.seek_to_idx(left); + } + Block::Large(block) => { + if key > block.key { + self.seek_to_idx(1); + } else { + self.seek_to_idx(0); + } + } + } + } + + /// MUST be called after seek_to_key() + pub fn back_to_key(&mut self, key: &[u8]) { + match self.block.as_ref() { + Block::Normal(block) => { + let mut left = self.next_idx; + let mut right = block.offsets.len(); + while left < right { + let mid = left + (right - left) / 2; + self.back_to_idx(mid as isize); + // prev idx <= next idx + if !self.has_next_back() { + return; + } + debug_assert!(self.has_next_back()); + if self.prev_key > key { + right = mid; + } else { + left = mid + 1; + } + } + self.back_to_idx(left as isize - 1); + } + Block::Large(block) => { + if key < block.key { + self.back_to_idx(-1); + } else { + self.back_to_idx(0); + } + } + } + } + + fn seek_to_idx(&mut self, idx: usize) { + match self.block.as_ref() { + Block::Normal(block) => { + if idx >= block.offsets.len() { + self.next_key.clear(); + self.next_value_range = 0..0; + self.next_idx = idx; + return; + } + let offset = block.offsets[idx] as usize; + self.seek_to_offset( + offset, + *block + .offsets + .get(idx + 1) + .unwrap_or(&(block.data.len() as u16)) as usize, + idx == 0, + ); + self.next_idx = idx; + } + Block::Large(block) => { + if idx > 0 { + self.next_key.clear(); + self.next_value_range = 0..0; + self.next_idx = idx; + return; + } + self.next_key = block.key.clone(); + self.next_value_range = 0..block.value_bytes.len(); + self.next_idx = idx; + } + } + } + + fn back_to_idx(&mut self, idx: isize) { + match self.block.as_ref() { + Block::Normal(block) => { + if idx < 0 { + self.prev_key.clear(); + self.prev_value_range = 0..0; + self.prev_idx = idx; + return; + } + let offset = block.offsets[idx as usize] as usize; + self.back_to_offset( + offset, + *block + .offsets + .get(idx as usize + 1) + .unwrap_or(&(block.data.len() as u16)) as usize, + idx == 0, + ); + self.prev_idx = idx; + } + Block::Large(block) => { + if idx < 0 { + self.prev_key.clear(); + self.prev_value_range = 0..0; + self.prev_idx = idx; + return; + } + self.prev_key = block.key.clone(); + self.prev_value_range = 0..block.value_bytes.len(); + self.prev_idx = idx; + } + } + } + + fn seek_to_offset(&mut self, offset: usize, offset_end: usize, is_first: bool) { + match self.block.as_ref() { + Block::Normal(block) => { + if is_first { + self.next_key = self.first_key.clone(); + self.next_value_range = offset..offset_end; + return; + } + let mut rest = &block.data[offset..]; + let common_prefix_len = rest.get_u8() as usize; + let key_suffix_len = rest.get_u16_le() as usize; + let mut next_key = Vec::with_capacity(common_prefix_len + key_suffix_len); + next_key.extend_from_slice(&self.first_key[..common_prefix_len]); + next_key.extend_from_slice(&rest[..key_suffix_len]); + self.next_key = next_key.into(); + let value_start = offset + SIZE_OF_U8 + SIZE_OF_U16 + key_suffix_len; + self.next_value_range = value_start..offset_end; + } + Block::Large(_) => { + unreachable!() + } + } + } + + fn back_to_offset(&mut self, offset: usize, offset_end: usize, is_first: bool) { + match self.block.as_ref() { + Block::Normal(block) => { + if is_first { + self.prev_key = self.first_key.clone(); + self.prev_value_range = offset..offset_end; + return; + } + let mut rest = &block.data[offset..]; + let common_prefix_len = rest.get_u8() as usize; + let key_suffix_len = rest.get_u16_le() as usize; + let mut prev_key = Vec::with_capacity(common_prefix_len + key_suffix_len); + prev_key.extend_from_slice(&self.first_key[..common_prefix_len]); + prev_key.extend_from_slice(&rest[..key_suffix_len]); + self.prev_key = prev_key.into(); + let value_start = offset + SIZE_OF_U8 + SIZE_OF_U16 + key_suffix_len; + self.prev_value_range = value_start..offset_end; + } + Block::Large(_) => { + unreachable!() + } + } + } + + pub fn peek_block(&self) -> &Arc { + &self.block + } + + pub fn finish(&mut self) { + self.next_key.clear(); + self.next_value_range = 0..0; + self.prev_key.clear(); + self.prev_value_range = 0..0; + } +} + +impl KvIterator for BlockIter { + fn peek_next_key(&self) -> Option { + self.peek_next_curr_key() + } + + fn peek_next_value(&self) -> Option { + self.peek_next_curr_value() + } + + fn next_(&mut self) { + self.next(); + } + + fn has_next(&self) -> bool { + self.has_next() + } + + fn peek_next_back_key(&self) -> Option { + self.peek_back_curr_key() + } + + fn peek_next_back_value(&self) -> Option { + self.peek_back_curr_value() + } + + fn next_back_(&mut self) { + self.next_back(); + } + + fn has_next_back(&self) -> bool { + self.has_next_back() + } +} + +impl Iterator for BlockIter { + type Item = (Bytes, Bytes); + + fn next(&mut self) -> Option { + if !self.has_next() { + return None; + } + let key = self.peek_next_curr_key().unwrap(); + let value = self.peek_next_curr_value().unwrap(); + self.next(); + Some((key, value)) + } +} + +impl DoubleEndedIterator for BlockIter { + fn next_back(&mut self) -> Option { + if !self.has_next_back() { + return None; + } + let key = self.peek_back_curr_key().unwrap(); + let value = self.peek_back_curr_value().unwrap(); + self.next_back(); + Some((key, value)) + } +} diff --git a/3rdparty/loro/crates/kv-store/src/compress.rs b/3rdparty/loro/crates/kv-store/src/compress.rs new file mode 100644 index 0000000000..db8f43336d --- /dev/null +++ b/3rdparty/loro/crates/kv-store/src/compress.rs @@ -0,0 +1,71 @@ +use std::io::{self, Write}; + +use bytes::Bytes; +use loro_common::LoroError; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CompressionType { + None, + LZ4, +} + +impl CompressionType { + pub fn is_none(&self) -> bool { + matches!(self, CompressionType::None) + } +} + +impl TryFrom for CompressionType { + type Error = LoroError; + + fn try_from(value: u8) -> Result { + match value { + 0 => Ok(CompressionType::None), + 1 => Ok(CompressionType::LZ4), + _ => Err(LoroError::DecodeError( + format!("Invalid compression type: {value}").into(), + )), + } + } +} + +impl From for u8 { + fn from(value: CompressionType) -> Self { + match value { + CompressionType::None => 0, + CompressionType::LZ4 => 1, + } + } +} + +pub fn compress(w: &mut Vec, data: &[u8], compression_type: CompressionType) { + match compression_type { + CompressionType::None => { + w.write_all(data).unwrap(); + } + CompressionType::LZ4 => { + let mut encoder = lz4_flex::frame::FrameEncoder::new(w); + encoder.write_all(data).unwrap(); + let _w = encoder.finish().unwrap(); + } + } +} + +pub fn decompress( + out: &mut Vec, + data: Bytes, + compression_type: CompressionType, +) -> Result<(), LoroError> { + match compression_type { + CompressionType::None => { + out.write_all(&data).unwrap(); + Ok(()) + } + CompressionType::LZ4 => { + let mut decoder = lz4_flex::frame::FrameDecoder::new(data.as_ref()); + io::copy(&mut decoder, out) + .map_err(|e| LoroError::DecodeError(e.to_string().into()))?; + Ok(()) + } + } +} diff --git a/3rdparty/loro/crates/kv-store/src/iter.rs b/3rdparty/loro/crates/kv-store/src/iter.rs new file mode 100644 index 0000000000..46db1a47ea --- /dev/null +++ b/3rdparty/loro/crates/kv-store/src/iter.rs @@ -0,0 +1,191 @@ +use std::fmt::Debug; + +use bytes::Bytes; + +/// When you need peek next key and value and next back key and value, use this trait. +pub trait KvIterator: Debug + DoubleEndedIterator { + fn peek_next_key(&self) -> Option; + fn peek_next_value(&self) -> Option; + fn next_(&mut self); + fn has_next(&self) -> bool; + fn peek_next_back_key(&self) -> Option; + fn peek_next_back_value(&self) -> Option; + fn next_back_(&mut self); + fn has_next_back(&self) -> bool; +} + +/// Merge multiple iterators into one. +/// +/// The iterators are merged in the order they are provided. +/// If two iterators have the same key, the value from the iterator with the smallest index is used. +/// +/// Note: This implementation is not optimized for lots of iterators. +/// You should only use this when you have a small number of iterators. +#[derive(Debug)] +pub struct MergeIterator { + iters: Vec, +} + +impl MergeIterator { + pub fn new(iters: Vec) -> Self { + Self { iters } + } +} + +impl Iterator for MergeIterator { + type Item = (Bytes, Bytes); + + fn next(&mut self) -> Option { + let mut min_key = None; + let mut min_index = None; + let mut has_to_remove = false; + for (i, iter) in self.iters.iter_mut().enumerate() { + if let Some(key) = iter.peek_next_key() { + if let Some(this_min_key) = &min_key { + match key.cmp(this_min_key) { + std::cmp::Ordering::Less => { + min_key = Some(key); + min_index = Some(i); + } + std::cmp::Ordering::Equal => { + // the same key, skip it + iter.next(); + } + std::cmp::Ordering::Greater => {} + } + } else { + min_key = Some(key); + min_index = Some(i); + } + } else { + has_to_remove = true; + } + } + + let ans = if let Some(idx) = min_index { + self.iters[idx].next() + } else { + None + }; + + if has_to_remove { + self.iters.retain(|x| x.has_next()); + } + ans + } +} + +impl DoubleEndedIterator for MergeIterator { + fn next_back(&mut self) -> Option { + let mut max_key = None; + let mut max_index = None; + let mut has_to_remove = false; + for (i, iter) in self.iters.iter_mut().enumerate() { + if let Some(key) = iter.peek_next_back_key() { + if let Some(this_max_key) = &max_key { + match key.cmp(this_max_key) { + std::cmp::Ordering::Less => {} + std::cmp::Ordering::Equal => { + // the same key, skip it + iter.next_back(); + } + std::cmp::Ordering::Greater => { + max_key = Some(key); + max_index = Some(i); + } + } + } else { + max_key = Some(key); + max_index = Some(i); + } + } else { + has_to_remove = true; + } + } + let ans = if let Some(idx) = max_index { + self.iters[idx].next_back() + } else { + None + }; + + if has_to_remove { + self.iters.retain(|x| x.has_next_back()); + } + ans + } +} + +#[cfg(test)] +mod tests { + use std::ops::Bound; + + use super::*; + use crate::{compress::CompressionType, sstable}; + use bytes::Bytes; + #[test] + fn test_merge_iterator() { + let a = Bytes::from("a"); + let b = Bytes::from("b"); + let c = Bytes::from("c"); + let d = Bytes::from("d"); + + let mut sstable1 = sstable::SsTableBuilder::new(10, CompressionType::LZ4, true); + sstable1.add(a.clone(), a.clone()); + sstable1.add(c.clone(), c.clone()); + let sstable1 = sstable1.build(); + let iter1 = sstable::SsTableIter::new_scan(&sstable1, Bound::Unbounded, Bound::Unbounded); + + let mut sstable2 = sstable::SsTableBuilder::new(10, CompressionType::LZ4, true); + sstable2.add(b.clone(), b.clone()); + sstable2.add(d.clone(), d.clone()); + let sstable2 = sstable2.build(); + let iter2 = sstable::SsTableIter::new_scan(&sstable2, Bound::Unbounded, Bound::Unbounded); + + let merged_iter = MergeIterator::new(vec![iter1.clone(), iter2.clone()]); + let ans = merged_iter.collect::>(); + assert_eq!( + ans, + vec![ + (a.clone(), a.clone()), + (b.clone(), b.clone()), + (c.clone(), c.clone()), + (d.clone(), d.clone()) + ] + ); + + let merged_iter = MergeIterator::new(vec![iter1.clone(), iter2.clone()]); + let ans2 = merged_iter.rev().collect::>(); + assert_eq!(ans2, ans.iter().rev().cloned().collect::>()); + } + + #[test] + fn same_key() { + let a = Bytes::from("a"); + let a2 = Bytes::from("a2"); + let c = Bytes::from("c"); + let d = Bytes::from("d"); + + let mut sstable1 = sstable::SsTableBuilder::new(10, CompressionType::LZ4, true); + sstable1.add(a.clone(), a.clone()); + sstable1.add(c.clone(), c.clone()); + let sstable1 = sstable1.build(); + let iter1 = sstable::SsTableIter::new_scan(&sstable1, Bound::Unbounded, Bound::Unbounded); + + let mut sstable2 = sstable::SsTableBuilder::new(10, CompressionType::LZ4, true); + sstable2.add(a.clone(), a2.clone()); + sstable2.add(d.clone(), d.clone()); + let sstable2 = sstable2.build(); + let iter2 = sstable::SsTableIter::new_scan(&sstable2, Bound::Unbounded, Bound::Unbounded); + + let merged_iter = MergeIterator::new(vec![iter1.clone(), iter2.clone()]); + let ans = merged_iter.collect::>(); + assert_eq!( + ans, + vec![ + (a.clone(), a.clone()), + (c.clone(), c.clone()), + (d.clone(), d.clone()) + ] + ); + } +} diff --git a/3rdparty/loro/crates/kv-store/src/lib.rs b/3rdparty/loro/crates/kv-store/src/lib.rs new file mode 100644 index 0000000000..2b9a629282 --- /dev/null +++ b/3rdparty/loro/crates/kv-store/src/lib.rs @@ -0,0 +1,143 @@ +//! # MemKvStore Documentation +//! +//! MemKvStore use SSTable as backend. The SSTable (Sorted String Table) is a persistent data structure +//! used for storing key-value pairs in a sorted manner. This document describes the binary format of +//! the SSTable. +//! +//! ## Overall Structure +//! +//! The SSTable consists of the following sections: +//! +//! ┌─────────────────────────────────────────────────────────────────────────────────────────────────┐ +//! │ MemKVStore │ +//! │┌ ─ ─ ─ ─ ─ ─ ─┌ ─ ─ ─ ─ ─ ─ ─ ─┌ ─ ─ ─ ─ ─ ─ ┬ ─ ─ ─┌ ─ ─ ─ ─ ─ ─ ─ ┬ ─ ─ ─ ─ ─ ─┌ ─ ─ ─ ─ ─ ─ ┐│ +//! │ Magic Number │ Schema Version │ Block Chunk ... │ Block Chunk Block Meta │ Meta Offset │ +//! ││ u32 │ u8 │ bytes │ │ bytes │ bytes │ u32 ││ +//! │ ─ ─ ─ ─ ─ ─ ─ ┘─ ─ ─ ─ ─ ─ ─ ─ ┘─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┘─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┘─ ─ ─ ─ ─ ─ ─ │ +//! └─────────────────────────────────────────────────────────────────────────────────────────────────┘ +//! +//! 1. Magic Number (4 bytes): A fixed value "LORO" to identify the file format. +//! 2. Schema Version (1 byte): The version of the MemKVStore schema. +//! 3. Block Chunks: A series of data blocks containing key-value pairs. +//! 4. Block Meta: Metadata for all blocks, including block offset, the first key of the block, `is_large` flag, and last key +//! if not large. +//! 5. Meta Offset (4 bytes): The offset of the Block Meta section from the beginning of the file. +//! +//! ## Block Types +//! +//! There are two types of blocks: Normal Blocks and Large Value Blocks. +//! +//! ### Normal Block +//! +//! Normal blocks store multiple key-value pairs with compressed keys. +//! +//! ┌────────────────────────────────────────────────────────────────────────────────────────────┐ +//! │Block │ +//! │┌ ─ ─ ─ ─ ─ ─ ─ ┬ ─ ─ ─┌ ─ ─ ─ ─ ─ ─ ─ ┬ ─ ─ ─ ─┌ ─ ─ ─┌ ─ ─ ─ ┬ ─ ─ ─ ─┌ ─ ─ ─ ─ ─ ─ ─ │ +//! │ Key Value Chunk ... │Key Value Chunk offset │ ... │ offset kv len │Block Checksum│ │ +//! ││ bytes │ │ bytes │ u16 │ │ u16 │ u16 │ u32 │ +//! │ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┘─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┘─ ─ ─ ┘─ ─ ─ ─ ─ ─ ─ ─ ┘─ ─ ─ ─ ─ ─ ─ ┘ │ +//! └────────────────────────────────────────────────────────────────────────────────────────────┘ +//! +//! Each Key Value Chunk is encoded as follows: +//! +//! ┌─────────────────────────────────────────────────────┐ +//! │ Key Value Chunk │ +//! │┌ ─ ─ ─ ─ ─ ─ ─ ─ ┬ ─ ─ ─ ─ ─ ─ ─┌ ─ ─ ─ ─ ─┬ ─ ─ ─ ┐│ +//! │ common prefix len key suffix len│key suffix│ value ││ +//! ││ u8 │ u16 │ bytes │ bytes ││ +//! │ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┘─ ─ ─ ─ ─ ┘─ ─ ─ ─┘│ +//! └─────────────────────────────────────────────────────┘ +//! +//! Because the key of the first chunk is equal to the first key of the block, +//! the first chunk can be simplified as: +//! ┌────────────────────┐ +//! │ Key Value Chunk │ +//! │┌ ─ ─ ─ ─ ─┬ ─ ─ ─ ┐│ +//! ││key suffix│ value ││ +//! ││ bytes │ bytes ││ +//! │ ─ ─ ─ ─ ─ ┘─ ─ ─ ─┘│ +//! └────────────────────┘ +//! +//! Encoding: +//! 1. Compress key-value pairs data as Key Value Chunk. +//! 2. Write offsets for each key-value pair. +//! 3. Write the number of key-value pairs. +//! 4. By default, **Compress** the entire block using LZ4. If you set `compression_type` to `None`, it will not compress the block. +//! - For now, there are two compression type: `None` and `LZ4`. +//! 5. Calculate and append xxhash_32 checksum. +//! +//! Decoding: +//! 1. Verify the xxhash_32 checksum. +//! 2. By default, **Decompress** the block using LZ4. If you set `compression_type` to `None`, it will not decompress the block. +//! 3. Read the number of key-value pairs. +//! 4. Read offsets for each key-value pair. +//! 5. Parse individual key-value chunks. +//! +//! ### Large Value Block +//! +//! Large Value Blocks store a single key-value pair with a large value. +//! +//! ┌──────────────────────────┐ +//! │Large Block │ +//! │┌ ─ ─ ─ ┬ ─ ─ ─ ─ ─ ─ ─ ─ │ +//! │ value Block Checksum ││ +//! ││ bytes │ u32 │ +//! │ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┘│ +//! └──────────────────────────┘ +//! +//! Encoding: +//! 1. Write the value bytes. +//! 2. Calculate and append xxhash_32 checksum. +//! +//! Decoding: +//! 1. Verify the xxhash_32 checksum. +//! 2. Read the value bytes. +//! +//! We need not encode the length of value, because we can get the whole Block by offset in meta. +//! +//! ## Block Meta +//! +//! The Block Meta section contains metadata for all blocks in the SSTable. +//! +//! ┌────────────────────────────────────────────────────────────┐ +//! │ All Block Meta │ +//! │┌ ─ ─ ─ ─ ─ ─ ─┌ ─ ─ ─ ─ ─ ─┌ ─ ─ ─┌ ─ ─ ─ ─ ─ ─┌ ─ ─ ─ ─ ─ │ +//! │ block number │ Block Meta │ ... │ Block Meta │ checksum ││ +//! ││ u32 │ bytes │ │ bytes │ u32 │ +//! │ ─ ─ ─ ─ ─ ─ ─ ┘─ ─ ─ ─ ─ ─ ┘─ ─ ─ ┘─ ─ ─ ─ ─ ─ ┘─ ─ ─ ─ ─ ┘│ +//! └────────────────────────────────────────────────────────────┘ +//! +//! Each Block Meta entry is encoded as follows: +//! +//! ┌──────────────────────────────────────────────────────────────────────────────────────────┐ +//! │ Block Meta │ +//! │┌ ─ ─ ─ ─ ─ ─ ─┌ ─ ─ ─ ─ ─ ─ ─ ┬ ─ ─ ─ ─ ─ ┬ ─ ─ ─ ─ ─ ─ ─┌ ─ ─ ─ ─ ─ ─ ─ ┬ ─ ─ ─ ─ ─ ─ ┐ │ +//! │ block offset │ first key len first key block type │ last key len last key │ +//! ││ u32 │ u16 │ bytes │ u8 │ u16(option) │bytes(option)│ │ +//! │ ─ ─ ─ ─ ─ ─ ─ ┘─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┘─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ │ +//! └──────────────────────────────────────────────────────────────────────────────────────────┘ +//! +//! Encoding: +//! 1. Write the number of blocks. +//! 2. For each block, write its metadata (offset, first key, block type, and last key if not large). +//! - block type: the first bit for is_large, the next 7 bits for compression_type. +//! 3. Calculate and append xxhash_32 checksum. +//! +//! Decoding: +//! 1. Read the number of blocks. +//! 2. For each block, read its metadata. +//! 3. Verify the xxhash_32 checksum. +//! +//! +//! Note: In this crate, the empty value is regarded as deleted. **only** [MemStoreIterator] will filter empty value. +//! Other iterators will still return empty value. +#![allow(clippy::uninlined_format_args)] +pub mod block; +pub mod compress; +pub mod iter; +pub mod mem_store; +pub mod sstable; +mod utils; +pub use iter::{KvIterator, MergeIterator}; +pub use mem_store::{MemKvStore, MemStoreIterator}; diff --git a/3rdparty/loro/crates/kv-store/src/mem_store.rs b/3rdparty/loro/crates/kv-store/src/mem_store.rs new file mode 100644 index 0000000000..5feefb49c1 --- /dev/null +++ b/3rdparty/loro/crates/kv-store/src/mem_store.rs @@ -0,0 +1,736 @@ +use crate::block::BlockIter; +use crate::compress::CompressionType; +use crate::sstable::{SsTable, SsTableBuilder, SsTableIter}; +use crate::{KvIterator, MergeIterator}; +use bytes::Bytes; + +use std::ops::Bound; +use std::{cmp::Ordering, collections::BTreeMap}; + +#[derive(Debug, Clone)] +pub struct MemKvStore { + mem_table: BTreeMap, + // From the oldest to the newest + ss_table: Vec, + block_size: usize, + compression_type: CompressionType, + /// It's only true when using it to fuzz. + /// Otherwise, importing and exporting GC snapshot relies on this field being false to work. + should_encode_none: bool, +} + +pub struct MemKvConfig { + block_size: usize, + compression_type: CompressionType, + should_encode_none: bool, +} + +impl Default for MemKvConfig { + fn default() -> Self { + Self { + block_size: MemKvStore::DEFAULT_BLOCK_SIZE, + compression_type: CompressionType::LZ4, + should_encode_none: false, + } + } +} + +impl MemKvConfig { + pub fn new() -> Self { + Self::default() + } + + pub fn block_size(mut self, block_size: usize) -> Self { + self.block_size = block_size; + self + } + + pub fn compression_type(mut self, compression_type: CompressionType) -> Self { + self.compression_type = compression_type; + self + } + + pub fn should_encode_none(mut self, should_encode_none: bool) -> Self { + self.should_encode_none = should_encode_none; + self + } + + pub fn build(self) -> MemKvStore { + MemKvStore::new(self) + } +} + +impl MemKvStore { + pub const DEFAULT_BLOCK_SIZE: usize = 4 * 1024; + pub fn new(config: MemKvConfig) -> Self { + Self { + mem_table: BTreeMap::new(), + ss_table: Vec::new(), + block_size: config.block_size, + compression_type: config.compression_type, + should_encode_none: config.should_encode_none, + } + } + + pub fn get(&self, key: &[u8]) -> Option { + if let Some(v) = self.mem_table.get(key) { + if v.is_empty() { + return None; + } + return Some(v.clone()); + } + + for table in self.ss_table.iter().rev() { + if table.first_key > key || table.last_key < key { + continue; + } + // table. + let idx = table.find_block_idx(key); + let block = table.read_block_cached(idx); + let block_iter = BlockIter::new_seek_to_key(block, key); + if let Some(k) = block_iter.peek_next_curr_key() { + let v = block_iter.peek_next_curr_value().unwrap(); + if k == key { + return if v.is_empty() { None } else { Some(v) }; + } + } + } + None + } + + pub fn set(&mut self, key: &[u8], value: Bytes) { + if key.is_empty() { + return; + } + + self.mem_table.insert(Bytes::copy_from_slice(key), value); + } + + pub fn compare_and_swap(&mut self, key: &[u8], old: Option, new: Bytes) -> bool { + if key.is_empty() { + return false; + } + + match self.get(key) { + Some(v) => { + if old == Some(v) { + self.set(key, new); + true + } else { + false + } + } + None => { + if old.is_none() { + self.set(key, new); + true + } else { + false + } + } + } + } + + pub fn remove(&mut self, key: &[u8]) { + self.set(key, Bytes::new()); + } + + /// Check if the key exists in the mem table or the sstable + /// + /// If the value is empty, it means the key is deleted + pub fn contains_key(&self, key: &[u8]) -> bool { + if key.is_empty() { + return false; + } + + if self.mem_table.contains_key(key) { + return !self.mem_table.get(key).unwrap().is_empty(); + } + + for table in self.ss_table.iter().rev() { + if table.contains_key(key) { + if let Some(v) = table.get(key) { + return !v.is_empty(); + } + } + } + false + } + + pub fn scan( + &self, + start: std::ops::Bound<&[u8]>, + end: std::ops::Bound<&[u8]>, + ) -> Box + '_> { + if self.ss_table.is_empty() { + return Box::new( + self.mem_table + .range::<[u8], _>((start, end)) + .filter(|(_, v)| !v.is_empty()) + .map(|(k, v)| (k.clone(), v.clone())), + ); + } + + Box::new(MemStoreIterator::new( + self.mem_table + .range::<[u8], _>((start, end)) + .map(|(k, v)| (k.clone(), v.clone())), + MergeIterator::new( + self.ss_table + .iter() + .rev() + .map(|table| SsTableIter::new_scan(table, start, end)) + .collect(), + ), + true, + )) + } + + /// The number of valid keys in the mem table and sstable, it's expensive to call + pub fn len(&self) -> usize { + // TODO: PERF + self.scan(Bound::Unbounded, Bound::Unbounded).count() + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + pub fn size(&self) -> usize { + self.mem_table + .iter() + .fold(0, |acc, (k, v)| acc + k.len() + v.len()) + + self + .ss_table + .iter() + .map(|table| table.data_size()) + .sum::() + } + + pub fn export_all(&mut self) -> Bytes { + if self.mem_table.is_empty() && self.ss_table.len() == 1 { + return self.ss_table[0].export_all(); + } + + if self.ss_table.len() == 1 { + return self.export_with_encoded_block(); + } + + let mut builder = SsTableBuilder::new( + self.block_size, + self.compression_type, + self.should_encode_none, + ); + // we could use scan() here, we should keep the empty value + let iter = MemStoreIterator::new( + self.mem_table + .range::<[u8], _>((Bound::Unbounded, Bound::Unbounded)) + .map(|(k, v)| (k.clone(), v.clone())), + MergeIterator::new( + self.ss_table + .iter() + .rev() + .map(|table| SsTableIter::new_scan(table, Bound::Unbounded, Bound::Unbounded)) + .collect(), + ), + false, + ); + + for (k, v) in iter { + builder.add(k, v); + } + + if builder.is_empty() { + return Bytes::new(); + } + self.mem_table.clear(); + let ss = builder.build(); + let ans = ss.export_all(); + let _ = std::mem::replace(&mut self.ss_table, vec![ss]); + ans + } + + /// Import a serialized store, verifying every block's checksum. Can be + /// called multiple times; later imports override earlier keys. + pub fn import_all(&mut self, bytes: Bytes) -> Result<(), String> { + if bytes.is_empty() { + return Ok(()); + } + + let ss_table = SsTable::import_all(bytes, false).map_err(|e| e.to_string())?; + self.ss_table.push(ss_table); + Ok(()) + } + + /// Like [`Self::import_all`] but skips per-block checksum verification. + /// + /// Only use this when the blob's integrity is already guaranteed by an outer + /// checksum (e.g. Loro's document-level snapshot checksum, verified before + /// the snapshot body is handed to the KV store). + pub fn import_all_unchecked(&mut self, bytes: Bytes) -> Result<(), String> { + if bytes.is_empty() { + return Ok(()); + } + + let ss_table = SsTable::import_all_unchecked(bytes).map_err(|e| e.to_string())?; + self.ss_table.push(ss_table); + Ok(()) + } + + #[tracing::instrument(level = "debug", skip(self))] + fn export_with_encoded_block(&mut self) -> Bytes { + ensure_cov::notify_cov("kv-store::mem_store::export_with_encoded_block"); + let mut mem_iter = self.mem_table.iter().peekable(); + let mut sstable_iter = self.ss_table[0].iter(); + let mut builder = SsTableBuilder::new( + self.block_size, + self.compression_type, + self.should_encode_none, + ); + 'outer: while let Some(next_mem_pair) = mem_iter.peek() { + let block = loop { + let Some(block) = sstable_iter.peek_next_block() else { + builder.add(next_mem_pair.0.clone(), next_mem_pair.1.clone()); + mem_iter.next(); + continue 'outer; + }; + if block.last_key() < next_mem_pair.0 { + builder.add_new_block(block.clone()); + sstable_iter.next_block(); + continue; + } + break block; + }; + + if block.first_key() > next_mem_pair.0 { + builder.add(next_mem_pair.0.clone(), next_mem_pair.1.clone()); + mem_iter.next(); + continue; + } + + // There are overlap between next_mem_pair and block + let mut iter = BlockIter::new(block.clone()); + let mut next_mem_pair = mem_iter.peek(); + while let Some(k) = iter.peek_next_key() { + loop { + match next_mem_pair { + Some(next_mem_pair_inner) => { + if k > next_mem_pair_inner.0 { + builder.add( + next_mem_pair_inner.0.clone(), + next_mem_pair_inner.1.clone(), + ); + mem_iter.next(); + next_mem_pair = mem_iter.peek(); + continue; + } + if k == next_mem_pair_inner.0 { + builder.add(k, next_mem_pair_inner.1.clone()); + mem_iter.next(); + next_mem_pair = mem_iter.peek(); + iter.next(); + break; + } + // k < next_mem_pair_inner.0 + builder.add(k, iter.peek_next_value().unwrap()); + iter.next(); + break; + } + None => { + builder.add(k, iter.peek_next_value().unwrap()); + iter.next(); + break; + } + } + } + } + + sstable_iter.next_block(); + } + + while let Some(block) = sstable_iter.peek_next_block() { + builder.add_new_block(block.clone()); + sstable_iter.next_block(); + } + + if builder.is_empty() { + return Bytes::new(); + } + + drop(mem_iter); + self.mem_table.clear(); + let ss = builder.build(); + let ans = ss.export_all(); + let _ = std::mem::replace(&mut self.ss_table, vec![ss]); + ans + } + + #[allow(unused)] + fn check_encode_data_correctness(&self, bytes: &Bytes) { + let this_data: BTreeMap = + self.scan(Bound::Unbounded, Bound::Unbounded).collect(); + let mut other_kv = MemKvStore::new(Default::default()); + other_kv.import_all(bytes.clone()).unwrap(); + let other_data: BTreeMap = + other_kv.scan(Bound::Unbounded, Bound::Unbounded).collect(); + assert_eq!(this_data, other_data); + } +} + +#[derive(Debug)] +pub struct MemStoreIterator { + mem: T, + sst: S, + current_mem: Option<(Bytes, Bytes)>, + current_sstable: Option<(Bytes, Bytes)>, + back_mem: Option<(Bytes, Bytes)>, + back_sstable: Option<(Bytes, Bytes)>, + filter_empty: bool, +} + +impl MemStoreIterator +where + T: DoubleEndedIterator, + S: DoubleEndedIterator, +{ + fn new(mut mem: T, sst: S, filter_empty: bool) -> Self { + let current_mem = mem.next(); + let back_mem = mem.next_back(); + Self { + mem, + sst, + current_mem, + back_mem, + current_sstable: None, + back_sstable: None, + filter_empty, + } + } +} + +impl Iterator for MemStoreIterator +where + T: DoubleEndedIterator, + S: DoubleEndedIterator, +{ + type Item = (Bytes, Bytes); + fn next(&mut self) -> Option { + loop { + if self.current_sstable.is_none() { + if let Some((k, v)) = self.sst.next() { + self.current_sstable = Some((k, v)); + } + } + + if self.current_mem.is_none() && self.back_mem.is_some() { + std::mem::swap(&mut self.back_mem, &mut self.current_mem); + } + let ans = match (&self.current_mem, &self.current_sstable) { + (Some((mem_key, _)), Some((iter_key, _))) => match mem_key.cmp(iter_key) { + Ordering::Less => self.current_mem.take().inspect(|_kv| { + self.current_mem = self.mem.next(); + }), + Ordering::Equal => { + self.current_sstable.take(); + self.current_mem.take().inspect(|_kv| { + self.current_mem = self.mem.next(); + }) + } + Ordering::Greater => self.current_sstable.take(), + }, + (Some(_), None) => self.current_mem.take().inspect(|_kv| { + self.current_mem = self.mem.next(); + }), + (None, Some(_)) => self.current_sstable.take(), + (None, None) => None, + }; + + if self.filter_empty { + if let Some((_k, v)) = &ans { + if v.is_empty() { + continue; + } + } + } + + return ans; + } + } +} + +impl DoubleEndedIterator for MemStoreIterator +where + T: DoubleEndedIterator, + S: DoubleEndedIterator, +{ + fn next_back(&mut self) -> Option { + if self.back_sstable.is_none() { + if let Some((k, v)) = self.sst.next_back() { + self.back_sstable = Some((k, v)); + } + } + + if self.back_mem.is_none() && self.current_mem.is_some() { + std::mem::swap(&mut self.back_mem, &mut self.current_mem); + } + + let ans = match (&self.back_mem, &self.back_sstable) { + (Some((mem_key, _)), Some((iter_key, _))) => match mem_key.cmp(iter_key) { + Ordering::Greater => self.back_mem.take().inspect(|_kv| { + self.back_mem = self.mem.next_back(); + }), + Ordering::Equal => { + self.back_sstable.take(); + self.back_mem.take().inspect(|_kv| { + self.back_mem = self.mem.next_back(); + }) + } + Ordering::Less => self.back_sstable.take(), + }, + (Some(_), None) => self.back_mem.take().inspect(|_kv| { + self.back_mem = self.mem.next_back(); + }), + (None, Some(_)) => self.back_sstable.take(), + (None, None) => None, + }; + if self.filter_empty { + if let Some((_k, v)) = &ans { + if v.is_empty() { + return self.next_back(); + } + } + } + ans + } +} + +#[cfg(test)] +mod tests { + use std::vec; + + use crate::{mem_store::MemKvConfig, MemKvStore}; + use bytes::Bytes; + #[test] + fn test_mem_kv_store() { + let key = &[0]; + let value = Bytes::from_static(&[0]); + + let key2 = &[0, 1]; + let value2 = Bytes::from_static(&[0, 1]); + let mut store = new_store(); + store.set(key, value.clone()); + assert_eq!(store.get(key), Some(value)); + store.remove(key); + assert!(store.is_empty()); + assert_eq!(store.get(key), None); + store.compare_and_swap(key, None, value2.clone()); + assert_eq!(store.get(key), Some(value2.clone())); + assert!(store.contains_key(key)); + assert!(!store.contains_key(key2)); + + store.set(key2, value2.clone()); + assert_eq!(store.get(key2), Some(value2.clone())); + assert_eq!(store.len(), 2); + assert_eq!(store.size(), 7); + let bytes = store.export_all(); + let mut new_store = new_store(); + assert_eq!(new_store.len(), 0); + assert_eq!(new_store.size(), 0); + new_store.import_all(bytes).unwrap(); + + let iter1 = store + .scan(std::ops::Bound::Unbounded, std::ops::Bound::Unbounded) + .collect::>(); + let iter2 = new_store + .scan(std::ops::Bound::Unbounded, std::ops::Bound::Unbounded) + .collect::>(); + assert_eq!(iter1, iter2); + + let iter1 = store + .scan(std::ops::Bound::Unbounded, std::ops::Bound::Unbounded) + .rev() + .collect::>(); + let iter2 = new_store + .scan(std::ops::Bound::Unbounded, std::ops::Bound::Unbounded) + .rev() + .collect::>(); + assert_eq!(iter1, iter2); + } + + #[test] + fn test_large_block() { + let mut store = new_store(); + let key = &[0]; + let value = Bytes::from_static(&[0]); + + let key2 = &[0, 1]; + let key3 = &[0, 1, 2]; + let large_value = Bytes::from_iter([0; 1024 * 8]); + let large_value2 = Bytes::from_iter([0; 1024 * 8]); + store.set(key, value.clone()); + store.set(key2, large_value.clone()); + let v2 = store.get(&[]); + assert_eq!(v2, None); + assert_eq!(store.get(key), Some(value.clone())); + assert_eq!(store.get(key2), Some(large_value.clone())); + store.export_all(); + store.set(key3, large_value2.clone()); + assert_eq!(store.get(key3), Some(large_value2.clone())); + assert_eq!(store.len(), 3); + + let iter = store + .scan(std::ops::Bound::Unbounded, std::ops::Bound::Unbounded) + .collect::>(); + assert_eq!( + iter, + vec![ + (Bytes::from_static(key), value.clone()), + (Bytes::from_static(key2), large_value.clone()), + (Bytes::from_static(key3), large_value2.clone()) + ] + ); + + let iter2 = store + .scan( + std::ops::Bound::Included(key), + std::ops::Bound::Included(key3), + ) + .collect::>(); + assert_eq!(iter, iter2); + + let iter3 = store + .scan( + std::ops::Bound::Excluded(key), + std::ops::Bound::Excluded(key3), + ) + .collect::>(); + assert_eq!(iter3.len(), 1); + assert_eq!(iter3[0], (Bytes::from_static(key2), large_value.clone())); + + let v = store.get(key2).unwrap(); + assert_eq!(v, large_value); + + let v2 = store.get(&[]); + assert_eq!(v2, None); + + store.compare_and_swap(key, Some(value.clone()), large_value.clone()); + assert!(store.contains_key(key)); + } + + #[test] + fn empty_key_mutations_are_ignored() { + let mut store = new_store(); + let value = Bytes::from_static(b"value"); + + store.set(&[], value.clone()); + assert_eq!(store.get(&[]), None); + assert!(!store.contains_key(&[])); + assert_eq!(store.len(), 0); + assert!(store.is_empty()); + assert!(!store.compare_and_swap(&[], None, value.clone())); + store.remove(&[]); + + let bytes = store.export_all(); + assert!(bytes.is_empty()); + + let mut imported = new_store(); + imported.import_all(bytes).unwrap(); + assert_eq!(imported.get(&[]), None); + assert_eq!(imported.len(), 0); + } + + #[test] + fn same_key() { + let mut store = new_store(); + let key = &[0]; + let value = Bytes::from_static(&[0]); + store.set(key, value.clone()); + store.export_all(); + store.set(key, Bytes::new()); + assert_eq!(store.get(key), None); + let iter = store + .scan(std::ops::Bound::Unbounded, std::ops::Bound::Unbounded) + .collect::>(); + assert_eq!(iter.len(), 0); + store.set(key, value.clone()); + assert_eq!(store.get(key), Some(value)); + } + + #[test] + fn import_several_times() { + dev_utils::setup_test_log(); + let a = Bytes::from_static(b"a"); + let b = Bytes::from_static(b"b"); + let c = Bytes::from_static(b"c"); + let d = Bytes::from_static(b"d"); + let e = Bytes::from_static(b"e"); + let mut store = new_store(); + store.set(&a, a.clone()); + store.export_all(); + store.set(&c, c.clone()); + let encode1 = store.export_all(); + let mut store2 = new_store(); + store2.set(&b, b.clone()); + store2.export_all(); + store2.set(&c, Bytes::new()); + let encode2 = store2.export_all(); + let mut store3 = new_store(); + store3.set(&d, d.clone()); + store3.set(&a, Bytes::new()); + tracing::info_span!("export da").in_scope(|| { + store3.export_all(); + }); + store3.set(&e, e.clone()); + store3.set(&c, c.clone()); + let encode3 = tracing::info_span!("export ec").in_scope(|| store3.export_all()); + + let mut store = new_store(); + store.import_all(encode1).unwrap(); + store.import_all(encode2).unwrap(); + store.import_all(encode3).unwrap(); + assert_eq!(store.get(&a), None); + assert_eq!(store.get(&b), Some(b.clone())); + assert_eq!(store.get(&c), Some(c.clone())); + assert_eq!(store.get(&d), Some(d.clone())); + assert_eq!(store.get(&e), Some(e.clone())); + } + + #[test] + fn large_config_splits_normal_blocks_before_u16_offsets_overflow() { + let mut store = MemKvStore::new( + MemKvConfig::default() + .block_size(usize::from(u16::MAX) * 2) + .should_encode_none(true), + ); + let a = Bytes::from(vec![1; 40_000]); + let b = Bytes::from(vec![2; 40_000]); + store.set(b"a", a.clone()); + store.set(b"b", b.clone()); + + let bytes = store.export_all(); + let mut imported = new_store(); + imported.import_all(bytes).unwrap(); + assert_eq!(imported.get(b"a"), Some(a)); + assert_eq!(imported.get(b"b"), Some(b)); + } + + #[test] + fn large_config_uses_large_block_for_values_bigger_than_normal_offsets() { + let mut store = MemKvStore::new( + MemKvConfig::default() + .block_size(usize::from(u16::MAX) * 2) + .should_encode_none(true), + ); + let value = Bytes::from(vec![7; usize::from(u16::MAX) + 1]); + store.set(b"large", value.clone()); + + let bytes = store.export_all(); + let mut imported = new_store(); + imported.import_all(bytes).unwrap(); + assert_eq!(imported.get(b"large"), Some(value)); + } + + fn new_store() -> MemKvStore { + MemKvStore::new(MemKvConfig::default().should_encode_none(true)) + } +} diff --git a/3rdparty/loro/crates/kv-store/src/sstable.rs b/3rdparty/loro/crates/kv-store/src/sstable.rs new file mode 100644 index 0000000000..796fc9ec31 --- /dev/null +++ b/3rdparty/loro/crates/kv-store/src/sstable.rs @@ -0,0 +1,1427 @@ +use super::block::BlockIter; +use crate::{ + block::{Block, BlockBuilder}, + compress::CompressionType, + iter::KvIterator, + utils::{get_u16_le, get_u32_le, get_u8_le}, +}; +use bytes::{Buf, BufMut, Bytes}; +use ensure_cov::*; +use loro_common::{LoroError, LoroResult}; +use std::{fmt::Debug, ops::Bound, sync::Arc}; + +pub(crate) const XXH_SEED: u32 = u32::from_le_bytes(*b"LORO"); +const MAGIC_BYTES: [u8; 4] = *b"LORO"; +const CURRENT_SCHEMA_VERSION: u8 = 0; +pub const SIZE_OF_U8: usize = std::mem::size_of::(); +pub const SIZE_OF_U16: usize = std::mem::size_of::(); +pub const SIZE_OF_U32: usize = std::mem::size_of::(); +// TODO: cache size +const DEFAULT_CACHE_SIZE: usize = 1 << 20; +const MAX_BLOCK_NUM: u32 = 10_000_000; + +/// ```log +/// ┌──────────────────────────────────────────────────────────────────────────────────────────┐ +/// │ Block Meta │ +/// │┌ ─ ─ ─ ─ ─ ─ ─┌ ─ ─ ─ ─ ─ ─ ─ ┬ ─ ─ ─ ─ ─ ┬ ─ ─ ─ ─ ─ ─ ─┌ ─ ─ ─ ─ ─ ─ ─ ┬ ─ ─ ─ ─ ─ ─ ┐ │ +/// │ block offset │ first key len first key block type │ last key len last key │ +/// ││ u32 │ u16 │ bytes │ u8 │ u16(option) │bytes(option)│ │ +/// │ ─ ─ ─ ─ ─ ─ ─ ┘─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┘─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ │ +/// └──────────────────────────────────────────────────────────────────────────────────────────┘ +/// ``` +#[derive(Debug, Clone)] +pub(crate) struct BlockMeta { + offset: usize, + is_large: bool, + compression_type: CompressionType, + first_key: Bytes, + last_key: Option, +} + +impl BlockMeta { + /// ┌────────────────────────────────────────────────────────────┐ + /// │ All Block Meta │ + /// │┌ ─ ─ ─ ─ ─ ─ ─┌ ─ ─ ─ ─ ─ ─┌ ─ ─ ─┌ ─ ─ ─ ─ ─ ─┌ ─ ─ ─ ─ ─ │ + /// │ block length │ Block Meta │ ... │ Block Meta │ checksum ││ + /// ││ u32 │ bytes │ │ bytes │ u32 │ + /// │ ─ ─ ─ ─ ─ ─ ─ ┘─ ─ ─ ─ ─ ─ ┘─ ─ ─ ┘─ ─ ─ ─ ─ ─ ┘─ ─ ─ ─ ─ ┘│ + /// └────────────────────────────────────────────────────────────┘ + fn encode_meta(meta: &[BlockMeta], buf: &mut Vec) { + // the number of blocks + let mut estimated_size = SIZE_OF_U32; + for m in meta { + // offset + estimated_size += SIZE_OF_U32; + // first key length + estimated_size += SIZE_OF_U16; + // first key + estimated_size += m.first_key.len(); + // is large + estimated_size += SIZE_OF_U8; + if m.is_large { + continue; + } + // last key length + estimated_size += SIZE_OF_U16; + // last key + estimated_size += m.last_key.as_ref().unwrap().len(); + } + // checksum + estimated_size += SIZE_OF_U32; + + buf.reserve(estimated_size); + let ori_length = buf.len(); + buf.put_u32_le(meta.len() as u32); + for m in meta { + buf.put_u32_le(m.offset as u32); + buf.put_u16_le(m.first_key.len() as u16); + buf.put_slice(&m.first_key); + let large_and_compress = ((m.is_large as u8) << 7) | m.compression_type as u8; + buf.put_u8(large_and_compress); + if m.is_large { + continue; + } + buf.put_u16_le(m.last_key.as_ref().unwrap().len() as u16); + buf.put_slice(m.last_key.as_ref().unwrap()); + } + let checksum = xxhash_rust::xxh32::xxh32(&buf[ori_length + 4..], XXH_SEED); + buf.put_u32_le(checksum); + } + + fn decode_meta(data: &[u8]) -> LoroResult> { + let (num, mut data) = get_u32_le(data)?; + if num > MAX_BLOCK_NUM { + return Err(LoroError::DecodeError("Invalid bytes".into())); + } + let meta_payload_len = data + .len() + .checked_sub(SIZE_OF_U32) + .ok_or_else(|| LoroError::DecodeError("Invalid bytes".into()))?; + let min_meta_entry_len = SIZE_OF_U32 + SIZE_OF_U16 + SIZE_OF_U8; + if (num as usize) + .checked_mul(min_meta_entry_len) + .is_none_or(|min_len| min_len > meta_payload_len) + { + return Err(LoroError::DecodeError("Invalid bytes".into())); + } + let mut ans = Vec::with_capacity(num as usize); + let checksum = xxhash_rust::xxh32::xxh32(&data[..data.len() - SIZE_OF_U32], XXH_SEED); + for _ in 0..num { + let (offset, buf) = get_u32_le(data)?; + let (first_key_len, mut buf) = get_u16_le(buf)?; + if buf.len() < first_key_len as usize { + return Err(LoroError::DecodeError("Invalid bytes".into())); + } + let first_key = buf.copy_to_bytes(first_key_len as usize); + let (is_large_and_compression_type, buf) = get_u8_le(buf)?; + let is_large = is_large_and_compression_type & 0b1000_0000 != 0; + let compression_type = is_large_and_compression_type & 0b0111_1111; + if is_large { + ans.push(BlockMeta { + offset: offset as usize, + is_large, + compression_type: compression_type.try_into()?, + first_key, + last_key: None, + }); + data = buf; + continue; + } + let (last_key_len, mut buf) = get_u16_le(buf)?; + if buf.len() < last_key_len as usize { + return Err(LoroError::DecodeError("Invalid bytes".into())); + } + let last_key = buf.copy_to_bytes(last_key_len as usize); + ans.push(BlockMeta { + offset: offset as usize, + is_large, + compression_type: compression_type.try_into()?, + first_key, + last_key: Some(last_key), + }); + data = buf; + } + let (checksum_read, _) = get_u32_le(data)?; + if checksum != checksum_read { + return Err(LoroError::DecodeChecksumMismatchError); + } + Ok(ans) + } +} + +pub(crate) struct SsTableBuilder { + block_builder: BlockBuilder, + first_key: Bytes, + last_key: Bytes, + data: Vec, + meta: Vec, + block_size: usize, + compression_type: CompressionType, + include_none: bool, // TODO: bloom filter +} + +impl SsTableBuilder { + pub fn new(block_size: usize, compression_type: CompressionType, include_none: bool) -> Self { + let mut data = Vec::with_capacity(5); + data.put_u32_le(u32::from_le_bytes(MAGIC_BYTES)); + data.put_u8(CURRENT_SCHEMA_VERSION); + Self { + block_builder: BlockBuilder::new(block_size), + first_key: Bytes::new(), + last_key: Bytes::new(), + data, + meta: Vec::new(), + block_size, + compression_type, + include_none, + } + } + + pub fn add(&mut self, key: Bytes, value: Bytes) { + if key.is_empty() { + return; + } + + if !self.include_none && value.is_empty() { + return; + } + + if self.first_key.is_empty() { + self.first_key = key.clone(); + } + + if self.block_builder.add(&key, &value) { + self.last_key = key; + return; + } + + self.finish_current_block(); + assert!(self.block_builder.add(&key, &value)); + self.first_key = key.clone(); + self.last_key = key; + } + + pub fn is_empty(&self) -> bool { + self.meta.is_empty() && self.block_builder.is_empty() + } + + pub(crate) fn finish_current_block(&mut self) { + if self.block_builder.is_empty() { + return; + } + let builder = + std::mem::replace(&mut self.block_builder, BlockBuilder::new(self.block_size)); + let block = builder.build(); + self.add_new_block_inner(&block); + } + + pub(crate) fn add_new_block(&mut self, block: Arc) { + let mut should_push_one_by_one = + self.block_builder.estimated_size() + block.data().len() + block.len() * 8 + < self.block_size; + if !should_push_one_by_one && !self.include_none { + for (_, v) in BlockIter::new(block.clone()) { + if v.is_empty() { + should_push_one_by_one = true; + } + } + } + + if should_push_one_by_one { + // data is small, push one by one + for (k, v) in BlockIter::new(block) { + self.add(k, v); + } + } else { + self.finish_current_block(); + if self.first_key.is_empty() { + self.first_key = block.first_key(); + } + + self.first_key = block.first_key(); + self.last_key = block.last_key(); + self.add_new_block_inner(&block); + } + } + + fn add_new_block_inner(&mut self, block: &Block) { + assert!(self.block_builder.is_empty()); + let offset = self.data.len(); + let real_compression_type = block.encode(&mut self.data, self.compression_type); + let is_large = block.is_large(); + let meta = BlockMeta { + offset, + is_large, + compression_type: real_compression_type, + first_key: std::mem::take(&mut self.first_key), + last_key: if is_large { + None + } else { + Some(std::mem::take(&mut self.last_key)) + }, + }; + self.meta.push(meta); + } + + /// ```log + /// ┌─────────────────────────────────────────────────────────────────────────────────────────────────┐ + /// │ SsTable │ + /// │┌ ─ ─ ─ ─ ─ ─ ─┌ ─ ─ ─ ─ ─ ─ ─ ─┌ ─ ─ ─ ─ ─ ─ ┬ ─ ─ ─┌ ─ ─ ─ ─ ─ ─ ─ ┬ ─ ─ ─ ─ ─ ─┌ ─ ─ ─ ─ ─ ─ ┐│ + /// │ Magic Number │ Schema Version │ Block Chunk ... │ Block Chunk Block Meta │ meta offset │ + /// ││ u32 │ u8 │ bytes │ │ bytes │ bytes │ u32 ││ + /// │ ─ ─ ─ ─ ─ ─ ─ ┘─ ─ ─ ─ ─ ─ ─ ─ ┘─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┘─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┘─ ─ ─ ─ ─ ─ ─ │ + /// └─────────────────────────────────────────────────────────────────────────────────────────────────┘ + /// ``` + pub fn build(mut self) -> SsTable { + self.finish_current_block(); + let mut buf = self.data; + let meta_offset = buf.len() as u32; + BlockMeta::encode_meta(&self.meta, &mut buf); + buf.put_u32_le(meta_offset); + let first_key = self + .meta + .first() + .map(|m| m.first_key.clone()) + .unwrap_or_default(); + let last_key = self + .meta + .last() + .map(|m| { + m.last_key.clone().unwrap_or( + self.meta + .last() + .map(|m| m.first_key.clone()) + .unwrap_or_default(), + ) + }) + .unwrap_or_default(); + SsTable { + data: Bytes::from(buf), + first_key, + last_key, + meta: self.meta, + meta_offset: meta_offset as usize, + block_cache: BlockCache::new(DEFAULT_CACHE_SIZE), + } + } +} + +type BlockCache = quick_cache::sync::Cache>; + +#[derive(Debug)] +pub struct SsTable { + // TODO: mmap? + data: Bytes, + pub(crate) first_key: Bytes, + pub(crate) last_key: Bytes, + meta: Vec, + meta_offset: usize, + block_cache: BlockCache, +} + +impl Clone for SsTable { + fn clone(&self) -> Self { + Self { + data: self.data.clone(), + first_key: self.first_key.clone(), + last_key: self.last_key.clone(), + meta: self.meta.clone(), + meta_offset: self.meta_offset, + block_cache: BlockCache::new(DEFAULT_CACHE_SIZE), + } + } +} + +impl SsTable { + pub fn export_all(&self) -> Bytes { + self.data.clone() + } + + pub fn iter(&self) -> SsTableIter<'_> { + SsTableIter::new(self) + } + + /// Import an SsTable, verifying every block's checksum. This is the safe + /// default for callers that don't have an outer integrity guarantee. + /// + /// `validate_blocks` additionally eagerly decodes every block to validate + /// block metadata and key ordering. The cheap structural + /// `validate_block_ranges` check always runs regardless. + /// + /// # Errors + /// - [LoroError::DecodeChecksumMismatchError] + /// - [LoroError::DecodeError] + /// - "Invalid magic number" + /// - "Invalid schema version" + pub fn import_all(bytes: Bytes, validate_blocks: bool) -> LoroResult { + Self::import_all_with(bytes, validate_blocks, true) + } + + /// Import without per-block checksum verification (and without eager block + /// validation). + /// + /// Only use this when the blob's integrity is already guaranteed by an outer + /// checksum — e.g. Loro verifies a document-wide checksum over the whole + /// snapshot body in `parse_header_and_body` before reaching here, so + /// re-hashing every block would redundantly cover bytes the outer checksum + /// already protects (this was ~38% of B4 snapshot-import time). + pub fn import_all_unchecked(bytes: Bytes) -> LoroResult { + Self::import_all_with(bytes, false, false) + } + + fn import_all_with( + bytes: Bytes, + validate_blocks: bool, + check_checksum: bool, + ) -> LoroResult { + // magic number + schema version + meta offset + if bytes.len() < SIZE_OF_U32 + SIZE_OF_U8 + SIZE_OF_U32 { + return Err(LoroError::DecodeError("Invalid sstable bytes".into())); + } + let magic_number = u32::from_le_bytes((&bytes[..SIZE_OF_U32]).try_into().unwrap()); + if magic_number != u32::from_le_bytes(MAGIC_BYTES) { + return Err(LoroError::DecodeError("Invalid magic number".into())); + } + let schema_version = bytes[SIZE_OF_U32]; + match schema_version { + CURRENT_SCHEMA_VERSION => {} + _ => { + return Err(LoroError::DecodeError( + format!( + "Invalid schema version {schema_version}, current support max version is {CURRENT_SCHEMA_VERSION}" + ) + .into(), + )) + } + } + let data_len = bytes.len(); + let meta_offset = (&bytes[data_len - SIZE_OF_U32..]).get_u32_le() as usize; + if meta_offset >= data_len - SIZE_OF_U32 { + return Err(LoroError::DecodeError("Invalid bytes".into())); + } + let raw_meta = &bytes[meta_offset..data_len - SIZE_OF_U32]; + let meta = BlockMeta::decode_meta(raw_meta)?; + Self::validate_block_ranges(&meta, meta_offset)?; + if validate_blocks { + Self::validate_blocks(&meta, &bytes, meta_offset)?; + } + if check_checksum { + Self::check_block_checksum(&meta, &bytes, meta_offset)?; + } + let first_key = meta + .first() + .map(|m| m.first_key.clone()) + .unwrap_or_default(); + let last_key = meta + .last() + .map(|m| { + m.last_key + .clone() + .unwrap_or(meta.last().map(|m| m.first_key.clone()).unwrap_or_default()) + }) + .unwrap_or_default(); + let ans = Self { + data: bytes, + first_key, + last_key, + meta, + meta_offset, + block_cache: BlockCache::new(DEFAULT_CACHE_SIZE), + }; + Ok(ans) + } + + fn validate_block_ranges(meta: &[BlockMeta], meta_offset: usize) -> LoroResult<()> { + if meta.is_empty() { + return Err(LoroError::DecodeError("Invalid bytes".into())); + } + + for i in 0..meta.len() { + let offset = meta[i].offset; + let offset_end = meta.get(i + 1).map_or(meta_offset, |m| m.offset); + if offset < SIZE_OF_U32 + SIZE_OF_U8 + || offset_end > meta_offset + || offset >= offset_end + || offset_end - offset < SIZE_OF_U32 + { + return Err(LoroError::DecodeError("Invalid bytes".into())); + } + } + + Ok(()) + } + + fn validate_blocks(meta: &[BlockMeta], bytes: &Bytes, meta_offset: usize) -> LoroResult<()> { + let mut last_key = None; + for i in 0..meta.len() { + let offset = meta[i].offset; + let offset_end = meta.get(i + 1).map_or(meta_offset, |m| m.offset); + let raw_block_and_check = bytes.slice(offset..offset_end); + let block = Block::try_decode( + raw_block_and_check, + meta[i].is_large, + meta[i].first_key.clone(), + meta[i].compression_type, + )?; + + let block_last_key = block.last_key(); + if meta[i].last_key.as_ref().unwrap_or(&meta[i].first_key) != &block_last_key { + return Err(LoroError::DecodeError("Invalid bytes".into())); + } + + if meta[i].first_key > block_last_key { + return Err(LoroError::DecodeError("Invalid bytes".into())); + } + + if last_key + .as_ref() + .is_some_and(|last_key| last_key >= &meta[i].first_key) + { + return Err(LoroError::DecodeError("Invalid bytes".into())); + } + + last_key = Some(block_last_key); + } + + Ok(()) + } + + fn check_block_checksum( + meta: &[BlockMeta], + bytes: &Bytes, + meta_offset: usize, + ) -> LoroResult<()> { + for i in 0..meta.len() { + let offset = meta[i].offset; + let offset_end = meta.get(i + 1).map_or(meta_offset, |m| m.offset); + if offset_end > bytes.len() { + return Err(LoroError::DecodeError("Invalid bytes".into())); + } + let raw_block_and_check = bytes.slice(offset..offset_end); + let checksum = raw_block_and_check + .slice(raw_block_and_check.len() - SIZE_OF_U32..) + .get_u32_le(); + if checksum + != xxhash_rust::xxh32::xxh32( + &raw_block_and_check[..raw_block_and_check.len() - SIZE_OF_U32], + XXH_SEED, + ) + { + return Err(LoroError::DecodeChecksumMismatchError); + } + } + Ok(()) + } + + pub fn find_block_idx(&self, key: &[u8]) -> usize { + self.meta + .partition_point(|meta| meta.first_key <= key) + .saturating_sub(1) + } + + pub fn find_back_block_idx(&self, key: &[u8]) -> usize { + self.meta + .partition_point(|meta| meta.last_key.as_ref().unwrap_or(&meta.first_key) <= key) + .min(self.meta.len() - 1) + } + + fn read_block(&self, block_idx: usize) -> Arc { + let offset = self.meta[block_idx].offset; + let offset_end = self + .meta + .get(block_idx + 1) + .map_or(self.meta_offset, |m| m.offset); + let raw_block_and_check = self.data.slice(offset..offset_end); + Arc::new(Block::decode( + raw_block_and_check, + self.meta[block_idx].is_large, + self.meta[block_idx].first_key.clone(), + self.meta[block_idx].compression_type, + )) + } + + pub(crate) fn read_block_cached(&self, block_idx: usize) -> Arc { + self.block_cache + .get_or_insert_with(&block_idx, || { + Ok::<_, LoroError>(self.read_block(block_idx)) + }) + .expect("validated SSTable block cache insert should succeed") + } + + pub fn contains_key(&self, key: &[u8]) -> bool { + if self.first_key > key || self.last_key < key { + return false; + } + let idx = self.find_block_idx(key); + let block = self.read_block_cached(idx); + let block_iter = BlockIter::new_seek_to_key(block, key); + block_iter.peek_next_curr_key() == Some(Bytes::copy_from_slice(key)) + } + + #[allow(unused)] + pub fn get(&self, key: &[u8]) -> Option { + if self.first_key > key || self.last_key < key { + return None; + } + let idx = self.find_block_idx(key); + let block = self.read_block_cached(idx); + let block_iter = BlockIter::new_seek_to_key(block, key); + block_iter.peek_next_curr_key().and_then(|k| { + if k == key { + block_iter.peek_next_curr_value() + } else { + None + } + }) + } + + pub fn data_size(&self) -> usize { + self.data.len() + } + + pub fn meta_len(&self) -> usize { + self.meta.len() + } +} + +#[derive(Clone)] +pub struct SsTableIter<'a> { + table: &'a SsTable, + iter: SsTableIterInner, + next_block_idx: usize, + back_block_idx: isize, +} + +impl Debug for SsTableIter<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SsTableIter") + .field("iter", &self.iter) + .field("next_block_idx", &self.next_block_idx) + .field("back_block_idx", &self.back_block_idx) + .finish() + } +} + +impl<'a> SsTableIter<'a> { + fn new(table: &'a SsTable) -> Self { + Self::new_scan(table, Bound::Unbounded, Bound::Unbounded) + } + + pub fn new_scan(table: &'a SsTable, start: Bound<&[u8]>, end: Bound<&[u8]>) -> Self { + let (table_idx, mut iter, excluded) = match start { + Bound::Included(start) => { + notify_cov("kv-store::SstableIter::new_scan::start included"); + let idx = table.find_block_idx(start); + let block = table.read_block_cached(idx); + let iter = BlockIter::new_seek_to_key(block, start); + (idx, iter, None) + } + Bound::Excluded(start) => { + notify_cov("kv-store::SstableIter::new_scan::start excluded"); + let idx = table.find_block_idx(start); + let block = table.read_block_cached(idx); + let iter = BlockIter::new_seek_to_key(block, start); + (idx, iter, Some(start)) + } + Bound::Unbounded => { + notify_cov("kv-store::SstableIter::new_scan::start unbounded"); + let block = table.read_block_cached(0); + let iter = BlockIter::new(block); + (0, iter, None) + } + }; + let (end_idx, end_iter, end_excluded) = match end { + Bound::Included(end) => { + notify_cov("kv-store::SstableIter::new_scan::end included"); + let end_idx = table.find_back_block_idx(end); + if end_idx == table_idx { + iter.back_to_key(end); + // if the next back is invalid, the next should also be invalid + if !iter.has_next_back() { + iter.next(); + } + (end_idx, None, None) + } else { + let block = table.read_block_cached(end_idx); + let iter = BlockIter::new_back_to_key(block, end); + (end_idx, Some(iter), None) + } + } + Bound::Excluded(end) => { + notify_cov("kv-store::SstableIter::new_scan::end excluded"); + let end_idx = table.find_back_block_idx(end); + if end_idx == table_idx { + iter.back_to_key(end); + // if the next back is invalid, the next should also be invalid + if !iter.has_next_back() { + iter.next(); + } + (end_idx, None, Some(end)) + } else { + let block = table.read_block_cached(end_idx); + let iter = BlockIter::new_back_to_key(block, end); + (end_idx, Some(iter), Some(end)) + } + } + Bound::Unbounded => { + notify_cov("kv-store::SstableIter::new_scan::end unbounded"); + let end_idx = table.meta.len() - 1; + if end_idx == table_idx { + notify_cov("kv-store::SstableIter::new_scan::unbounded equal"); + (end_idx, None, None) + } else { + let block = table.read_block_cached(end_idx); + let iter = BlockIter::new(block); + (end_idx, Some(iter), None) + } + } + }; + + let mut ans = if let Some(end_iter) = end_iter { + debug_assert!(end_idx > table_idx); + SsTableIter { + table, + iter: SsTableIterInner::Double { + front: iter, + back: end_iter, + }, + next_block_idx: table_idx, + back_block_idx: end_idx as isize, + } + } else { + debug_assert!(end_idx == table_idx); + SsTableIter { + table, + iter: SsTableIterInner::Same(iter), + next_block_idx: table_idx, + back_block_idx: end_idx as isize, + } + }; + // the current iter may be empty, but has next iter. we need to skip the empty iter + ans.skip_next_empty(); + ans.skip_next_back_empty(); + + if let Some(key) = excluded { + if ans.has_next() && ans.peek_next_key().unwrap() == key { + ans.next(); + } + } + if let Some(key) = end_excluded { + if ans.has_next_back() && ans.peek_next_back_key().unwrap() == key { + ans.next_back(); + } + } + ans + } + + fn skip_next_empty(&mut self) { + while self.has_next() && !self.iter.front_iter().has_next() { + self.next(); + } + } + + fn skip_next_back_empty(&mut self) { + while self.has_next_back() && !self.iter.back_iter().has_next_back() { + self.next_back(); + } + } + + fn has_next(&self) -> bool { + self.iter.front_iter().has_next() || (self.next_block_idx as isize) < self.back_block_idx + } + + pub fn peek_next_key(&self) -> Option { + if self.has_next() { + self.iter.front_iter().peek_next_curr_key() + } else { + None + } + } + + pub fn peek_next_value(&self) -> Option { + if self.has_next() { + self.iter.front_iter().peek_next_curr_value() + } else { + None + } + } + + fn has_next_back(&self) -> bool { + self.iter.back_iter().has_next_back() + || (self.next_block_idx as isize) < self.back_block_idx + } + + pub fn peek_next_back_key(&self) -> Option { + if !self.has_next_back() { + return None; + } + self.iter.back_iter().peek_back_curr_key() + } + + pub fn peek_next_back_value(&self) -> Option { + if !self.has_next_back() { + return None; + } + self.iter.back_iter().peek_back_curr_value() + } + + pub fn next(&mut self) { + self.iter.front_iter_mut().next(); + if !self.iter.front_iter().has_next() { + let this = &mut *self; + this.next_block_idx += 1; + if this.next_block_idx > this.back_block_idx as usize { + return; + } + if this.next_block_idx == this.back_block_idx as usize && !this.iter.is_same() { + this.iter.convert_back_as_same(); + } else if this.next_block_idx < this.table.meta.len() { + let block = this.table.read_block_cached(this.next_block_idx); + this.iter.reset_front(BlockIter::new(block)); + this.skip_next_empty(); + } else { + unreachable!() + } + } + } + + pub fn next_block(&mut self) { + self.next_block_idx += 1; + if self.next_block_idx > self.back_block_idx as usize { + match &mut self.iter { + SsTableIterInner::Same(block_iter) => block_iter.finish(), + SsTableIterInner::Double { .. } => unreachable!(), + } + return; + } + if self.next_block_idx == self.back_block_idx as usize && !self.iter.is_same() { + self.iter.convert_back_as_same(); + } else if self.next_block_idx < self.table.meta.len() { + let block = self.table.read_block_cached(self.next_block_idx); + self.iter.reset_front(BlockIter::new(block)); + self.skip_next_empty(); + } else { + unreachable!() + } + } + + pub fn next_back(&mut self) { + let iter = self.iter.back_iter_mut(); + iter.next_back(); + if !iter.has_next_back() { + self.back_block_idx -= 1; + if self.next_block_idx > self.back_block_idx as usize { + return; + } + if self.next_block_idx == self.back_block_idx as usize && !self.iter.is_same() { + self.iter.convert_front_as_same(); + } else if self.back_block_idx > 0 { + let block = self.table.read_block_cached(self.back_block_idx as usize); + self.iter.reset_back(BlockIter::new(block)); + self.skip_next_back_empty(); + } + } + } + + pub fn peek_next_block(&self) -> Option<&Arc> { + if self.has_next() { + Some(self.iter.front_iter().peek_block()) + } else { + None + } + } +} + +impl KvIterator for SsTableIter<'_> { + fn peek_next_key(&self) -> Option { + self.peek_next_key() + } + + fn peek_next_value(&self) -> Option { + self.peek_next_value() + } + + fn next_(&mut self) { + self.next() + } + + fn has_next(&self) -> bool { + self.has_next() + } + + fn peek_next_back_key(&self) -> Option { + self.peek_next_back_key() + } + + fn peek_next_back_value(&self) -> Option { + self.peek_next_back_value() + } + + fn next_back_(&mut self) { + self.next_back() + } + + fn has_next_back(&self) -> bool { + self.has_next_back() + } +} + +impl Iterator for SsTableIter<'_> { + type Item = (Bytes, Bytes); + fn next(&mut self) -> Option { + if !self.has_next() { + return None; + } + let key = self.peek_next_key().unwrap(); + let value = self.peek_next_value().unwrap(); + self.next(); + Some((key, value)) + } +} + +impl DoubleEndedIterator for SsTableIter<'_> { + fn next_back(&mut self) -> Option { + if !self.has_next_back() { + return None; + } + let key = self.peek_next_back_key().unwrap(); + let value = self.peek_next_back_value().unwrap(); + self.next_back(); + Some((key, value)) + } +} + +pub(crate) fn get_common_prefix_len_and_strip<'a, T: AsRef<[u8]> + ?Sized>( + this: &'a T, + last: &T, +) -> (u8, &'a [u8]) { + let mut common_prefix_len = 0; + for (i, (a, b)) in this.as_ref().iter().zip(last.as_ref().iter()).enumerate() { + if a != b || i == 255 { + common_prefix_len = i; + break; + } + } + + let suffix = &this.as_ref()[common_prefix_len..]; + (common_prefix_len as u8, suffix) +} + +#[derive(Clone, Debug)] +enum SsTableIterInner { + Same(BlockIter), + Double { front: BlockIter, back: BlockIter }, +} + +impl SsTableIterInner { + fn front_iter(&self) -> &BlockIter { + match self { + SsTableIterInner::Same(iter) => iter, + SsTableIterInner::Double { front, .. } => front, + } + } + + fn front_iter_mut(&mut self) -> &mut BlockIter { + match self { + SsTableIterInner::Same(iter) => iter, + SsTableIterInner::Double { front, .. } => front, + } + } + + fn back_iter(&self) -> &BlockIter { + match self { + SsTableIterInner::Same(iter) => iter, + SsTableIterInner::Double { back, .. } => back, + } + } + + fn back_iter_mut(&mut self) -> &mut BlockIter { + match self { + SsTableIterInner::Same(iter) => iter, + SsTableIterInner::Double { back, .. } => back, + } + } + + fn is_same(&self) -> bool { + matches!(self, SsTableIterInner::Same(_)) + } + + fn reset_front(&mut self, iter: BlockIter) { + debug_assert!(!self.is_same()); + let SsTableIterInner::Double { front, back: _ } = self else { + unreachable!() + }; + *front = iter; + } + + fn reset_back(&mut self, iter: BlockIter) { + debug_assert!(!self.is_same()); + let SsTableIterInner::Double { front: _, back } = self else { + unreachable!() + }; + *back = iter; + } + + fn convert_front_as_same(&mut self) { + debug_assert!(!self.is_same()); + let SsTableIterInner::Double { front, back: _ } = self else { + unreachable!() + }; + *self = SsTableIterInner::Same(front.clone()); + } + + fn convert_back_as_same(&mut self) { + debug_assert!(!self.is_same()); + let SsTableIterInner::Double { front: _, back } = self else { + unreachable!() + }; + *self = SsTableIterInner::Same(back.clone()); + } +} +#[cfg(test)] +mod test { + + use crate::block::BlockBuilder; + + use super::*; + use std::sync::Arc; + + fn malformed_sstable_bytes(block_bytes: &[u8], meta: &[BlockMeta]) -> Bytes { + let mut bytes = Vec::new(); + bytes.extend_from_slice(&MAGIC_BYTES); + bytes.push(CURRENT_SCHEMA_VERSION); + bytes.extend_from_slice(block_bytes); + let meta_offset = bytes.len(); + BlockMeta::encode_meta(meta, &mut bytes); + bytes.extend_from_slice(&(meta_offset as u32).to_le_bytes()); + bytes.into() + } + + fn normal_block_bytes(key: &[u8], value: &[u8]) -> Vec { + let mut builder = BlockBuilder::new(4096); + builder.add(key, value); + let block = builder.build(); + let mut bytes = Vec::new(); + block.encode(&mut bytes, CompressionType::None); + bytes + } + + fn normal_block_bytes_from_pairs(pairs: &[(&[u8], &[u8])]) -> Vec { + let mut builder = BlockBuilder::new(4096); + for (key, value) in pairs { + assert!(builder.add(key, value)); + } + let block = builder.build(); + let mut bytes = Vec::new(); + block.encode(&mut bytes, CompressionType::None); + bytes + } + + fn large_block_bytes(value: &[u8]) -> Vec { + let mut bytes = value.to_vec(); + let checksum = xxhash_rust::xxh32::xxh32(value, XXH_SEED); + bytes.extend_from_slice(&checksum.to_le_bytes()); + bytes + } + + #[test] + fn block_double_end_iter() { + let mut builder = BlockBuilder::new(4096); + builder.add(b"key1", b"value1"); + builder.add(b"key2", b"value2"); + builder.add(b"key3", b"value3"); + let block = builder.build(); + let mut iter = BlockIter::new(Arc::new(block)); + println!("{iter:?}"); + let (k1, v1) = Iterator::next(&mut iter).unwrap(); + let (k3, v3) = DoubleEndedIterator::next_back(&mut iter).unwrap(); + let (k2, v2) = Iterator::next(&mut iter).unwrap(); + assert_eq!(k1, Bytes::from_static(b"key1")); + assert_eq!(v1, Bytes::from_static(b"value1")); + assert_eq!(k2, Bytes::from_static(b"key2")); + assert_eq!(v2, Bytes::from_static(b"value2")); + assert_eq!(k3, Bytes::from_static(b"key3")); + assert_eq!(v3, Bytes::from_static(b"value3")); + assert!(Iterator::next(&mut iter).is_none()); + assert!(DoubleEndedIterator::next_back(&mut iter).is_none()); + } + + #[test] + fn block_range_iter() { + let mut builder = BlockBuilder::new(4096); + builder.add(b"key1", b"value1"); + builder.add(b"key2", b"value2"); + builder.add(b"key3", b"value3"); + let block = builder.build(); + let mut iter = BlockIter::new_scan( + Arc::new(block), + Bound::Included(b"key0"), + Bound::Included(b"key4"), + ); + let (k1, v1) = Iterator::next(&mut iter).unwrap(); + let (k3, v3) = DoubleEndedIterator::next_back(&mut iter).unwrap(); + let (k2, v2) = Iterator::next(&mut iter).unwrap(); + assert_eq!(k1, Bytes::from_static(b"key1")); + assert_eq!(v1, Bytes::from_static(b"value1")); + assert_eq!(k2, Bytes::from_static(b"key2")); + assert_eq!(v2, Bytes::from_static(b"value2")); + assert_eq!(k3, Bytes::from_static(b"key3")); + assert_eq!(v3, Bytes::from_static(b"value3")); + assert!(Iterator::next(&mut iter).is_none()); + assert!(DoubleEndedIterator::next_back(&mut iter).is_none()); + + let mut builder = BlockBuilder::new(4096); + builder.add(b"key1", b"value1"); + builder.add(b"key2", b"value2"); + builder.add(b"key3", b"value3"); + let block = builder.build(); + let mut iter = BlockIter::new_scan( + Arc::new(block), + Bound::Included(b"key1"), + Bound::Included(b"key3"), + ); + let (k1, v1) = Iterator::next(&mut iter).unwrap(); + let (k3, v3) = DoubleEndedIterator::next_back(&mut iter).unwrap(); + let (k2, v2) = Iterator::next(&mut iter).unwrap(); + assert_eq!(k1, Bytes::from_static(b"key1")); + assert_eq!(v1, Bytes::from_static(b"value1")); + assert_eq!(k2, Bytes::from_static(b"key2")); + assert_eq!(v2, Bytes::from_static(b"value2")); + assert_eq!(k3, Bytes::from_static(b"key3")); + assert_eq!(v3, Bytes::from_static(b"value3")); + assert!(Iterator::next(&mut iter).is_none()); + assert!(DoubleEndedIterator::next_back(&mut iter).is_none()); + + let mut builder = BlockBuilder::new(4096); + builder.add(b"key0", b"value0"); + builder.add(b"key2", b"value2"); + builder.add(b"key3", b"value3"); + let block = builder.build(); + let mut iter = BlockIter::new_scan( + Arc::new(block), + Bound::Included(b"key1"), + Bound::Included(b"key3"), + ); + let (k1, v1) = Iterator::next(&mut iter).unwrap(); + let (k2, v2) = DoubleEndedIterator::next_back(&mut iter).unwrap(); + assert_eq!(k1, Bytes::from_static(b"key2")); + assert_eq!(v1, Bytes::from_static(b"value2")); + assert_eq!(k2, Bytes::from_static(b"key3")); + assert_eq!(v2, Bytes::from_static(b"value3")); + assert!(Iterator::next(&mut iter).is_none()); + assert!(DoubleEndedIterator::next_back(&mut iter).is_none()); + } + + #[test] + fn block_scan() { + let mut builder = BlockBuilder::new(4096); + builder.add(b"key1", b"value1"); + builder.add(b"key2", b"value2"); + builder.add(b"key3", b"value3"); + let block = builder.build(); + let mut iter = + BlockIter::new_scan(Arc::new(block), Bound::Excluded(b"key1"), Bound::Unbounded); + let (k1, v1) = Iterator::next(&mut iter).unwrap(); + let (k2, v2) = DoubleEndedIterator::next_back(&mut iter).unwrap(); + assert_eq!(k1, Bytes::from_static(b"key2")); + assert_eq!(v1, Bytes::from_static(b"value2")); + assert_eq!(k2, Bytes::from_static(b"key3")); + assert_eq!(v2, Bytes::from_static(b"value3")); + assert!(Iterator::next(&mut iter).is_none()); + assert!(DoubleEndedIterator::next_back(&mut iter).is_none()); + } + + #[test] + fn block_scan2() { + let mut builder = BlockBuilder::new(4096); + builder.add(b"key1", b"value1"); + builder.add(b"key2", b"value2"); + builder.add(b"key3", b"value3"); + let block = builder.build(); + let mut iter = + BlockIter::new_scan(Arc::new(block), Bound::Unbounded, Bound::Excluded(b"key3")); + let (k1, v1) = Iterator::next(&mut iter).unwrap(); + let (k2, v2) = DoubleEndedIterator::next_back(&mut iter).unwrap(); + assert_eq!(k1, Bytes::from_static(b"key1")); + assert_eq!(v1, Bytes::from_static(b"value1")); + assert_eq!(k2, Bytes::from_static(b"key2")); + assert_eq!(v2, Bytes::from_static(b"value2")); + assert!(Iterator::next(&mut iter).is_none()); + assert!(DoubleEndedIterator::next_back(&mut iter).is_none()); + } + + #[test] + fn block_double_end_iter_with_delete() { + let mut builder = BlockBuilder::new(4096); + builder.add(b"key1", b"value1"); + builder.add(b"key2", b"value2"); + builder.add(b"key4", b""); + builder.add(b"key3", b"value3"); + let block = builder.build(); + let mut iter = BlockIter::new(Arc::new(block)); + let (k1, v1) = Iterator::next(&mut iter).unwrap(); + let (k3, v3) = DoubleEndedIterator::next_back(&mut iter).unwrap(); + let (k2, v2) = Iterator::next(&mut iter).unwrap(); + let (k4, v4) = DoubleEndedIterator::next_back(&mut iter).unwrap(); + assert_eq!(k1, Bytes::from_static(b"key1")); + assert_eq!(v1, Bytes::from_static(b"value1")); + assert_eq!(k2, Bytes::from_static(b"key2")); + assert_eq!(v2, Bytes::from_static(b"value2")); + assert_eq!(k3, Bytes::from_static(b"key3")); + assert_eq!(v3, Bytes::from_static(b"value3")); + assert_eq!(k4, Bytes::from_static(b"key4")); + assert_eq!(v4, Bytes::new()); + assert!(Iterator::next(&mut iter).is_none()); + assert!(DoubleEndedIterator::next_back(&mut iter).is_none()); + } + + #[test] + fn sstable_iter() { + let mut builder = SsTableBuilder::new(10, CompressionType::LZ4, true); + builder.add(Bytes::from_static(b"key1"), Bytes::from_static(b"value1")); + builder.add(Bytes::from_static(b"key2"), Bytes::from_static(b"value2")); + builder.add(Bytes::from_static(b"key3"), Bytes::from_static(b"value3")); + let table = builder.build(); + let mut iter = table.iter(); + let (k1, v1) = Iterator::next(&mut iter).unwrap(); + let (k3, v3) = DoubleEndedIterator::next_back(&mut iter).unwrap(); + let (k2, v2) = Iterator::next(&mut iter).unwrap(); + assert_eq!(k1, Bytes::from_static(b"key1")); + assert_eq!(v1, Bytes::from_static(b"value1")); + assert_eq!(k3, Bytes::from_static(b"key3")); + assert_eq!(v3, Bytes::from_static(b"value3")); + assert_eq!(k2, Bytes::from_static(b"key2")); + assert_eq!(v2, Bytes::from_static(b"value2")); + assert!(Iterator::next(&mut iter).is_none()); + assert!(DoubleEndedIterator::next_back(&mut iter).is_none()); + } + + #[test] + fn sstable_iter_with_delete() { + let mut builder = SsTableBuilder::new(10, CompressionType::LZ4, true); + builder.add(Bytes::from_static(b"key1"), Bytes::from_static(b"value1")); + builder.add(Bytes::from_static(b"key4"), Bytes::new()); + builder.add(Bytes::from_static(b"key2"), Bytes::from_static(b"value2")); + builder.add(Bytes::from_static(b"key5"), Bytes::new()); + builder.add(Bytes::from_static(b"key3"), Bytes::from_static(b"value3")); + let table = builder.build(); + let mut iter = table.iter(); + let (k1, v1) = Iterator::next(&mut iter).unwrap(); + let (k3, v3) = DoubleEndedIterator::next_back(&mut iter).unwrap(); + let (k4, v4) = Iterator::next(&mut iter).unwrap(); + let (k5, v5) = DoubleEndedIterator::next_back(&mut iter).unwrap(); + let (k2, v2) = Iterator::next(&mut iter).unwrap(); + assert_eq!(k1, Bytes::from_static(b"key1")); + assert_eq!(v1, Bytes::from_static(b"value1")); + assert_eq!(k3, Bytes::from_static(b"key3")); + assert_eq!(v3, Bytes::from_static(b"value3")); + assert_eq!(k4, Bytes::from_static(b"key4")); + assert_eq!(v4, Bytes::new()); + assert_eq!(k5, Bytes::from_static(b"key5")); + assert_eq!(v5, Bytes::new()); + assert_eq!(k2, Bytes::from_static(b"key2")); + assert_eq!(v2, Bytes::from_static(b"value2")); + assert!(Iterator::next(&mut iter).is_none()); + assert!(DoubleEndedIterator::next_back(&mut iter).is_none()); + } + + #[test] + fn sstable_scan() { + let mut builder = SsTableBuilder::new(10, CompressionType::LZ4, true); + builder.add(Bytes::from_static(b"key1"), Bytes::from_static(b"value1")); + builder.add(Bytes::from_static(b"key4"), Bytes::new()); + builder.add(Bytes::from_static(b"key2"), Bytes::from_static(b"value2")); + builder.add(Bytes::from_static(b"key5"), Bytes::new()); + builder.add(Bytes::from_static(b"key3"), Bytes::from_static(b"value3")); + let table = builder.build(); + assert!(table.contains_key(b"key1")); + let mut iter = SsTableIter::new_scan(&table, Bound::Excluded(b"key1"), Bound::Unbounded); + let (k3, v3) = DoubleEndedIterator::next_back(&mut iter).unwrap(); + let (k4, v4) = Iterator::next(&mut iter).unwrap(); + let (k5, v5) = DoubleEndedIterator::next_back(&mut iter).unwrap(); + let (k2, v2) = Iterator::next(&mut iter).unwrap(); + assert_eq!(k3, Bytes::from_static(b"key3")); + assert_eq!(v3, Bytes::from_static(b"value3")); + assert_eq!(k4, Bytes::from_static(b"key4")); + assert_eq!(v4, Bytes::new()); + assert_eq!(k5, Bytes::from_static(b"key5")); + assert_eq!(v5, Bytes::new()); + assert_eq!(k2, Bytes::from_static(b"key2")); + assert_eq!(v2, Bytes::from_static(b"value2")); + assert!(Iterator::next(&mut iter).is_none()); + assert!(DoubleEndedIterator::next_back(&mut iter).is_none()); + } + + #[test] + fn sstable_import_checksum() { + // Create an SSTable in memory + let mut builder = SsTableBuilder::new(10, CompressionType::LZ4, true); + builder.add(Bytes::from_static(b"key1"), Bytes::from_static(b"value1")); + builder.add(Bytes::from_static(b"key2"), Bytes::from_static(b"value2")); + builder.add(Bytes::from_static(b"key3"), Bytes::from_static(b"value3")); + let original_table = builder.build(); + let mut buffer = original_table.export_all().to_vec(); + buffer[11] = 123; + assert!(SsTable::import_all(buffer.into(), true).is_err()); + } + + #[test] + fn sstable_import_rejects_empty_meta() { + assert!(SsTable::import_all(malformed_sstable_bytes(&[], &[]), false).is_err()); + } + + #[test] + fn sstable_import_rejects_invalid_block_ranges() { + let first_key = Bytes::from_static(b"key"); + let meta = [BlockMeta { + offset: SIZE_OF_U32 + SIZE_OF_U8, + is_large: false, + compression_type: CompressionType::None, + first_key: first_key.clone(), + last_key: Some(first_key.clone()), + }]; + assert!(SsTable::import_all(malformed_sstable_bytes(&[0, 1, 2], &meta), false).is_err()); + + let meta = [BlockMeta { + offset: SIZE_OF_U32 + SIZE_OF_U8 + 8, + is_large: false, + compression_type: CompressionType::None, + first_key: first_key.clone(), + last_key: Some(first_key), + }]; + assert!(SsTable::import_all(malformed_sstable_bytes(&[0, 1, 2, 3], &meta), false).is_err()); + } + + #[test] + fn sstable_import_rejects_undecodable_block_payload() { + let first_key = Bytes::from_static(b"key"); + let meta = [BlockMeta { + offset: SIZE_OF_U32 + SIZE_OF_U8, + is_large: false, + compression_type: CompressionType::None, + first_key: first_key.clone(), + last_key: Some(first_key), + }]; + let checksum_only = xxhash_rust::xxh32::xxh32(&[], XXH_SEED).to_le_bytes(); + assert!(SsTable::import_all(malformed_sstable_bytes(&checksum_only, &meta), true).is_err()); + } + + #[test] + fn sstable_import_rejects_meta_last_key_that_mismatches_block() { + let first_key = Bytes::from_static(b"key"); + let meta = [BlockMeta { + offset: SIZE_OF_U32 + SIZE_OF_U8, + is_large: false, + compression_type: CompressionType::None, + first_key: first_key.clone(), + last_key: Some(Bytes::from_static(b"a")), + }]; + + assert!(SsTable::import_all( + malformed_sstable_bytes(&normal_block_bytes(b"key", b"value"), &meta), + true + ) + .is_err()); + } + + #[test] + fn sstable_import_rejects_unsorted_block_key_ranges() { + let first_block = normal_block_bytes(b"b", b"value"); + let second_block = normal_block_bytes(b"a", b"value"); + let second_offset = SIZE_OF_U32 + SIZE_OF_U8 + first_block.len(); + let meta = [ + BlockMeta { + offset: SIZE_OF_U32 + SIZE_OF_U8, + is_large: false, + compression_type: CompressionType::None, + first_key: Bytes::from_static(b"b"), + last_key: Some(Bytes::from_static(b"b")), + }, + BlockMeta { + offset: second_offset, + is_large: false, + compression_type: CompressionType::None, + first_key: Bytes::from_static(b"a"), + last_key: Some(Bytes::from_static(b"a")), + }, + ]; + let mut block_bytes = first_block; + block_bytes.extend_from_slice(&second_block); + + assert!(SsTable::import_all(malformed_sstable_bytes(&block_bytes, &meta), true).is_err()); + } + + #[test] + fn sstable_import_rejects_unsorted_keys_inside_block() { + let block_bytes = + normal_block_bytes_from_pairs(&[(b"a", b"1"), (b"c", b"2"), (b"b", b"3")]); + let meta = [BlockMeta { + offset: SIZE_OF_U32 + SIZE_OF_U8, + is_large: false, + compression_type: CompressionType::None, + first_key: Bytes::from_static(b"a"), + last_key: Some(Bytes::from_static(b"b")), + }]; + + assert!(SsTable::import_all(malformed_sstable_bytes(&block_bytes, &meta), true).is_err()); + } + + #[test] + fn sstable_import_rejects_empty_large_block_key() { + let meta = [BlockMeta { + offset: SIZE_OF_U32 + SIZE_OF_U8, + is_large: true, + compression_type: CompressionType::None, + first_key: Bytes::new(), + last_key: None, + }]; + + assert!(SsTable::import_all( + malformed_sstable_bytes(&large_block_bytes(b"value"), &meta), + true + ) + .is_err()); + } + + #[test] + fn sstable_import_fast_path_skips_block_validation() { + let first_key = Bytes::from_static(b"key"); + let meta = [BlockMeta { + offset: SIZE_OF_U32 + SIZE_OF_U8, + is_large: false, + compression_type: CompressionType::None, + first_key: first_key.clone(), + last_key: Some(Bytes::from_static(b"a")), + }]; + + let bytes = malformed_sstable_bytes(&normal_block_bytes(b"key", b"value"), &meta); + assert!(SsTable::import_all(bytes, false).is_ok()); + } + + #[test] + fn sstable_import_block_checksum_only_skipped_when_unchecked() { + // The public `import_all` always verifies per-block checksums (with or + // without the heavier `validate_blocks` decode). Only the explicit + // `import_all_unchecked` fast path skips them, and its caller is then + // responsible for integrity via an outer checksum — that is the + // redundant-checksum skip that makes full snapshot import faster. + let first_key = Bytes::from_static(b"key"); + let mut block_bytes = normal_block_bytes(b"key", b"value"); + *block_bytes.last_mut().unwrap() ^= 0xff; + let meta = [BlockMeta { + offset: SIZE_OF_U32 + SIZE_OF_U8, + is_large: false, + compression_type: CompressionType::None, + first_key: first_key.clone(), + last_key: Some(first_key), + }]; + + let bytes = malformed_sstable_bytes(&block_bytes, &meta); + // Full validation catches the corrupted checksum. + assert!(SsTable::import_all(bytes.clone(), true).is_err()); + // The cheap per-block checksum (always on for the public API) catches it too. + assert!(SsTable::import_all(bytes.clone(), false).is_err()); + // Only the explicit unchecked fast path skips the check. + assert!(SsTable::import_all_unchecked(bytes).is_ok()); + } +} diff --git a/3rdparty/loro/crates/kv-store/src/utils.rs b/3rdparty/loro/crates/kv-store/src/utils.rs new file mode 100644 index 0000000000..037a9a6a32 --- /dev/null +++ b/3rdparty/loro/crates/kv-store/src/utils.rs @@ -0,0 +1,24 @@ +use loro_common::{LoroError, LoroResult}; + +pub(crate) fn get_u32_le(bytes: &[u8]) -> LoroResult<(u32, &[u8])> { + if bytes.len() < 4 { + return Err(LoroError::DecodeError("Invalid bytes".into())); + } + let ans = u32::from_le_bytes(bytes[..4].try_into().unwrap()); + Ok((ans, &bytes[4..])) +} + +pub(crate) fn get_u8_le(bytes: &[u8]) -> LoroResult<(u8, &[u8])> { + if bytes.is_empty() { + return Err(LoroError::DecodeError("Invalid bytes".into())); + } + Ok((bytes[0], &bytes[1..])) +} + +pub(crate) fn get_u16_le(bytes: &[u8]) -> LoroResult<(u16, &[u8])> { + if bytes.len() < 2 { + return Err(LoroError::DecodeError("Invalid bytes".into())); + } + let ans = u16::from_le_bytes(bytes[..2].try_into().unwrap()); + Ok((ans, &bytes[2..])) +} diff --git a/3rdparty/loro/crates/loro-common/Cargo.toml b/3rdparty/loro/crates/loro-common/Cargo.toml new file mode 100644 index 0000000000..c8e6608b39 --- /dev/null +++ b/3rdparty/loro/crates/loro-common/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "loro-common" +version = "1.13.1" +edition = "2021" +license = "MIT" +description = "Common types and functions for Loro. This is an internal lib of Loro." +documentation = "https://docs.rs/loro/" +homepage = "https://loro.dev" +repository = "https://github.com/loro-dev/loro" +authors = ["Zixuan Chen", "Liang Zhao"] +categories = [] +keywords = ["crdt", "local-first"] + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +rle = { path = "../rle", version = "1.6.0", package = "loro-rle" } +serde = { workspace = true } +serde_json = { workspace = true, optional = true } +thiserror = "1.0.43" +wasm-bindgen = { version = "=0.2.100", optional = true } +rustc-hash = "2.1.1" +enum-as-inner = "0.6.0" +arbitrary = { version = "1.3.0", features = ["derive"] } +js-sys = { version = "0.3.60", optional = true } +serde_columnar = { workspace = true } +nonmax = "0.5.5" +leb128 = "0.2.5" +tracing = { workspace = true, optional = true } + +[features] +wasm = ["wasm-bindgen", "js-sys"] +logging = ["tracing"] +counter = [] diff --git a/3rdparty/loro/crates/loro-common/LICENSE b/3rdparty/loro/crates/loro-common/LICENSE new file mode 100644 index 0000000000..3cf271d30f --- /dev/null +++ b/3rdparty/loro/crates/loro-common/LICENSE @@ -0,0 +1,21 @@ +MIT License Copyright (c) 2023 Zixuan Chen + +Permission is hereby granted, free of +charge, to any person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the Software without +restriction, including without limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to the +following conditions: + +The above copyright notice and this permission notice +(including the next paragraph) shall be included in all copies or substantial +portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO +EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. \ No newline at end of file diff --git a/3rdparty/loro/crates/loro-common/README.md b/3rdparty/loro/crates/loro-common/README.md new file mode 100644 index 0000000000..f0ad755991 --- /dev/null +++ b/3rdparty/loro/crates/loro-common/README.md @@ -0,0 +1,4 @@ +# loro-common + +This crate contains the common basic types for the Loro project. +It's shared by `loro-internal` and `loro-preload`. diff --git a/3rdparty/loro/crates/loro-common/src/error.rs b/3rdparty/loro/crates/loro-common/src/error.rs new file mode 100644 index 0000000000..884a65d976 --- /dev/null +++ b/3rdparty/loro/crates/loro-common/src/error.rs @@ -0,0 +1,274 @@ +use std::{ + backtrace::{Backtrace, BacktraceStatus}, + panic::Location, +}; + +use serde_columnar::ColumnarError; +use thiserror::Error; + +use crate::{ContainerID, InternalString, PeerID, TreeID, ID}; + +pub type LoroResult = Result; + +#[derive(Error, Debug, PartialEq)] +pub enum LoroError { + #[error("Context's client_id({found:?}) does not match Container's client_id({expected:?})")] + UnmatchedContext { expected: PeerID, found: PeerID }, + #[error("Decode error: Version vector error. Please provide correct version.")] + DecodeVersionVectorError, + #[error("Decode error: ({0})")] + DecodeError(Box), + #[error( + // This should not happen after v1.0.0 + "Decode error: The data is either corrupted or originates from an older version that is incompatible due to a breaking change." + )] + DecodeDataCorruptionError, + #[error("Decode error: Checksum mismatch. The data is corrupted.")] + DecodeChecksumMismatchError, + #[error("Decode error: Encoding version \"{0}\" is incompatible. Loro's encoding is backward compatible but not forward compatible. Please upgrade the version of Loro to support this version of the exported data.")] + IncompatibleFutureEncodingError(usize), + #[error("Js error ({0})")] + JsError(Box), + #[error("Cannot get lock or the lock is poisoned")] + LockError, + #[error("Each AppState can only have one transaction at a time")] + DuplicatedTransactionError, + #[error("Cannot find ({0})")] + NotFoundError(Box), + #[error("Transaction error ({0})")] + TransactionError(Box), + #[error("Index out of bound. The given pos is {pos}, but the length is {len}. {info}")] + OutOfBound { + pos: usize, + len: usize, + info: Box, + }, + #[error("Every op id should be unique. ID {id} has been used. You should use a new PeerID to edit the content. ")] + UsedOpID { id: ID }, + #[error("Concurrent ops with the same peer id is not allowed. PeerID: {peer}, LastCounter: {last_counter}, CurrentCounter: {current}")] + ConcurrentOpsWithSamePeerID { + peer: PeerID, + last_counter: i32, + current: i32, + }, + #[error("Movable Tree Error: {0}")] + TreeError(#[from] LoroTreeError), + #[error("Invalid argument ({0})")] + ArgErr(Box), + #[error("Auto commit has not started. The doc is readonly when detached and detached editing is not enabled.")] + AutoCommitNotStarted, + #[error("Style configuration missing for \"({0:?})\". Please provide the style configuration using `configTextStyle` on your Loro doc.")] + StyleConfigMissing(InternalString), + #[error("Unknown Error ({0})")] + Unknown(Box), + #[error("The given ID ({0}) is not contained by the doc")] + FrontiersNotFound(ID), + #[error("Cannot import when the doc is in a transaction")] + ImportWhenInTxn, + #[error("The given method ({method}) is not allowed when the container is detached. You should insert the container to the doc first.")] + MisuseDetachedContainer { method: &'static str }, + #[error("Not implemented: {0}")] + NotImplemented(&'static str), + #[error("Reattach a container that is already attached")] + ReattachAttachedContainer, + #[error("Edit is not allowed when the doc is in the detached mode.")] + EditWhenDetached, + #[error("The given ID ({0}) is not contained by the doc")] + UndoInvalidIdSpan(ID), + #[error("PeerID cannot be changed. Expected: {expected:?}, Actual: {actual:?}")] + UndoWithDifferentPeerId { expected: PeerID, actual: PeerID }, + #[error("There is already an active undo group, call `group_end` first")] + UndoGroupAlreadyStarted, + #[error("There is no active undo group, call `group_start` first")] + InvalidJsonSchema, + #[error("Cannot insert or delete utf-8 in the middle of the codepoint in Unicode")] + UTF8InUnicodeCodePoint { pos: usize }, + #[error("Cannot insert or delete utf-16 in the middle of the codepoint in Unicode")] + UTF16InUnicodeCodePoint { pos: usize }, + #[error("The end index cannot be less than the start index")] + EndIndexLessThanStartIndex { start: usize, end: usize }, + #[error("Invalid root container name! Don't include '/' or '\\0'")] + InvalidRootContainerName, + #[error("Import Failed: The dependencies of the importing updates are not included in the shallow history of the doc.")] + ImportUpdatesThatDependsOnOutdatedVersion, + #[error( + "You cannot switch a document to a version before the shallow history's start version." + )] + SwitchToVersionBeforeShallowRoot, + #[error( + "The container {container} is deleted. You cannot apply the op on a deleted container." + )] + ContainerDeleted { container: Box }, + #[error("You cannot set the `PeerID` with `PeerID::MAX`, which is an internal specific value")] + InvalidPeerID, + #[error("The containers {containers:?} are not found in the doc")] + ContainersNotFound { containers: Box> }, + #[error("Import failed: Deprecated encoding mode")] + ImportUnsupportedEncodingMode, +} + +impl LoroError { + #[track_caller] + pub fn internal(message: impl Into) -> Self { + Self::Unknown(format_internal_error_message( + message.into(), + Location::caller(), + )) + } +} + +#[derive(Error, Debug, PartialEq)] +pub enum LoroTreeError { + #[error("`Cycle move` occurs when moving tree nodes.")] + CyclicMoveError, + #[error("The provided parent id is invalid")] + InvalidParent, + #[error("The parent of tree node is not found {0:?}")] + TreeNodeParentNotFound(TreeID), + #[error("TreeID {0:?} doesn't exist")] + TreeNodeNotExist(TreeID), + #[error("The index({index}) should be <= the length of children ({len})")] + IndexOutOfBound { len: usize, index: usize }, + #[error("Fractional index is not enabled, you should enable it first by `LoroTree::set_enable_fractional_index`")] + FractionalIndexNotEnabled, + #[error("TreeID {0:?} is deleted or does not exist")] + TreeNodeDeletedOrNotExist(TreeID), +} + +#[non_exhaustive] +#[derive(Error, Debug, PartialEq)] +pub enum LoroEncodeError { + #[error("The frontiers are not found in this doc: {0}")] + FrontiersNotFound(String), + #[error("Shallow snapshot incompatible with old snapshot format. Use new snapshot format or avoid shallow snapshots for storage.")] + ShallowSnapshotIncompatibleWithOldFormat, + #[error("Cannot export shallow snapshot with unknown container type. Please upgrade the Loro version.")] + UnknownContainer, + #[error("Export failed: {0}")] + InternalError(Box), +} + +impl LoroEncodeError { + #[track_caller] + pub fn internal(message: impl Into) -> Self { + Self::InternalError(format_internal_error_message( + message.into(), + Location::caller(), + )) + } +} + +#[cfg(feature = "wasm")] +pub mod wasm { + use wasm_bindgen::JsValue; + + use crate::{LoroEncodeError, LoroError}; + + impl From for JsValue { + fn from(value: LoroError) -> Self { + JsValue::from_str(&value.to_string()) + } + } + + impl From for JsValue { + fn from(value: LoroEncodeError) -> Self { + JsValue::from_str(&value.to_string()) + } + } + + impl From for LoroError { + fn from(v: JsValue) -> Self { + Self::JsError( + v.as_string() + .unwrap_or_else(|| "unknown error".to_owned()) + .into_boxed_str(), + ) + } + } +} + +impl From for LoroError { + fn from(e: ColumnarError) -> Self { + match e { + ColumnarError::ColumnarDecodeError(_) + | ColumnarError::RleEncodeError(_) + | ColumnarError::RleDecodeError(_) + | ColumnarError::OverflowError => { + LoroError::DecodeError(format!("Failed to decode Columnar: {e}").into_boxed_str()) + } + e => LoroError::Unknown(e.to_string().into_boxed_str()), + } + } +} + +impl From for LoroError { + fn from(value: LoroEncodeError) -> Self { + match value { + LoroEncodeError::FrontiersNotFound(frontiers) => { + LoroError::NotFoundError(frontiers.into_boxed_str()) + } + LoroEncodeError::ShallowSnapshotIncompatibleWithOldFormat + | LoroEncodeError::UnknownContainer => { + LoroError::Unknown(value.to_string().into_boxed_str()) + } + LoroEncodeError::InternalError(msg) => LoroError::Unknown(msg), + } + } +} + +impl From for LoroEncodeError { + fn from(value: LoroError) -> Self { + match value { + LoroError::FrontiersNotFound(id) => { + LoroEncodeError::FrontiersNotFound(format!("{id:?}")) + } + LoroError::NotFoundError(msg) => LoroEncodeError::FrontiersNotFound(msg.into()), + LoroError::Unknown(msg) => LoroEncodeError::InternalError(msg), + other => LoroEncodeError::InternalError(other.to_string().into_boxed_str()), + } + } +} + +fn format_internal_error_message( + message: String, + location: &'static Location<'static>, +) -> Box { + let mut formatted = format!( + "{message} at {}:{}:{}", + location.file(), + location.line(), + location.column() + ); + let backtrace = Backtrace::capture(); + if backtrace.status() == BacktraceStatus::Captured { + formatted.push_str("\nBacktrace:\n"); + formatted.push_str(&backtrace.to_string()); + } + + formatted.into_boxed_str() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn internal_loro_error_contains_caller_location() { + let err = LoroError::internal("boom"); + let LoroError::Unknown(msg) = err else { + panic!("expected unknown error"); + }; + assert!(msg.contains("boom")); + assert!(msg.contains("crates/loro-common/src/error.rs")); + } + + #[test] + fn internal_encode_error_contains_caller_location() { + let err = LoroEncodeError::internal("boom"); + let LoroEncodeError::InternalError(msg) = err else { + panic!("expected internal encode error"); + }; + assert!(msg.contains("boom")); + assert!(msg.contains("crates/loro-common/src/error.rs")); + } +} diff --git a/3rdparty/loro/crates/loro-common/src/id.rs b/3rdparty/loro/crates/loro-common/src/id.rs new file mode 100644 index 0000000000..ce49f793f1 --- /dev/null +++ b/3rdparty/loro/crates/loro-common/src/id.rs @@ -0,0 +1,261 @@ +use crate::{span::IdSpan, CounterSpan, IdFull, IdLp, IdLpSpan, Lamport}; + +use super::{Counter, LoroError, PeerID, ID}; +const UNKNOWN: PeerID = 404; +use std::{ + fmt::{Debug, Display}, + ops::RangeBounds, +}; + +impl Debug for ID { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(format!("{}@{}", self.counter, self.peer).as_str()) + } +} + +impl Debug for IdLp { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(format!("L{}@{}", self.lamport, self.peer).as_str()) + } +} + +impl Display for ID { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(format!("{}@{}", self.counter, self.peer).as_str()) + } +} + +impl Display for IdLp { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(format!("L{}@{}", self.lamport, self.peer).as_str()) + } +} + +impl TryFrom<&str> for ID { + type Error = LoroError; + + fn try_from(value: &str) -> Result { + if value.split('@').count() != 2 { + return Err(LoroError::DecodeError("Invalid ID format".into())); + } + + let mut iter = value.split('@'); + let counter = iter + .next() + .unwrap() + .parse::() + .map_err(|_| LoroError::DecodeError("Invalid ID format".into()))?; + let client_id = iter + .next() + .unwrap() + .parse::() + .map_err(|_| LoroError::DecodeError("Invalid ID format".into()))?; + Ok(ID { + peer: client_id, + counter, + }) + } +} + +impl TryFrom<&str> for IdLp { + type Error = LoroError; + + fn try_from(value: &str) -> Result { + if value.split('@').count() != 2 || !value.starts_with('L') { + return Err(LoroError::DecodeError("Invalid ID format".into())); + } + + let mut iter = value[1..].split('@'); + let lamport = iter + .next() + .unwrap() + .parse::() + .map_err(|_| LoroError::DecodeError("Invalid ID format".into()))?; + let client_id = iter + .next() + .unwrap() + .parse::() + .map_err(|_| LoroError::DecodeError("Invalid ID format".into()))?; + Ok(IdLp { + peer: client_id, + lamport, + }) + } +} + +impl PartialOrd for ID { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for ID { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + match self.peer.cmp(&other.peer) { + core::cmp::Ordering::Equal => self.counter.cmp(&other.counter), + ord => ord, + } + } +} + +pub const ROOT_ID: ID = ID { + peer: PeerID::MAX, + counter: i32::MAX, +}; + +impl From for ID { + fn from(id: u128) -> Self { + ID { + peer: (id >> 64) as PeerID, + counter: id as Counter, + } + } +} + +impl ID { + /// The ID of the null object. This should be use rarely. + pub const NONE_ID: ID = ID::new(u64::MAX, 0); + + #[inline] + pub const fn new(peer: PeerID, counter: Counter) -> Self { + ID { peer, counter } + } + + #[inline] + pub fn new_root() -> Self { + ROOT_ID + } + + #[inline] + pub fn is_null(&self) -> bool { + self.peer == PeerID::MAX + } + + #[inline] + pub fn to_span(&self, len: usize) -> IdSpan { + IdSpan { + peer: self.peer, + counter: CounterSpan::new(self.counter, self.counter + len as Counter), + } + } + + #[inline] + pub fn unknown(counter: Counter) -> Self { + ID { + peer: UNKNOWN, + counter, + } + } + + #[inline] + pub fn is_unknown(&self) -> bool { + self.peer == UNKNOWN + } + + #[inline] + #[allow(dead_code)] + pub(crate) fn is_connected_id(&self, other: &Self, self_len: usize) -> bool { + self.peer == other.peer && self.counter + self_len as Counter == other.counter + } + + #[inline] + pub fn inc(&self, inc: i32) -> Self { + ID { + peer: self.peer, + counter: self.counter.saturating_add(inc), + } + } + + #[inline] + pub fn contains(&self, len: Counter, target: ID) -> bool { + self.peer == target.peer + && self.counter <= target.counter + && target.counter < self.counter + len + } +} + +impl From for u128 { + fn from(id: ID) -> Self { + ((id.peer as u128) << 64) | id.counter as u128 + } +} + +impl RangeBounds for (ID, ID) { + fn start_bound(&self) -> std::ops::Bound<&ID> { + std::ops::Bound::Included(&self.0) + } + + fn end_bound(&self) -> std::ops::Bound<&ID> { + std::ops::Bound::Excluded(&self.1) + } +} + +impl IdLp { + pub const NONE_ID: IdLp = IdLp::new(u64::MAX, 0); + + #[inline] + pub const fn new(peer: PeerID, lp: Lamport) -> Self { + Self { peer, lamport: lp } + } + + pub fn inc(&self, offset: i32) -> IdLp { + IdLp { + peer: self.peer, + lamport: (self.lamport as i32 + offset) as Lamport, + } + } + + pub fn is_none(&self) -> bool { + self.peer == PeerID::MAX + } +} + +impl From for IdLpSpan { + fn from(value: IdLp) -> Self { + IdLpSpan { + peer: value.peer, + lamport: crate::LamportSpan { + start: value.lamport, + end: value.lamport + 1, + }, + } + } +} + +impl IdFull { + pub const NONE_ID: IdFull = IdFull { + peer: PeerID::MAX, + lamport: 0, + counter: 0, + }; + + pub fn new(peer: PeerID, counter: Counter, lamport: Lamport) -> Self { + Self { + peer, + lamport, + counter, + } + } + + pub fn inc(&self, offset: i32) -> IdFull { + IdFull { + peer: self.peer, + lamport: (self.lamport as i32 + offset) as Lamport, + counter: self.counter + offset as Counter, + } + } + + pub fn id(&self) -> ID { + ID { + peer: self.peer, + counter: self.counter, + } + } + + pub fn idlp(&self) -> IdLp { + IdLp { + peer: self.peer, + lamport: self.lamport, + } + } +} diff --git a/3rdparty/loro/crates/loro-common/src/internal_string.rs b/3rdparty/loro/crates/loro-common/src/internal_string.rs new file mode 100644 index 0000000000..cdb6077a2c --- /dev/null +++ b/3rdparty/loro/crates/loro-common/src/internal_string.rs @@ -0,0 +1,315 @@ +use rustc_hash::FxHashSet; +use serde::{Deserialize, Serialize}; +use std::borrow::Borrow; +use std::slice; +use std::sync::LazyLock; +use std::{ + fmt::Display, + num::NonZeroU64, + ops::Deref, + sync::{atomic::AtomicUsize, Arc, Mutex}, +}; + +const INLINE_TAG: u8 = 0b_01; +const TAG_MASK: u64 = 0b_11; +const LEN_OFFSET: u64 = 4; +const LEN_MASK: u64 = 0xF0; + +#[derive(Clone)] +pub struct InternalString { + data: InternalStringData, +} + +#[derive(Clone)] +enum InternalStringData { + Inline(NonZeroU64), + Dynamic(Arc>), +} + +impl std::fmt::Debug for InternalString { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("InternalString(")?; + std::fmt::Debug::fmt(self.as_str(), f)?; + f.write_str(")") + } +} + +impl std::hash::Hash for InternalString { + fn hash(&self, state: &mut H) { + self.as_str().hash(state); + } +} + +impl PartialEq for InternalString { + fn eq(&self, other: &Self) -> bool { + self.as_str() == other.as_str() + } +} + +impl Eq for InternalString {} + +impl PartialOrd for InternalString { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for InternalString { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.as_str().cmp(other.as_str()) + } +} + +impl Serialize for InternalString { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(self.as_str()) + } +} + +impl<'de> Deserialize<'de> for InternalString { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let s = String::deserialize(deserializer)?; + Ok(InternalString::from(s.as_str())) + } +} + +impl Default for InternalString { + fn default() -> Self { + let v: u64 = INLINE_TAG as u64; + Self { + // SAFETY: INLINE_TAG is non-zero + data: InternalStringData::new_inline(unsafe { NonZeroU64::new_unchecked(v) }), + } + } +} + +impl InternalString { + pub fn as_str(&self) -> &str { + match &self.data { + InternalStringData::Inline(inline) => unsafe { + let len = (inline.get() & LEN_MASK) >> LEN_OFFSET; + let src = inline_atom_slice(inline); + // SAFETY: the chosen range is guaranteed to be valid str + std::str::from_utf8_unchecked(&src[..(len as usize)]) + }, + InternalStringData::Dynamic(dynamic) => dynamic.deref(), + } + } +} + +impl InternalStringData { + fn new_inline(inline: NonZeroU64) -> Self { + debug_assert_eq!((inline.get() & TAG_MASK) as u8, INLINE_TAG); + Self::Inline(inline) + } + + fn new_dynamic(dynamic: Arc>) -> Self { + Self::Dynamic(dynamic) + } +} + +impl Drop for InternalStringData { + fn drop(&mut self) { + if let InternalStringData::Dynamic(arc) = self { + if Arc::strong_count(arc) == 2 { + drop_cache(arc.clone()); + } + } + } +} + +impl AsRef for InternalString { + fn as_ref(&self) -> &str { + self.as_str() + } +} + +impl From<&str> for InternalString { + #[inline(always)] + fn from(s: &str) -> Self { + if s.len() <= 7 { + let mut v: u64 = (INLINE_TAG as u64) | ((s.len() as u64) << LEN_OFFSET); + let arr = inline_atom_slice_mut(&mut v); + arr[..s.len()].copy_from_slice(s.as_bytes()); + Self { + // SAFETY: The tag is 1 + data: InternalStringData::new_inline(unsafe { NonZeroU64::new_unchecked(v) }), + } + } else { + let ans: Arc> = get_or_init_internalized_string(s); + Self { + data: InternalStringData::new_dynamic(ans), + } + } + } +} + +#[inline(always)] +fn inline_atom_slice(x: &NonZeroU64) -> &[u8] { + unsafe { + let x: *const NonZeroU64 = x; + let mut data = x as *const u8; + // All except the lowest byte, which is first in little-endian, last in big-endian. + if cfg!(target_endian = "little") { + data = data.offset(1); + } + let len = 7; + slice::from_raw_parts(data, len) + } +} + +#[inline(always)] +fn inline_atom_slice_mut(x: &mut u64) -> &mut [u8] { + unsafe { + let x: *mut u64 = x; + let mut data = x as *mut u8; + // All except the lowest byte, which is first in little-endian, last in big-endian. + if cfg!(target_endian = "little") { + data = data.offset(1); + } + let len = 7; + slice::from_raw_parts_mut(data, len) + } +} + +impl From for InternalString { + fn from(s: String) -> Self { + Self::from(s.as_str()) + } +} + +impl From<&InternalString> for String { + #[inline(always)] + fn from(value: &InternalString) -> Self { + value.as_str().to_string() + } +} + +impl Display for InternalString { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.as_str().fmt(f) + } +} + +impl Deref for InternalString { + type Target = str; + + fn deref(&self) -> &Self::Target { + self.as_str() + } +} + +#[derive(Hash, PartialEq, Eq)] +struct ArcWrapper(Arc>); + +impl Borrow for ArcWrapper { + fn borrow(&self) -> &str { + &self.0 + } +} + +static STRING_SET: LazyLock>> = + LazyLock::new(|| Mutex::new(FxHashSet::default())); + +fn get_or_init_internalized_string(s: &str) -> Arc> { + static MAX_MET_CACHE_SIZE: AtomicUsize = AtomicUsize::new(1 << 16); + + let mut set = STRING_SET.lock().unwrap(); + if let Some(v) = set.get(s) { + v.0.clone() + } else { + let ans: Arc> = Arc::new(Box::from(s)); + set.insert(ArcWrapper(ans.clone())); + let max = MAX_MET_CACHE_SIZE.load(std::sync::atomic::Ordering::Relaxed); + if set.capacity() >= max { + let old = set.len(); + set.retain(|s| Arc::strong_count(&s.0) > 1); + let new = set.len(); + if old - new > new / 2 { + set.shrink_to_fit(); + } + + MAX_MET_CACHE_SIZE.store(max * 2, std::sync::atomic::Ordering::Relaxed); + } + + ans + } +} + +fn drop_cache(s: Arc>) { + let mut set = STRING_SET.lock().unwrap(); + set.remove(&ArcWrapper(s)); + if set.len() < set.capacity() / 2 && set.capacity() > 128 { + set.shrink_to_fit(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_string_cache() { + let s1 = InternalString::from("hello"); + let s3 = InternalString::from("world"); + + // Content should match + assert_eq!("hello", s1.as_str()); + assert_eq!(s3.as_str(), "world"); + } + + #[cfg(all(miri, target_pointer_width = "32"))] + #[test] + fn miri_dynamic_string_does_not_read_uninitialized_tag_bytes_on_32_bit() { + let s = InternalString::from("long enough to use the dynamic representation"); + + assert_eq!(s.as_str(), "long enough to use the dynamic representation"); + } + + #[test] + fn test_long_string_cache() { + let long_str1 = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua."; + let long_str2 = "A very long string that contains lots of repeated characters: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + + let s1 = InternalString::from(long_str1); + let s2 = InternalString::from(long_str1); + let s3 = InternalString::from(long_str2); + + // Same long strings should be equal + assert_eq!(s1, s2); + + // Different long strings should be different + assert_ne!(s1, s3); + + // Content should match exactly + assert_eq!(s1.as_str(), long_str1); + assert_eq!(s1.as_str(), long_str1); + assert_eq!(s2.as_str(), long_str1); + assert_eq!(s3.as_str(), long_str2); + + // Internal pointers should be same for equal strings + assert!(std::ptr::eq(s1.as_str().as_ptr(), s2.as_str().as_ptr())); + assert!(!std::ptr::eq(s1.as_str().as_ptr(), s3.as_str().as_ptr())); + } + + #[test] + fn test_long_string_cache_drop() { + { + let set = STRING_SET.lock().unwrap(); + assert_eq!(set.len(), 0); + } + { + let s1 = InternalString::from("hello".repeat(10)); + let s2 = InternalString::from("hello".repeat(10)); + assert!(std::ptr::eq(s1.as_str().as_ptr(), s2.as_str().as_ptr())); + } + let set = STRING_SET.lock().unwrap(); + assert_eq!(set.len(), 0); + } +} diff --git a/3rdparty/loro/crates/loro-common/src/lib.rs b/3rdparty/loro/crates/loro-common/src/lib.rs new file mode 100644 index 0000000000..9503c9e151 --- /dev/null +++ b/3rdparty/loro/crates/loro-common/src/lib.rs @@ -0,0 +1,1521 @@ +#![allow(unused_assignments)] + +use std::{fmt::Display, io::Write}; + +use arbitrary::Arbitrary; +use enum_as_inner::EnumAsInner; + +use nonmax::{NonMaxI32, NonMaxU32}; +use serde::{Deserialize, Serialize}; + +mod error; +mod id; +mod internal_string; +mod logging; +mod macros; +mod span; +mod value; + +pub use error::{LoroEncodeError, LoroError, LoroResult, LoroTreeError}; +pub use internal_string::InternalString; +pub use logging::log::*; +#[doc(hidden)] +pub use rustc_hash::FxHashMap; +pub use span::*; +pub use value::{ + to_value, LoroBinaryValue, LoroListValue, LoroMapValue, LoroStringValue, LoroValue, +}; + +/// Unique id for each peer. It's a random u64 by default. +pub type PeerID = u64; +/// If it's the nth Op of a peer, the counter will be n. +pub type Counter = i32; +/// It's the [Lamport clock](https://en.wikipedia.org/wiki/Lamport_timestamp) +pub type Lamport = u32; + +/// It's the unique ID of an Op represented by [PeerID] and [Counter]. +#[derive(PartialEq, Eq, Hash, Clone, Copy, Serialize, Deserialize)] +pub struct ID { + pub peer: PeerID, + pub counter: Counter, +} + +impl ID { + pub fn to_bytes(&self) -> [u8; 12] { + let mut bytes = [0; 12]; + bytes[..8].copy_from_slice(&self.peer.to_be_bytes()); + bytes[8..].copy_from_slice(&self.counter.to_be_bytes()); + bytes + } + + pub fn from_bytes(bytes: &[u8]) -> Self { + if bytes.len() != 12 { + panic!( + "Invalid ID bytes. Expected 12 bytes but got {} bytes", + bytes.len() + ); + } + + Self { + peer: u64::from_be_bytes(bytes[..8].try_into().unwrap()), + counter: i32::from_be_bytes(bytes[8..].try_into().unwrap()), + } + } +} + +/// It's the unique ID of an Op represented by [PeerID] and [Counter]. +#[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)] +pub struct CompactId { + pub peer: PeerID, + pub counter: NonMaxI32, +} + +/// Namespace prefix used to encode mergeable container IDs into Root container names. +/// +/// The 🤝 ("handshake") sentinel mirrors Loro's `🦜:` brand convention +/// (`crates/loro-common/src/value.rs::LORO_CONTAINER_ID_PREFIX`) and signals +/// "two peers agreeing on the same cid." The trailing `:` separates the brand +/// from the flattened path payload; the container kind is carried on the +/// `Root` itself and is not duplicated in the name. `check_root_container_name` +/// rejects user-created root names that start with this prefix. +/// +/// **Cost of nesting mergeable maps inside mergeable maps**: the payload embeds +/// the nearest non-mergeable map ancestor once, followed by escaped map keys. +/// Names grow linearly with nesting depth × average key length and ride through +/// every op header, snapshot, and event path. +pub const MERGEABLE_NAMESPACE_PREFIX: &str = "🤝:"; + +fn write_len_prefixed_segment(out: &mut Vec, bytes: &[u8]) { + leb128::write::unsigned(out, bytes.len() as u64).unwrap(); + out.extend_from_slice(bytes); +} + +fn push_mergeable_escaped(out: &mut String, segment: &str) { + for ch in segment.chars() { + match ch { + '\\' => out.push_str("\\\\"), + '>' => out.push_str("\\>"), + '/' => out.push_str("\\s"), + '\0' => out.push_str("\\0"), + _ => out.push(ch), + } + } +} + +fn decode_mergeable_segment(segment: &str) -> Option { + let mut out = String::with_capacity(segment.len()); + let mut chars = segment.chars(); + while let Some(ch) = chars.next() { + match ch { + '\\' => { + let escaped = chars.next()?; + match escaped { + '\\' => out.push('\\'), + '>' => out.push('>'), + 's' => out.push('/'), + '0' => out.push('\0'), + _ => return None, + } + } + '/' | '\0' => return None, + _ => out.push(ch), + } + } + + Some(out) +} + +fn find_first_mergeable_separator(payload: &str) -> Option { + let mut escaped = false; + for (idx, ch) in payload.char_indices() { + if escaped { + escaped = false; + continue; + } + + match ch { + '\\' => escaped = true, + '>' => return Some(idx), + _ => {} + } + } + + None +} + +fn find_last_mergeable_separator(payload: &str) -> Option { + let mut last = None; + let mut escaped = false; + for (idx, ch) in payload.char_indices() { + if escaped { + escaped = false; + continue; + } + + match ch { + '\\' => escaped = true, + '>' => last = Some(idx), + _ => {} + } + } + + if escaped { + None + } else { + last + } +} + +fn push_u64_base36(out: &mut String, mut value: u64) { + const DIGITS: &[u8; 36] = b"0123456789abcdefghijklmnopqrstuvwxyz"; + if value == 0 { + out.push('0'); + return; + } + + let mut buf = [0u8; 13]; + let mut idx = buf.len(); + while value > 0 { + idx -= 1; + buf[idx] = DIGITS[(value % 36) as usize]; + value /= 36; + } + + out.push_str(std::str::from_utf8(&buf[idx..]).unwrap()); +} + +fn push_i32_base36(out: &mut String, value: i32) { + if value < 0 { + out.push('-'); + } + let magnitude = if value < 0 { + (-(i64::from(value))) as u64 + } else { + value as u64 + }; + push_u64_base36(out, magnitude); +} + +fn parse_base36_digit(byte: u8) -> Option { + match byte { + b'0'..=b'9' => Some(u64::from(byte - b'0')), + b'a'..=b'z' => Some(u64::from(byte - b'a' + 10)), + _ => None, + } +} + +fn parse_u64_base36(s: &str) -> Option { + if s.is_empty() { + return None; + } + if s.len() > 1 && s.starts_with('0') { + return None; + } + + let mut value = 0u64; + for byte in s.bytes() { + value = value.checked_mul(36)?; + value = value.checked_add(parse_base36_digit(byte)?)?; + } + Some(value) +} + +fn parse_i32_base36(s: &str) -> Option { + if let Some(rest) = s.strip_prefix('-') { + let value = parse_u64_base36(rest)?; + if value == 0 { + return None; + } + let signed = -(i64::try_from(value).ok()?); + i32::try_from(signed).ok() + } else { + let value = parse_u64_base36(s)?; + i32::try_from(value).ok() + } +} + +fn parse_mergeable_base_parent(segment: &str) -> Option { + let mut chars = segment.chars(); + match chars.next()? { + '$' => { + let name = &segment['$'.len_utf8()..]; + if !check_root_container_name(name) { + return None; + } + Some(ContainerID::Root { + name: name.into(), + container_type: ContainerType::Map, + }) + } + '@' => { + let rest = &segment['@'.len_utf8()..]; + let (peer, counter) = rest.split_once(':')?; + Some(ContainerID::Normal { + peer: parse_u64_base36(peer)?, + counter: parse_i32_base36(counter)?, + container_type: ContainerType::Map, + }) + } + _ => None, + } +} + +fn validate_mergeable_payload(payload: &str) -> Option<()> { + let separator = find_first_mergeable_separator(payload)?; + validate_mergeable_base_parent(&payload[..separator])?; + validate_mergeable_key_path(&payload[separator + '>'.len_utf8()..]) +} + +fn validate_mergeable_base_parent(segment: &str) -> Option<()> { + match segment.as_bytes().first().copied()? { + b'$' => validate_mergeable_root_base_parent(&segment['$'.len_utf8()..]), + b'@' => validate_mergeable_normal_base_parent(&segment['@'.len_utf8()..]), + _ => None, + } +} + +fn validate_mergeable_root_base_parent(name: &str) -> Option<()> { + if name.is_empty() || name.starts_with(MERGEABLE_NAMESPACE_PREFIX) { + return None; + } + + let mut chars = name.chars(); + while let Some(ch) = chars.next() { + match ch { + '\\' => match chars.next()? { + '\\' | '>' => {} + // Root container names cannot contain slash or NUL after decoding. + 's' | '0' => return None, + _ => return None, + }, + '/' | '\0' => return None, + _ => {} + } + } + + Some(()) +} + +fn validate_mergeable_normal_base_parent(segment: &str) -> Option<()> { + let (peer, counter) = segment.split_once(':')?; + parse_u64_base36(peer)?; + parse_i32_base36(counter)?; + Some(()) +} + +fn validate_mergeable_key_path(path: &str) -> Option<()> { + let mut chars = path.chars(); + while let Some(ch) = chars.next() { + match ch { + '\\' => match chars.next()? { + '\\' | '>' | 's' | '0' => {} + _ => return None, + }, + '/' | '\0' => return None, + _ => {} + } + } + + Some(()) +} + +fn parse_mergeable_payload(payload: &str) -> Option<(ContainerID, String)> { + validate_mergeable_payload(payload)?; + let separator = find_last_mergeable_separator(payload)?; + let parent_payload = &payload[..separator]; + let key = decode_mergeable_segment(&payload[separator + '>'.len_utf8()..])?; + + let parent = if find_last_mergeable_separator(parent_payload).is_some() { + ContainerID::Root { + name: format!("{MERGEABLE_NAMESPACE_PREFIX}{parent_payload}").into(), + container_type: ContainerType::Map, + } + } else { + let parent_segment = decode_mergeable_segment(parent_payload)?; + parse_mergeable_base_parent(&parent_segment)? + }; + + Some((parent, key)) +} + +fn mergeable_payload_for_parent(parent: &ContainerID) -> Option<&str> { + let ContainerID::Root { + name, + container_type: ContainerType::Map, + } = parent + else { + return None; + }; + let payload = name.as_str().strip_prefix(MERGEABLE_NAMESPACE_PREFIX)?; + validate_mergeable_payload(payload)?; + Some(payload) +} + +fn push_mergeable_parent(out: &mut String, parent: &ContainerID) { + if let Some(payload) = mergeable_payload_for_parent(parent) { + out.push_str(payload); + return; + } + + match parent { + ContainerID::Root { name, .. } => { + out.push('$'); + push_mergeable_escaped(out, name.as_str()); + } + ContainerID::Normal { peer, counter, .. } => { + out.push('@'); + push_u64_base36(out, *peer); + out.push(':'); + push_i32_base36(out, *counter); + } + } +} + +/// Return whether the given name is a valid root container name. +pub fn check_root_container_name(name: &str) -> bool { + !name.is_empty() + && !name.starts_with(MERGEABLE_NAMESPACE_PREFIX) + && name.char_indices().all(|(_, x)| x != '/' && x != '\0') +} + +/// Binary marker stored in a parent map slot to activate a mergeable child. +/// +/// This is a compact mergeable-child container ref, not the child cid itself. The child cid is +/// derived deterministically from `(parent, key, kind)`; this value only records that the parent map +/// slot currently activates that child kind. +/// +/// Only this specially constructed binary value activates a mergeable child. +/// Old clients that do not understand mergeable containers should see an inert binary scalar rather +/// than a fake child container edge or a reserved-looking user string. +pub const MERGEABLE_MARKER_MAGIC: [u8; 4] = [0x00, b'L', b'M', 0x01]; + +const MERGEABLE_MARKER_DIGEST_LEN: usize = 3; +const MERGEABLE_MARKER_LEN: usize = 4 + 1 + MERGEABLE_MARKER_DIGEST_LEN; +const MERGEABLE_MARKER_CRC_DOMAIN: &[u8] = b"loro.mergeable.marker.v1"; + +/// Build the [`LoroValue`] a parent map stores at a mergeable key for `container_type`. +/// +/// Layout: `MAGIC[4] + KIND[1] + CRC24(parent_id, key, kind)[3]`. +pub fn mergeable_marker( + parent: &ContainerID, + key: &str, + container_type: ContainerType, +) -> LoroValue { + let mut marker = Vec::with_capacity(MERGEABLE_MARKER_LEN); + marker.extend_from_slice(&MERGEABLE_MARKER_MAGIC); + marker.push(container_type.to_u8()); + let digest = mergeable_marker_crc24(parent, key, container_type); + marker.extend_from_slice(&digest); + LoroValue::Binary(marker.into()) +} + +/// Parse a parent map slot value back into the mergeable [`ContainerType`] it activates. +/// +/// The marker is bound to `(parent, key, kind)`, so copying it to another map/key does not activate +/// a mergeable child there. Malformed markers and arbitrary non-marker values are treated as +/// ordinary user values. +pub fn parse_mergeable_marker( + parent: &ContainerID, + key: &str, + value: &LoroValue, +) -> Option { + let LoroValue::Binary(bytes) = value else { + return None; + }; + if bytes.len() != MERGEABLE_MARKER_LEN || !bytes.starts_with(&MERGEABLE_MARKER_MAGIC) { + return None; + } + + let kind = ContainerType::try_from_u8(bytes[MERGEABLE_MARKER_MAGIC.len()]).ok()?; + if matches!(kind, ContainerType::Unknown(_)) { + return None; + } + + let digest_start = MERGEABLE_MARKER_MAGIC.len() + 1; + let expected = mergeable_marker_crc24(parent, key, kind); + if &bytes[digest_start..] != expected.as_slice() { + return None; + } + + Some(kind) +} + +/// Translate a raw map slot value into the user-visible Container view for a Map at `parent`. +/// +/// Mergeable child activation lives as a binary marker in the parent map's value table. This +/// is the canonical conversion from that marker to the deterministic child cid; every read +/// boundary (`MapHandler` getters, Map diff emission, local-event hints) goes through it so +/// callers see the same shape as a regular child container. +pub fn translate_mergeable_marker_value( + parent: &ContainerID, + key: &str, + value: LoroValue, +) -> LoroValue { + match parse_mergeable_marker(parent, key, &value) { + Some(kind) => LoroValue::Container(ContainerID::new_mergeable(parent, key, kind)), + None => value, + } +} + +fn mergeable_marker_crc24(parent: &ContainerID, key: &str, kind: ContainerType) -> [u8; 3] { + let mut input = Vec::new(); + input.extend_from_slice(MERGEABLE_MARKER_CRC_DOMAIN); + write_len_prefixed_segment(&mut input, &parent.to_bytes()); + write_len_prefixed_segment(&mut input, key.as_bytes()); + input.push(kind.to_u8()); + + let crc = crc32(&input) & 0x00ff_ffff; + [ + ((crc >> 16) & 0xff) as u8, + ((crc >> 8) & 0xff) as u8, + (crc & 0xff) as u8, + ] +} + +fn crc32(bytes: &[u8]) -> u32 { + let mut crc = 0xffff_ffff_u32; + for &byte in bytes { + crc ^= u32::from(byte); + for _ in 0..8 { + let mask = (crc & 1).wrapping_neg(); + crc = (crc >> 1) ^ (0xedb8_8320 & mask); + } + } + !crc +} + +impl CompactId { + pub fn new(peer: PeerID, counter: Counter) -> Self { + Self { + peer, + counter: NonMaxI32::new(counter).unwrap(), + } + } + + pub fn to_id(&self) -> ID { + ID { + peer: self.peer, + counter: self.counter.get(), + } + } + + pub fn inc(&self, start: i32) -> CompactId { + Self { + peer: self.peer, + counter: NonMaxI32::new(start + self.counter.get()).unwrap(), + } + } +} + +impl TryFrom for CompactId { + type Error = ID; + + fn try_from(id: ID) -> Result { + if id.counter == i32::MAX { + return Err(id); + } + + Ok(Self::new(id.peer, id.counter)) + } +} + +/// It's the unique ID of an Op represented by [PeerID] and [Lamport] clock. +/// It's used to define the total order of Ops. +#[derive(PartialEq, Eq, Hash, Clone, Copy, Serialize, Deserialize, PartialOrd, Ord)] +pub struct IdLp { + pub lamport: Lamport, + pub peer: PeerID, +} + +impl IdLp { + pub fn compact(self) -> CompactIdLp { + CompactIdLp::new(self.peer, self.lamport) + } +} + +/// It's the unique ID of an Op represented by [PeerID] and [Counter]. +#[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)] +pub struct CompactIdLp { + pub peer: PeerID, + pub lamport: NonMaxU32, +} + +impl CompactIdLp { + pub fn new(peer: PeerID, lamport: Lamport) -> Self { + Self { + peer, + lamport: NonMaxU32::new(lamport).unwrap(), + } + } + + pub fn to_id(&self) -> IdLp { + IdLp { + peer: self.peer, + lamport: self.lamport.get(), + } + } +} + +impl TryFrom for CompactIdLp { + type Error = IdLp; + + fn try_from(id: IdLp) -> Result { + if id.lamport == u32::MAX { + return Err(id); + } + + Ok(Self::new(id.peer, id.lamport)) + } +} + +/// It's the unique ID of an Op represented by [PeerID], [Lamport] clock and [Counter]. +#[derive(Debug, PartialEq, Eq, Hash, Clone, Copy, Serialize, Deserialize)] +pub struct IdFull { + pub peer: PeerID, + pub lamport: Lamport, + pub counter: Counter, +} + +/// [ContainerID] includes the Op's [ID] and the type. So it's impossible to have +/// the same [ContainerID] with conflict [ContainerType]. +/// +/// This structure is really cheap to clone. +/// +/// String representation: +/// +/// - Root Container: `/:` +/// - Normal Container: `@:` +/// +/// Note: It will be encoded into binary format, so the order of its fields should not be changed. +#[derive(Hash, PartialEq, Eq, Clone, Serialize, Deserialize, EnumAsInner)] +pub enum ContainerID { + /// Root container does not need an op to create. It can be created implicitly. + Root { + name: InternalString, + container_type: ContainerType, + }, + Normal { + peer: PeerID, + counter: Counter, + container_type: ContainerType, + }, +} + +impl ContainerID { + pub fn encode(&self, writer: &mut W) -> Result<(), std::io::Error> { + match self { + Self::Root { + name, + container_type, + } => { + let first_byte = container_type.to_u8() | 0b10000000; + writer.write_all(&[first_byte])?; + leb128::write::unsigned(writer, name.len() as u64)?; + writer.write_all(name.as_bytes())?; + } + Self::Normal { + peer, + counter, + container_type, + } => { + let first_byte = container_type.to_u8(); + writer.write_all(&[first_byte])?; + writer.write_all(&peer.to_le_bytes())?; + writer.write_all(&counter.to_le_bytes())?; + } + } + + Ok(()) + } + + pub fn to_bytes(&self) -> Vec { + // normal need 13 bytes + let mut bytes = Vec::with_capacity(13); + self.encode(&mut bytes).unwrap(); + bytes + } + + pub fn from_bytes(bytes: &[u8]) -> Self { + Self::try_from_bytes(bytes).unwrap() + } + + pub fn try_from_bytes(bytes: &[u8]) -> LoroResult { + if bytes.is_empty() { + return Err(LoroError::DecodeError( + "Decode container id failed".to_string().into_boxed_str(), + )); + } + + let first_byte = bytes[0]; + let container_type = ContainerType::try_from_u8(first_byte & 0b01111111)?; + let is_root = (first_byte & 0b10000000) != 0; + + let mut reader = &bytes[1..]; + match is_root { + true => { + let name_len = leb128::read::unsigned(&mut reader).map_err(|_| { + LoroError::DecodeError( + "Decode container id failed".to_string().into_boxed_str(), + ) + })?; + let name_len = usize::try_from(name_len).map_err(|_| { + LoroError::DecodeError( + "Decode container id failed".to_string().into_boxed_str(), + ) + })?; + if reader.len() != name_len { + return Err(LoroError::DecodeError( + "Decode container id failed".to_string().into_boxed_str(), + )); + } + + let name = std::str::from_utf8(&reader[..name_len]).map_err(|_| { + LoroError::DecodeError( + "Decode container id failed".to_string().into_boxed_str(), + ) + })?; + Ok(Self::Root { + name: InternalString::from(name), + container_type, + }) + } + false => { + if reader.len() != 12 { + return Err(LoroError::DecodeError( + "Decode container id failed".to_string().into_boxed_str(), + )); + } + + let peer = PeerID::from_le_bytes(reader[..8].try_into().unwrap()); + let counter = i32::from_le_bytes(reader[8..12].try_into().unwrap()); + Ok(Self::Normal { + peer, + counter, + container_type, + }) + } + } + } + + const LORO_CONTAINER_ID_PREFIX: &str = "🦜:"; + pub fn to_loro_value_string(&self) -> String { + format!("{}{}", Self::LORO_CONTAINER_ID_PREFIX, self) + } + + pub fn try_from_loro_value_string(s: &str) -> Option { + if let Some(s) = s.strip_prefix(Self::LORO_CONTAINER_ID_PREFIX) { + Self::try_from(s).ok() + } else { + None + } + } +} + +impl std::fmt::Debug for ContainerID { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Root { + name, + container_type, + } => { + write!(f, "Root(\"{name}\" {container_type:?})") + } + Self::Normal { + peer, + counter, + container_type, + } => { + write!(f, "Normal({container_type:?} {counter}@{peer})") + } + } + } +} + +// TODO: add non_exhausted +// Note: It will be encoded into binary format, so the order of its fields should not be changed. +#[derive(Arbitrary, Debug, PartialEq, Eq, Hash, Clone, Copy, PartialOrd, Ord)] +pub enum ContainerType { + Text, + Map, + List, + MovableList, + Tree, + #[cfg(feature = "counter")] + Counter, + Unknown(u8), +} + +impl ContainerType { + #[cfg(feature = "counter")] + pub const ALL_TYPES: [ContainerType; 6] = [ + ContainerType::Map, + ContainerType::List, + ContainerType::Text, + ContainerType::Tree, + ContainerType::MovableList, + ContainerType::Counter, + ]; + #[cfg(not(feature = "counter"))] + pub const ALL_TYPES: [ContainerType; 5] = [ + ContainerType::Map, + ContainerType::List, + ContainerType::Text, + ContainerType::Tree, + ContainerType::MovableList, + ]; + + pub fn default_value(&self) -> LoroValue { + match self { + ContainerType::Map => LoroValue::Map(Default::default()), + ContainerType::List => LoroValue::List(Default::default()), + ContainerType::Text => LoroValue::String(Default::default()), + ContainerType::Tree => LoroValue::List(Default::default()), + ContainerType::MovableList => LoroValue::List(Default::default()), + #[cfg(feature = "counter")] + ContainerType::Counter => LoroValue::Double(0.), + ContainerType::Unknown(_) => unreachable!(), + } + } + + pub fn to_u8(self) -> u8 { + match self { + ContainerType::Map => 0, + ContainerType::List => 1, + ContainerType::Text => 2, + ContainerType::Tree => 3, + ContainerType::MovableList => 4, + #[cfg(feature = "counter")] + ContainerType::Counter => 5, + ContainerType::Unknown(k) => k, + } + } + + pub fn try_from_u8(v: u8) -> LoroResult { + match v { + 0 => Ok(ContainerType::Map), + 1 => Ok(ContainerType::List), + 2 => Ok(ContainerType::Text), + 3 => Ok(ContainerType::Tree), + 4 => Ok(ContainerType::MovableList), + #[cfg(feature = "counter")] + 5 => Ok(ContainerType::Counter), + x => Ok(ContainerType::Unknown(x)), + } + } +} + +#[derive(Serialize, Deserialize)] +#[serde(rename = "ContainerType")] +enum ContainerTypeSerdeRepr { + Text, + Map, + List, + MovableList, + Tree, + #[cfg(feature = "counter")] + Counter, + Unknown(u8), +} + +// For some historical reason, we have another to_byte format for ContainerType, +// it was used for serde of ContainerType +fn historical_container_type_to_byte(c: ContainerType) -> u8 { + match c { + ContainerType::Text => 0, + ContainerType::Map => 1, + ContainerType::List => 2, + ContainerType::MovableList => 3, + ContainerType::Tree => 4, + #[cfg(feature = "counter")] + ContainerType::Counter => 5, + ContainerType::Unknown(k) => k, + } +} + +fn historical_try_byte_to_container(byte: u8) -> ContainerType { + match byte { + 0 => ContainerType::Text, + 1 => ContainerType::Map, + 2 => ContainerType::List, + 3 => ContainerType::MovableList, + 4 => ContainerType::Tree, + #[cfg(feature = "counter")] + 5 => ContainerType::Counter, + _ => ContainerType::Unknown(byte), + } +} + +impl From for ContainerTypeSerdeRepr { + fn from(value: ContainerType) -> Self { + match value { + ContainerType::Text => Self::Text, + ContainerType::Map => Self::Map, + ContainerType::List => Self::List, + ContainerType::MovableList => Self::MovableList, + ContainerType::Tree => Self::Tree, + #[cfg(feature = "counter")] + ContainerType::Counter => Self::Counter, + ContainerType::Unknown(value) => Self::Unknown(value), + } + } +} + +impl From for ContainerType { + fn from(value: ContainerTypeSerdeRepr) -> Self { + match value { + ContainerTypeSerdeRepr::Text => ContainerType::Text, + ContainerTypeSerdeRepr::Map => ContainerType::Map, + ContainerTypeSerdeRepr::List => ContainerType::List, + ContainerTypeSerdeRepr::MovableList => ContainerType::MovableList, + ContainerTypeSerdeRepr::Tree => ContainerType::Tree, + #[cfg(feature = "counter")] + ContainerTypeSerdeRepr::Counter => ContainerType::Counter, + ContainerTypeSerdeRepr::Unknown(value) => ContainerType::Unknown(value), + } + } +} + +impl Serialize for ContainerType { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + if serializer.is_human_readable() { + ContainerTypeSerdeRepr::from(*self).serialize(serializer) + } else { + serializer.serialize_u8(historical_container_type_to_byte(*self)) + } + } +} + +impl<'de> Deserialize<'de> for ContainerType { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + if deserializer.is_human_readable() { + let repr = ContainerTypeSerdeRepr::deserialize(deserializer)?; + Ok(repr.into()) + } else { + let value = u8::deserialize(deserializer)?; + Ok(historical_try_byte_to_container(value)) + } + } +} + +pub type IdSpanVector = rustc_hash::FxHashMap; + +mod container { + use super::*; + + impl Display for ContainerType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + ContainerType::Map => "Map", + ContainerType::List => "List", + ContainerType::MovableList => "MovableList", + ContainerType::Text => "Text", + ContainerType::Tree => "Tree", + #[cfg(feature = "counter")] + ContainerType::Counter => "Counter", + ContainerType::Unknown(k) => return f.write_fmt(format_args!("Unknown({k})")), + }) + } + } + + impl Display for ContainerID { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ContainerID::Root { + name, + container_type, + } => f.write_fmt(format_args!("cid:root-{name}:{container_type}"))?, + ContainerID::Normal { + peer, + counter, + container_type, + } => f.write_fmt(format_args!( + "cid:{id}:{container_type}", + id = ID::new(*peer, *counter), + container_type = container_type + ))?, + }; + Ok(()) + } + } + + impl TryFrom<&str> for ContainerID { + type Error = (); + + fn try_from(mut s: &str) -> Result { + if !s.starts_with("cid:") { + return Err(()); + } + + s = &s[4..]; + if s.starts_with("root-") { + // root container + s = &s[5..]; + let split = s.rfind(':').ok_or(())?; + if split == 0 { + return Err(()); + } + let kind = ContainerType::try_from(&s[split + 1..]).map_err(|_| ())?; + let name = &s[..split]; + Ok(ContainerID::Root { + name: name.into(), + container_type: kind, + }) + } else { + let mut iter = s.split(':'); + let id = iter.next().ok_or(())?; + let kind = iter.next().ok_or(())?; + if iter.next().is_some() { + return Err(()); + } + + let id = ID::try_from(id).map_err(|_| ())?; + let kind = ContainerType::try_from(kind).map_err(|_| ())?; + Ok(ContainerID::Normal { + peer: id.peer, + counter: id.counter, + container_type: kind, + }) + } + } + } + + impl ContainerID { + #[inline] + pub fn new_normal(id: ID, container_type: ContainerType) -> Self { + ContainerID::Normal { + peer: id.peer, + counter: id.counter, + container_type, + } + } + + #[inline] + pub fn new_root(name: &str, container_type: ContainerType) -> Self { + if !check_root_container_name(name) { + panic!( + "Invalid root container name, it should not be empty or contain '/' or '\\0'" + ); + } else { + ContainerID::Root { + name: name.into(), + container_type, + } + } + } + + #[inline] + pub fn name(&self) -> &InternalString { + match self { + ContainerID::Root { name, .. } => name, + ContainerID::Normal { .. } => unreachable!(), + } + } + + #[inline] + pub fn container_type(&self) -> ContainerType { + match self { + ContainerID::Root { container_type, .. } => *container_type, + ContainerID::Normal { container_type, .. } => *container_type, + } + } + + pub fn is_unknown(&self) -> bool { + matches!(self.container_type(), ContainerType::Unknown(_)) + } + + /// Create a mergeable container ID for the given parent, key, and container type. + /// + /// The cid is a Root container with a reserved namespace prefix and a flattened + /// map-path payload, so two peers calling this with the same arguments produce the + /// identical `ContainerID`. + /// + /// The container kind is intentionally *not* encoded into the name: it already + /// lives in `ContainerID::Root::container_type`, so encoding it twice would waste two + /// bytes per cid, and `ContainerID` equality already keeps two `(parent, key)` + /// mergeable cids of different kinds distinct. + /// + /// Only maps can be mergeable parents. The encoded payload records the nearest + /// non-mergeable map ancestor (`$root-name` or `@peer:counter`) once, then appends + /// escaped map keys separated by `>`. This keeps nested mergeable map names linear + /// in path length instead of recursively embedding the full parent cid. + pub fn new_mergeable( + parent: &ContainerID, + key: &str, + container_type: ContainerType, + ) -> Self { + assert_eq!( + parent.container_type(), + ContainerType::Map, + "mergeable child parent must be a map" + ); + let parent_len_hint = match parent { + ContainerID::Root { name, .. } => name.len(), + ContainerID::Normal { .. } => 24, + }; + let cap = MERGEABLE_NAMESPACE_PREFIX.len() + parent_len_hint + key.len() + 16; + let mut name = String::with_capacity(cap); + name.push_str(MERGEABLE_NAMESPACE_PREFIX); + push_mergeable_parent(&mut name, parent); + name.push('>'); + push_mergeable_escaped(&mut name, key); + + Self::Root { + name: name.into(), + container_type, + } + } + + /// Returns `true` if this is a valid mergeable container ID (i.e. created via + /// `new_mergeable`). + /// + /// A Root whose name merely starts with the reserved mergeable prefix but does not + /// decode to a valid path payload is an ordinary root, not a mergeable + /// container. The check is structural; a fabricated `"🤝:not-valid"` Root is not + /// treated as mergeable. + /// + /// Constant-time short-circuit when the name doesn't start with the mergeable prefix + /// (most container ids). For names that do match the prefix, validates segment + /// escaping and the compact base-parent encoding. + pub fn is_mergeable(&self) -> bool { + let Self::Root { name, .. } = self else { + return false; + }; + name.as_str() + .strip_prefix(MERGEABLE_NAMESPACE_PREFIX) + .and_then(validate_mergeable_payload) + .is_some() + } + + /// Decode a mergeable container ID back into its `(parent, key, container_type)` components. + /// + /// Returns `None` if this is not a valid mergeable container ID. The returned + /// `container_type` is the type carried on the `Root` itself; the encoded payload only + /// carries `(parent, key)`. + pub fn parse_mergeable(&self) -> Option<(ContainerID, String, ContainerType)> { + let Self::Root { + name, + container_type, + } = self + else { + return None; + }; + let payload = name.strip_prefix(MERGEABLE_NAMESPACE_PREFIX)?; + let (parent, key) = parse_mergeable_payload(payload)?; + Some((parent, key, *container_type)) + } + } + + impl TryFrom<&str> for ContainerType { + type Error = LoroError; + + fn try_from(value: &str) -> Result { + match value { + "Map" | "map" => Ok(ContainerType::Map), + "List" | "list" => Ok(ContainerType::List), + "Text" | "text" => Ok(ContainerType::Text), + "Tree" | "tree" => Ok(ContainerType::Tree), + "MovableList" | "movableList" => Ok(ContainerType::MovableList), + #[cfg(feature = "counter")] + "Counter" | "counter" => Ok(ContainerType::Counter), + a => { + if a.ends_with(')') { + let start = a.find('(').ok_or_else(|| { + LoroError::DecodeError( + format!("Invalid container type string \"{value}\"").into(), + ) + })?; + let k = a[start+1..a.len() - 1].parse().map_err(|_| { + LoroError::DecodeError( + format!("Unknown container type \"{value}\". The valid options are Map|List|Text|Tree|MovableList.").into(), + ) + })?; + match ContainerType::try_from_u8(k) { + Ok(k) => Ok(k), + Err(_) => Ok(ContainerType::Unknown(k)), + } + } else { + Err(LoroError::DecodeError( + format!("Unknown container type \"{value}\". The valid options are Map|List|Text|Tree|MovableList.").into(), + )) + } + } + } + } + } +} + +/// In movable tree, we use a specific [`TreeID`] to represent the root of **ALL** deleted tree node. +/// +/// Deletion operation is equivalent to move target tree node to [`DELETED_TREE_ROOT`]. +pub const DELETED_TREE_ROOT: TreeID = TreeID { + peer: PeerID::MAX, + counter: Counter::MAX, +}; + +/// Each node of movable tree has a unique [`TreeID`] generated by Loro. +/// +/// To further represent the metadata (a MapContainer) associated with each node, +/// we also use [`TreeID`] as [`ID`] portion of [`ContainerID`]. +/// This not only allows for convenient association of metadata with each node, +/// but also ensures the uniqueness of the MapContainer. +/// +/// Special ID: +/// - [`DELETED_TREE_ROOT`]: the root of all deleted nodes. To get it by [`TreeID::delete_root()`] +#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] + +pub struct TreeID { + pub peer: PeerID, + // TODO: can use a NonMax here + pub counter: Counter, +} + +impl TreeID { + #[inline(always)] + pub fn new(peer: PeerID, counter: Counter) -> Self { + Self { peer, counter } + } + + /// return [`DELETED_TREE_ROOT`] + pub const fn delete_root() -> Self { + DELETED_TREE_ROOT + } + + /// return `true` if the `TreeID` is deleted root + pub fn is_deleted_root(&self) -> bool { + self == &DELETED_TREE_ROOT + } + + pub fn from_id(id: ID) -> Self { + Self { + peer: id.peer, + counter: id.counter, + } + } + + pub fn id(&self) -> ID { + ID { + peer: self.peer, + counter: self.counter, + } + } + + pub fn associated_meta_container(&self) -> ContainerID { + ContainerID::new_normal(self.id(), ContainerType::Map) + } +} + +impl Display for TreeID { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.id().fmt(f) + } +} + +impl TryFrom<&str> for TreeID { + type Error = LoroError; + fn try_from(value: &str) -> Result { + let id = ID::try_from(value)?; + Ok(TreeID { + peer: id.peer, + counter: id.counter, + }) + } +} + +#[cfg(feature = "wasm")] +pub mod wasm { + use crate::{LoroError, TreeID}; + use wasm_bindgen::JsValue; + impl From for JsValue { + fn from(value: TreeID) -> Self { + JsValue::from_str(&format!("{value}")) + } + } + + impl TryFrom for TreeID { + type Error = LoroError; + fn try_from(value: JsValue) -> Result { + let id = value.as_string().unwrap(); + TreeID::try_from(id.as_str()) + } + } +} + +#[cfg(test)] +mod test { + use crate::{ + mergeable_marker, parse_mergeable_marker, ContainerID, ContainerType, LoroValue, ID, + MERGEABLE_MARKER_MAGIC, + }; + + #[test] + fn mergeable_marker_round_trips_every_kind() { + let parent = ContainerID::new_root("state", ContainerType::Map); + let key = "field"; + let kinds = [ + ContainerType::Map, + ContainerType::List, + ContainerType::Text, + ContainerType::Tree, + ContainerType::MovableList, + #[cfg(feature = "counter")] + ContainerType::Counter, + ]; + for kind in kinds { + assert_eq!( + parse_mergeable_marker(&parent, key, &mergeable_marker(&parent, key, kind)), + Some(kind), + "marker value must parse back to {kind:?}" + ); + } + } + + #[test] + fn mergeable_marker_exact_layout() { + let parent = ContainerID::new_root("state", ContainerType::Map); + let value = mergeable_marker(&parent, "field", ContainerType::List); + let LoroValue::Binary(bytes) = value else { + panic!("mergeable marker must be a binary value"); + }; + + assert_eq!(bytes.len(), 8); + assert_eq!( + &bytes[..MERGEABLE_MARKER_MAGIC.len()], + MERGEABLE_MARKER_MAGIC + ); + assert_eq!( + bytes[MERGEABLE_MARKER_MAGIC.len()], + ContainerType::List.to_u8() + ); + } + + #[test] + fn parse_mergeable_marker_rejects_non_markers() { + let parent = ContainerID::new_root("state", ContainerType::Map); + + // Plain user strings are never markers. + assert_eq!( + parse_mergeable_marker(&parent, "field", &LoroValue::String("Map".into())), + None + ); + assert_eq!( + parse_mergeable_marker(&parent, "field", &LoroValue::String("not-a-marker".into())), + None + ); + + // Non-binary values are never markers. + assert_eq!( + parse_mergeable_marker(&parent, "field", &LoroValue::Double(1.0)), + None + ); + assert_eq!( + parse_mergeable_marker(&parent, "field", &LoroValue::Null), + None + ); + + // Malformed binary values stay ordinary binary values. + assert_eq!( + parse_mergeable_marker( + &parent, + "field", + &LoroValue::Binary(vec![0x00, b'L', b'M'].into()), + ), + None + ); + + let mut wrong_magic = marker_bytes(mergeable_marker(&parent, "field", ContainerType::Map)); + wrong_magic[0] = 0xff; + assert_eq!( + parse_mergeable_marker(&parent, "field", &LoroValue::Binary(wrong_magic.into()),), + None + ); + + let marker = mergeable_marker(&parent, "field", ContainerType::Map); + assert_eq!( + parse_mergeable_marker(&parent, "other", &marker), + None, + "marker digest binds the marker to its map key" + ); + + let other_parent = ContainerID::new_root("other", ContainerType::Map); + assert_eq!( + parse_mergeable_marker(&other_parent, "field", &marker), + None, + "marker digest binds the marker to its parent container" + ); + + let mut wrong_digest = marker_bytes(marker); + *wrong_digest.last_mut().unwrap() ^= 1; + assert_eq!( + parse_mergeable_marker(&parent, "field", &LoroValue::Binary(wrong_digest.into()),), + None + ); + } + + #[test] + fn is_mergeable_rejects_prefix_only_roots() { + let missing_key = ContainerID::Root { + name: "🤝:$state".into(), + container_type: ContainerType::Map, + }; + assert!( + !missing_key.is_mergeable(), + "payload without a key segment must not be mergeable" + ); + assert!(missing_key.parse_mergeable().is_none()); + + let bad_escape = ContainerID::Root { + name: "🤝:$state>not\\xvalid".into(), + container_type: ContainerType::Map, + }; + assert!( + !bad_escape.is_mergeable(), + "unknown escapes in the payload must not be mergeable" + ); + + let parent = ContainerID::new_root("state", ContainerType::Map); + let real = ContainerID::new_mergeable(&parent, "field", ContainerType::Map); + assert!( + real.is_mergeable(), + "valid mergeable cid must remain mergeable" + ); + } + + /// A `🤝:` payload can be malformed in several ways. `parse_mergeable` returns + /// `Option`, so such input must yield `None`, never panic. + #[test] + fn parse_mergeable_rejects_malformed_path_payloads() { + for name in [ + "🤝:", + "🤝:$>key", + "🤝:%state>key", + "🤝:@1>key", + "🤝:@:1>key", + "🤝:@1:zzzzzzzzzzzzzz>key", + "🤝:@Z:0>key", + "🤝:@001:0>key", + "🤝:@1:00>key", + "🤝:@1:-0>key", + "🤝:$root\\sname>key", + "🤝:$root\\0name>key", + "🤝:$state>dangling\\", + "🤝:$state>unknown\\xescape", + "🤝:$state>raw/slash", + "🤝:$state>raw\0nul", + ] { + let cid = ContainerID::Root { + name: name.into(), + container_type: ContainerType::Map, + }; + assert_eq!(cid.parse_mergeable(), None, "payload {name:?}"); + assert!(!cid.is_mergeable(), "payload {name:?}"); + } + } + + #[test] + fn parse_mergeable_marker_rejects_unknown_kind() { + let parent = ContainerID::new_root("state", ContainerType::Map); + let mut marker = marker_bytes(mergeable_marker(&parent, "field", ContainerType::Map)); + marker[MERGEABLE_MARKER_MAGIC.len()] = u8::MAX; + + assert_eq!( + parse_mergeable_marker(&parent, "field", &LoroValue::Binary(marker.into())), + None + ); + } + + fn marker_bytes(value: LoroValue) -> Vec { + let LoroValue::Binary(bytes) = value else { + panic!("expected binary mergeable marker"); + }; + bytes.to_vec() + } + + #[test] + fn test_container_id_convert_to_and_from_str() { + let id = ContainerID::Root { + name: "name".into(), + container_type: crate::ContainerType::Map, + }; + let id_str = id.to_string(); + assert_eq!(id_str.as_str(), "cid:root-name:Map"); + assert_eq!(ContainerID::try_from(id_str.as_str()).unwrap(), id); + + let id = ContainerID::Normal { + counter: 10, + peer: 255, + container_type: crate::ContainerType::Map, + }; + let id_str = id.to_string(); + assert_eq!(id_str.as_str(), "cid:10@255:Map"); + assert_eq!(ContainerID::try_from(id_str.as_str()).unwrap(), id); + + let id = ContainerID::try_from("cid:root-a:b:c:Tree").unwrap(); + assert_eq!( + id, + ContainerID::new_root("a:b:c", crate::ContainerType::Tree) + ); + } + + #[test] + fn test_convert_invalid_container_id_str() { + assert!(ContainerID::try_from("cid:root-:Map").is_err()); + assert!(ContainerID::try_from("cid:0@:Map").is_err()); + assert!(ContainerID::try_from("cid:@:Map").is_err()); + assert!(ContainerID::try_from("cid:x@0:Map").is_err()); + assert!(ContainerID::try_from("id:0@0:Map").is_err()); + assert!(ContainerID::try_from("cid:0@0:Unknown(6)").is_ok()); + } + + #[test] + fn test_container_id_encode_and_decode() { + let id = ContainerID::new_normal(ID::new(1, 2), ContainerType::Map); + let bytes = id.to_bytes(); + assert_eq!(ContainerID::from_bytes(&bytes), id); + + let id = ContainerID::new_normal(ID::new(u64::MAX, i32::MAX), ContainerType::Text); + let bytes = id.to_bytes(); + assert_eq!(ContainerID::from_bytes(&bytes), id); + + let id = ContainerID::new_root("test_root", ContainerType::List); + let bytes = id.to_bytes(); + assert_eq!(ContainerID::from_bytes(&bytes), id); + + let id = ContainerID::new_normal(ID::new(0, 0), ContainerType::MovableList); + let bytes = id.to_bytes(); + assert_eq!(ContainerID::from_bytes(&bytes), id); + + let id = ContainerID::new_root(&"x".repeat(1024), ContainerType::Tree); + let bytes = id.to_bytes(); + assert_eq!(ContainerID::from_bytes(&bytes), id); + + #[cfg(feature = "counter")] + { + let id = ContainerID::new_normal(ID::new(42, 100), ContainerType::Counter); + let bytes = id.to_bytes(); + assert_eq!(ContainerID::from_bytes(&bytes), id); + } + + let id = ContainerID::new_normal(ID::new(1, 1), ContainerType::Unknown(100)); + let bytes = id.to_bytes(); + assert_eq!(ContainerID::from_bytes(&bytes), id); + } + + #[test] + fn container_id_try_from_bytes_rejects_trailing_bytes() { + let normal = ContainerID::new_normal(ID::new(1, 2), ContainerType::Map); + let mut normal_bytes = normal.to_bytes(); + normal_bytes.push(0); + assert!(ContainerID::try_from_bytes(&normal_bytes).is_err()); + + let root = ContainerID::new_root("root", ContainerType::List); + let mut root_bytes = root.to_bytes(); + root_bytes.push(0); + assert!(ContainerID::try_from_bytes(&root_bytes).is_err()); + } +} diff --git a/3rdparty/loro/crates/loro-common/src/logging.rs b/3rdparty/loro/crates/loro-common/src/logging.rs new file mode 100644 index 0000000000..f847a1bc0e --- /dev/null +++ b/3rdparty/loro/crates/loro-common/src/logging.rs @@ -0,0 +1,28 @@ +#[cfg(not(feature = "logging"))] +pub mod log { + #[macro_export] + macro_rules! info { + ($($t:tt)*) => {}; + } + #[macro_export] + macro_rules! trace { + ($($t:tt)*) => {}; + } + #[macro_export] + macro_rules! debug { + ($($t:tt)*) => {}; + } + #[macro_export] + macro_rules! error { + ($($t:tt)*) => {}; + } + #[macro_export] + macro_rules! warn { + ($($t:tt)*) => {}; + } +} + +#[cfg(feature = "logging")] +pub mod log { + pub use tracing::{debug, error, info, trace, warn}; +} diff --git a/3rdparty/loro/crates/loro-common/src/macros.rs b/3rdparty/loro/crates/loro-common/src/macros.rs new file mode 100644 index 0000000000..b565789546 --- /dev/null +++ b/3rdparty/loro/crates/loro-common/src/macros.rs @@ -0,0 +1,316 @@ +/// A macro for creating `LoroValue`. It works just like the `json!` macro in +/// `serde_json`. +/// +/// # Example +/// +/// ``` +/// use loro_common::loro_value; +/// loro_value!({ +/// "name": "John", +/// "age": 12, +/// "collections": [ +/// { +/// "binary-data": b"1,2,3" +/// } +/// ], +/// "float": 1.2, +/// "null": null, +/// "bool": true, +/// }); +/// ``` +/// +#[macro_export(local_inner_macros)] +macro_rules! loro_value { + // Hide distracting implementation details from the generated rustdoc. + ($($json:tt)+) => { + value_internal!($($json)+) + }; +} + +// Rocket relies on this because they export their own `json!` with a different +// doc comment than ours, and various Rust bugs prevent them from calling our +// `json!` from their `json!` so they call `value_internal!` directly. Check with +// @SergioBenitez before making breaking changes to this macro. +// +// Changes are fine as long as `value_internal!` does not call any new helper +// macros and can still be invoked as `value_internal!($($json)+)`. +#[macro_export(local_inner_macros)] +#[doc(hidden)] +macro_rules! value_internal { + ////////////////////////////////////////////////////////////////////////// + // TT muncher for parsing the inside of an array [...]. Produces a vec![...] + // of the elements. + // + // Must be invoked as: value_internal!(@array [] $($tt)*) + ////////////////////////////////////////////////////////////////////////// + + // Done with trailing comma. + (@array [$($elems:expr,)*]) => { + json_internal_vec![$($elems,)*] + }; + + // Done without trailing comma. + (@array [$($elems:expr),*]) => { + json_internal_vec![$($elems),*] + }; + + // Next element is `null`. + (@array [$($elems:expr,)*] null $($rest:tt)*) => { + value_internal!(@array [$($elems,)* value_internal!(null)] $($rest)*) + }; + + // Next element is `true`. + (@array [$($elems:expr,)*] true $($rest:tt)*) => { + value_internal!(@array [$($elems,)* value_internal!(true)] $($rest)*) + }; + + // Next element is `false`. + (@array [$($elems:expr,)*] false $($rest:tt)*) => { + value_internal!(@array [$($elems,)* value_internal!(false)] $($rest)*) + }; + + // Next element is an array. + (@array [$($elems:expr,)*] [$($array:tt)*] $($rest:tt)*) => { + value_internal!(@array [$($elems,)* value_internal!([$($array)*])] $($rest)*) + }; + + // Next element is a map. + (@array [$($elems:expr,)*] {$($map:tt)*} $($rest:tt)*) => { + value_internal!(@array [$($elems,)* value_internal!({$($map)*})] $($rest)*) + }; + + // Next element is an expression followed by comma. + (@array [$($elems:expr,)*] $next:expr, $($rest:tt)*) => { + value_internal!(@array [$($elems,)* value_internal!($next),] $($rest)*) + }; + + // Last element is an expression with no trailing comma. + (@array [$($elems:expr,)*] $last:expr) => { + value_internal!(@array [$($elems,)* value_internal!($last)]) + }; + + // Comma after the most recent element. + (@array [$($elems:expr),*] , $($rest:tt)*) => { + value_internal!(@array [$($elems,)*] $($rest)*) + }; + + // Unexpected token after most recent element. + (@array [$($elems:expr),*] $unexpected:tt $($rest:tt)*) => { + json_unexpected!($unexpected) + }; + + ////////////////////////////////////////////////////////////////////////// + // TT muncher for parsing the inside of an object {...}. Each entry is + // inserted into the given map variable. + // + // Must be invoked as: value_internal!(@object $map () ($($tt)*) ($($tt)*)) + // + // We require two copies of the input tokens so that we can match on one + // copy and trigger errors on the other copy. + ////////////////////////////////////////////////////////////////////////// + + // Done. + (@object $object:ident () () ()) => {}; + + // Insert the current entry followed by trailing comma. + (@object $object:ident [$($key:tt)+] ($value:expr) , $($rest:tt)*) => { + let _ = $object.insert(($($key)+).into(), $value); + value_internal!(@object $object () ($($rest)*) ($($rest)*)); + }; + + // Current entry followed by unexpected token. + (@object $object:ident [$($key:tt)+] ($value:expr) $unexpected:tt $($rest:tt)*) => { + json_unexpected!($unexpected); + }; + + // Insert the last entry without trailing comma. + (@object $object:ident [$($key:tt)+] ($value:expr)) => { + let _ = $object.insert(($($key)+).into(), $value); + }; + + // Next value is `null`. + (@object $object:ident ($($key:tt)+) (: null $($rest:tt)*) $copy:tt) => { + value_internal!(@object $object [$($key)+] (value_internal!(null)) $($rest)*); + }; + + // Next value is `true`. + (@object $object:ident ($($key:tt)+) (: true $($rest:tt)*) $copy:tt) => { + value_internal!(@object $object [$($key)+] (value_internal!(true)) $($rest)*); + }; + + // Next value is `false`. + (@object $object:ident ($($key:tt)+) (: false $($rest:tt)*) $copy:tt) => { + value_internal!(@object $object [$($key)+] (value_internal!(false)) $($rest)*); + }; + + // Next value is an array. + (@object $object:ident ($($key:tt)+) (: [$($array:tt)*] $($rest:tt)*) $copy:tt) => { + value_internal!(@object $object [$($key)+] (value_internal!([$($array)*])) $($rest)*); + }; + + // Next value is a map. + (@object $object:ident ($($key:tt)+) (: {$($map:tt)*} $($rest:tt)*) $copy:tt) => { + value_internal!(@object $object [$($key)+] (value_internal!({$($map)*})) $($rest)*); + }; + + // Next value is an expression followed by comma. + (@object $object:ident ($($key:tt)+) (: $value:expr , $($rest:tt)*) $copy:tt) => { + value_internal!(@object $object [$($key)+] (value_internal!($value)) , $($rest)*); + }; + + // Last value is an expression with no trailing comma. + (@object $object:ident ($($key:tt)+) (: $value:expr) $copy:tt) => { + value_internal!(@object $object [$($key)+] (value_internal!($value))); + }; + + // Missing value for last entry. Trigger a reasonable error message. + (@object $object:ident ($($key:tt)+) (:) $copy:tt) => { + // "unexpected end of macro invocation" + value_internal!(); + }; + + // Missing colon and value for last entry. Trigger a reasonable error + // message. + (@object $object:ident ($($key:tt)+) () $copy:tt) => { + // "unexpected end of macro invocation" + value_internal!(); + }; + + // Misplaced colon. Trigger a reasonable error message. + (@object $object:ident () (: $($rest:tt)*) ($colon:tt $($copy:tt)*)) => { + // Takes no arguments so "no rules expected the token `:`". + json_unexpected!($colon); + }; + + // Found a comma inside a key. Trigger a reasonable error message. + (@object $object:ident ($($key:tt)*) (, $($rest:tt)*) ($comma:tt $($copy:tt)*)) => { + // Takes no arguments so "no rules expected the token `,`". + json_unexpected!($comma); + }; + + // Key is fully parenthesized. This avoids clippy double_parens false + // positives because the parenthesization may be necessary here. + (@object $object:ident () (($key:expr) : $($rest:tt)*) $copy:tt) => { + value_internal!(@object $object ($key) (: $($rest)*) (: $($rest)*)); + }; + + // Refuse to absorb colon token into key expression. + (@object $object:ident ($($key:tt)*) (: $($unexpected:tt)+) $copy:tt) => { + json_expect_expr_comma!($($unexpected)+); + }; + + // Munch a token into the current key. + (@object $object:ident ($($key:tt)*) ($tt:tt $($rest:tt)*) $copy:tt) => { + value_internal!(@object $object ($($key)* $tt) ($($rest)*) ($($rest)*)); + }; + + ////////////////////////////////////////////////////////////////////////// + // The main implementation. + // + // Must be invoked as: value_internal!($($json)+) + ////////////////////////////////////////////////////////////////////////// + + (null) => { + $crate::LoroValue::Null + }; + + (true) => { + $crate::LoroValue::Bool(true) + }; + + (false) => { + $crate::LoroValue::Bool(false) + }; + + ([]) => { + $crate::LoroValue::List(json_internal_vec![].into()) + }; + + ([ $($tt:tt)+ ]) => { + $crate::LoroValue::List(value_internal!(@array [] $($tt)+).into()) + }; + + ({}) => { + $crate::LoroValue::Map(Default::default()) + }; + + ({ $($tt:tt)+ }) => { + ({ + let mut object = $crate::FxHashMap::default(); + value_internal!(@object object () ($($tt)+) ($($tt)+)); + $crate::LoroValue::Map(object.into()) + }) + }; + + // Any Serialize type: numbers, strings, struct literals, variables etc. + // Must be below every other rule. + ($other:expr) => { + $crate::to_value($other) + }; +} + +#[macro_export] +#[doc(hidden)] +macro_rules! json_unexpected { + () => {}; +} + +// The json_internal macro above cannot invoke vec directly because it uses +// local_inner_macros. A vec invocation there would resolve to $crate::vec. +// Instead invoke vec here outside of local_inner_macros. +#[macro_export] +#[doc(hidden)] +macro_rules! json_internal_vec { + ($($content:tt)*) => { + vec![$($content)*] + }; +} + +#[cfg(test)] +mod test { + #[test] + fn test_value_macro() { + let v = loro_value!([1, 2, 3]); + let list = v.into_list().unwrap(); + assert_eq!(&*list, &[1.into(), 2.into(), 3.into()]); + + let map = loro_value!({ + "hi": true, + "false": false, + "null": null, + "list": [], + "integer": 123, + "float": 123.123, + "map": { + "a": "1" + }, + "binary": b"123", + }); + + let map = map.into_map().unwrap(); + assert_eq!(map.len(), 8); + assert!(*map.get("hi").unwrap().as_bool().unwrap()); + assert!(!(*map.get("false").unwrap().as_bool().unwrap())); + assert!(map.get("null").unwrap().is_null()); + assert_eq!(map.get("list").unwrap().as_list().unwrap().len(), 0); + assert_eq!(*map.get("integer").unwrap().as_i64().unwrap(), 123); + assert_eq!(*map.get("float").unwrap().as_double().unwrap(), 123.123); + assert_eq!(map.get("map").unwrap().as_map().unwrap().len(), 1); + assert_eq!( + &**map + .get("map") + .unwrap() + .as_map() + .unwrap() + .get("a") + .unwrap() + .as_string() + .unwrap(), + "1" + ); + assert_eq!( + &**map.get("binary").unwrap().as_binary().unwrap(), + &b"123".to_vec() + ); + } +} diff --git a/3rdparty/loro/crates/loro-common/src/span.rs b/3rdparty/loro/crates/loro-common/src/span.rs new file mode 100644 index 0000000000..200681e12b --- /dev/null +++ b/3rdparty/loro/crates/loro-common/src/span.rs @@ -0,0 +1,586 @@ +use std::fmt::Debug; + +use crate::{Counter, IdLp, IdSpanVector, Lamport, PeerID, ID}; +use rle::{HasLength, Mergable, Slice, Sliceable}; + +/// This struct supports reverse repr: `from` can be less than `to`. +/// We need this because it'll make merging deletions easier. +/// +/// But we should use it behavior conservatively. +/// If it is not necessary to be reverse, it should not. +#[derive(Clone, Copy, PartialEq, Eq)] +pub struct CounterSpan { + // TODO: should be private. user should not be able to change start from smaller than end to be greater than end + pub start: Counter, + // TODO: should be private + pub end: Counter, +} + +pub trait HasLamport { + fn lamport(&self) -> Lamport; +} + +pub trait HasLamportSpan: HasLamport + rle::HasLength { + /// end is the exclusive end, last the inclusive end. + fn lamport_end(&self) -> Lamport { + self.lamport() + self.content_len() as Lamport + } + + /// end is the exclusive end, last the inclusive end. + fn lamport_last(&self) -> Lamport { + self.lamport() + self.content_len() as Lamport - 1 + } +} + +impl Debug for CounterSpan { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(format!("{}~{}", self.start, self.end).as_str()) + } +} + +impl CounterSpan { + #[inline] + pub fn new(from: Counter, to: Counter) -> Self { + CounterSpan { + start: from, + end: to, + } + } + + #[inline] + pub fn reverse(&mut self) { + if self.start == self.end { + return; + } + + if self.start < self.end { + (self.start, self.end) = (self.end - 1, self.start - 1); + } else { + (self.start, self.end) = (self.end + 1, self.start + 1); + } + } + + /// Make end greater than start + pub fn normalize_(&mut self) { + if self.end < self.start { + self.reverse(); + } + } + + #[inline(always)] + pub fn bidirectional(&self) -> bool { + (self.end - self.start).abs() == 1 + } + + #[inline(always)] + pub fn direction(&self) -> i32 { + if self.start < self.end { + 1 + } else { + -1 + } + } + + #[inline(always)] + pub fn is_reversed(&self) -> bool { + self.end < self.start + } + + #[inline] + pub fn min(&self) -> Counter { + if self.start < self.end { + self.start + } else { + self.end + 1 + } + } + + pub fn set_min(&mut self, min: Counter) { + if self.start < self.end { + self.start = min; + } else { + self.end = min - 1; + } + } + + #[inline(always)] + pub fn max(&self) -> Counter { + if self.start > self.end { + self.start + } else { + self.end - 1 + } + } + + #[inline(always)] + /// Normalized end value. + /// + /// This is different from end. start may be greater than end. This is the max of start+1 and end + pub fn norm_end(&self) -> i32 { + if self.start < self.end { + self.end + } else { + self.start + 1 + } + } + + #[inline] + pub fn contains(&self, v: Counter) -> bool { + if self.start < self.end { + self.start <= v && v < self.end + } else { + self.start >= v && v > self.end + } + } + + pub fn set_start(&mut self, new_start: Counter) { + if self.start < self.end { + self.start = new_start.min(self.end); + } else { + self.start = new_start.max(self.end); + } + } + + pub fn set_end(&mut self, new_end: Counter) { + if self.start < self.end { + self.end = new_end.max(self.start); + } else { + self.end = new_end.min(self.start); + } + } + + pub fn extend_include(&mut self, new_start: Counter, new_end: Counter) { + self.set_start(new_start); + self.set_end(new_end); + } + + /// if we can merge element on the left, this method return the last atom of it + fn prev_pos(&self) -> i32 { + if self.start < self.end { + self.start - 1 + } else { + self.start + 1 + } + } + + /// if we can merge element on the right, this method return the first atom of it + fn next_pos(&self) -> i32 { + self.end + } + + fn get_intersection(&self, counter: &CounterSpan) -> Option { + let start = self.start.max(counter.start); + let end = self.end.min(counter.end); + if start < end { + Some(CounterSpan { start, end }) + } else { + None + } + } +} + +impl HasLength for CounterSpan { + #[inline] + fn content_len(&self) -> usize { + if self.start < self.end { + (self.end - self.start) as usize + } else { + (self.start - self.end) as usize + } + } +} + +impl Sliceable for CounterSpan { + fn slice(&self, from: usize, to: usize) -> Self { + assert!(from <= to); + let len = to - from; + assert!(len <= self.content_len()); + if self.start < self.end { + CounterSpan { + start: self.start + from as Counter, + end: self.start + to as Counter, + } + } else { + CounterSpan { + start: self.start - from as Counter, + end: self.start - to as Counter, + } + } + } +} + +impl Mergable for CounterSpan { + #[inline] + fn is_mergable(&self, other: &Self, _: &()) -> bool { + match (self.bidirectional(), other.bidirectional()) { + (true, true) => self.start + 1 == other.start || self.start == other.start + 1, + (true, false) => self.start == other.prev_pos(), + (false, true) => self.next_pos() == other.start, + (false, false) => { + self.next_pos() == other.start && self.direction() == other.direction() + } + } + } + + #[inline] + fn merge(&mut self, other: &Self, _: &()) { + match (self.bidirectional(), other.bidirectional()) { + (true, true) => { + if self.start + 1 == other.start { + self.end = self.start + 2; + } else if self.start - 1 == other.start { + self.end = self.start - 2; + } + } + (true, false) => self.end = other.end, + (false, true) => self.end += self.direction(), + (false, false) => { + self.end = other.end; + } + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct LamportSpan { + pub start: Lamport, + pub end: Lamport, +} + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct IdLpSpan { + pub peer: PeerID, + pub lamport: LamportSpan, +} + +impl HasLength for IdLpSpan { + fn content_len(&self) -> usize { + (self.lamport.end - self.lamport.start) as usize + } +} + +impl IdLpSpan { + pub fn new(peer: PeerID, from: Lamport, to: Lamport) -> Self { + Self { + peer, + lamport: LamportSpan { + start: from, + end: to, + }, + } + } + + pub fn contains(&self, id: IdLp) -> bool { + self.peer == id.peer && self.lamport.start <= id.lamport && id.lamport < self.lamport.end + } +} + +/// This struct supports reverse repr: [CounterSpan]'s from can be less than to. But we should use it conservatively. +/// We need this because it'll make merging deletions easier. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct IdSpan { + pub peer: PeerID, + pub counter: CounterSpan, +} + +impl IdSpan { + #[inline] + pub fn new(peer: PeerID, from: Counter, to: Counter) -> Self { + Self { + peer, + counter: CounterSpan { + start: from, + end: to, + }, + } + } + + #[inline] + pub fn contains(&self, id: ID) -> bool { + self.peer == id.peer && self.counter.contains(id.counter) + } + + #[inline(always)] + pub fn is_reversed(&self) -> bool { + self.counter.end < self.counter.start + } + + #[inline(always)] + pub fn reverse(&mut self) { + self.counter.reverse(); + } + + #[inline(always)] + pub fn normalize_(&mut self) { + self.counter.normalize_(); + } + + /// This is different from id_start. id_start may be greater than id_end, but this is the min of id_start and id_end-1 + #[inline] + pub fn norm_id_start(&self) -> ID { + ID::new(self.peer, self.counter.min()) + } + + /// This is different from id_end. id_start may be greater than id_end. This is the max of id_start+1 and id_end + #[inline] + pub fn norm_id_end(&self) -> ID { + ID::new(self.peer, self.counter.norm_end()) + } + + pub fn to_id_span_vec(self) -> IdSpanVector { + let mut out = IdSpanVector::default(); + out.insert(self.peer, self.counter); + out + } + + pub fn get_intersection(&self, other: &Self) -> Option { + if self.peer != other.peer { + return None; + } + + let counter = self.counter.get_intersection(&other.counter)?; + Some(Self { + peer: self.peer, + counter, + }) + } + + pub fn get_slice_range_on(&self, other: &Self) -> Option<(usize, usize)> { + if self.peer != other.peer { + return None; + } + + let self_start = self.counter.start.min(self.counter.end); + let self_end = self.counter.start.max(self.counter.end); + let other_start = other.counter.start.min(other.counter.end); + let other_end = other.counter.start.max(other.counter.end); + + if self_start >= other_end || self_end <= other_start { + return None; + } + + let start = (self_start.max(other_start) - other_start) as usize; + let end = (self_end.min(other_end) - other_start) as usize; + + Some((start, end)) + } +} + +impl HasLength for IdSpan { + #[inline] + fn content_len(&self) -> usize { + self.counter.content_len() + } +} + +impl Sliceable for IdSpan { + #[inline] + fn slice(&self, from: usize, to: usize) -> Self { + IdSpan { + peer: self.peer, + counter: self.counter.slice(from, to), + } + } +} + +impl Mergable for IdSpan { + fn is_mergable(&self, other: &Self, _: &()) -> bool { + self.peer == other.peer && self.counter.is_mergable(&other.counter, &()) + } + + fn merge(&mut self, other: &Self, _: &()) { + self.counter.merge(&other.counter, &()) + } +} + +pub trait HasId { + fn id_start(&self) -> ID; +} + +pub trait HasCounter { + fn ctr_start(&self) -> Counter; +} + +pub trait HasCounterSpan: HasCounter + HasLength { + /// end is the exclusive end, last the inclusive end. + fn ctr_end(&self) -> Counter { + self.ctr_start() + self.atom_len() as Counter + } + + /// end is the exclusive end, last the inclusive end. + fn ctr_last(&self) -> Counter { + self.ctr_start() + self.atom_len() as Counter - 1 + } + + fn ctr_span(&self) -> CounterSpan { + CounterSpan { + start: self.ctr_start(), + end: self.ctr_end(), + } + } +} + +impl HasCounterSpan for T {} + +pub trait HasIdSpan: HasId + HasLength { + fn intersect(&self, other: &T) -> bool { + let self_start = self.id_start(); + let other_start = self.id_start(); + if self_start.peer != other_start.peer { + false + } else { + let self_start = self_start.counter; + let other_start = other_start.counter; + let self_end = self.id_end().counter; + let other_end = other.id_end().counter; + self_start < other_end && other_start < self_end + } + } + + fn id_span(&self) -> IdSpan { + let id = self.id_start(); + IdSpan::new( + id.peer, + id.counter, + id.counter + self.content_len() as Counter, + ) + } + + /// end is the exclusive end, last the inclusive end. + fn id_end(&self) -> ID { + self.id_start().inc(self.content_len() as i32) + } + + /// end is the exclusive end, last the inclusive end. + fn id_last(&self) -> ID { + self.id_start().inc(self.content_len() as i32 - 1) + } + + fn contains_id(&self, id: ID) -> bool { + let id_start = self.id_start(); + if id.peer != id_start.peer { + return false; + } + + id_start.counter <= id.counter + && id.counter < id_start.counter + self.content_len() as Counter + } +} +impl HasIdSpan for T {} + +impl HasLamportSpan for T {} + +impl HasId for IdSpan { + #[inline] + fn id_start(&self) -> ID { + self.norm_id_start() + } +} + +impl HasCounter for IdSpan { + #[inline] + fn ctr_start(&self) -> Counter { + self.counter.min() + } +} + +impl<'a> From> for IdSpan { + fn from(slice: Slice<'a, IdSpan>) -> Self { + slice.value.slice(slice.start, slice.end) + } +} + +impl HasId for (&PeerID, &CounterSpan) { + fn id_start(&self) -> ID { + ID { + peer: *self.0, + counter: self.1.min(), + } + } +} + +impl HasId for (PeerID, CounterSpan) { + fn id_start(&self) -> ID { + ID { + peer: self.0, + counter: self.1.min(), + } + } +} + +impl From for IdSpan { + fn from(value: ID) -> Self { + Self::new(value.peer, value.counter, value.counter + 1) + } +} + +#[cfg(feature = "wasm")] +mod wasm { + use js_sys::Object; + use wasm_bindgen::JsValue; + + use super::CounterSpan; + + impl From for JsValue { + fn from(value: CounterSpan) -> Self { + let obj = Object::new(); + js_sys::Reflect::set( + &obj, + &JsValue::from_str("start"), + &JsValue::from(value.start), + ) + .unwrap(); + js_sys::Reflect::set(&obj, &JsValue::from_str("end"), &JsValue::from(value.end)) + .unwrap(); + obj.into() + } + } +} + +#[cfg(test)] +mod test_id_span { + use super::*; + + #[test] + fn merge() { + let mut a = CounterSpan::new(0, 2); + let b = CounterSpan::new(2, 1); + assert!(a.is_mergable(&b, &())); + a.merge(&b, &()); + assert_eq!(a, CounterSpan::new(0, 3)); + + let mut a = CounterSpan::new(3, 2); + let b = CounterSpan::new(2, 1); + assert!(a.is_mergable(&b, &())); + a.merge(&b, &()); + assert_eq!(a, CounterSpan::new(3, 1)); + + let mut a = CounterSpan::new(4, 2); + let b = CounterSpan::new(2, 3); + assert!(a.is_mergable(&b, &())); + a.merge(&b, &()); + assert_eq!(a, CounterSpan::new(4, 1)); + + let mut a = CounterSpan::new(8, 9); + let b = CounterSpan::new(9, 8); + assert!(a.is_mergable(&b, &())); + a.merge(&b, &()); + assert_eq!(a, CounterSpan::new(8, 10)); + + let a = CounterSpan::new(8, 9); + let b = CounterSpan::new(10, 11); + assert!(!a.is_mergable(&b, &())); + + let mut a = CounterSpan::new(0, 2); + let b = CounterSpan::new(2, 4); + assert!(a.is_mergable(&b, &())); + a.merge(&b, &()); + assert_eq!(a, CounterSpan::new(0, 4)); + + let mut a = CounterSpan::new(4, 2); + let b = CounterSpan::new(2, 0); + assert!(a.is_mergable(&b, &())); + a.merge(&b, &()); + assert_eq!(a, CounterSpan::new(4, 0)); + } +} diff --git a/3rdparty/loro/crates/loro-common/src/value.rs b/3rdparty/loro/crates/loro-common/src/value.rs new file mode 100644 index 0000000000..39c18c3a7f --- /dev/null +++ b/3rdparty/loro/crates/loro-common/src/value.rs @@ -0,0 +1,1017 @@ +use std::{collections::HashMap, hash::Hash, ops::Index, sync::Arc}; + +use arbitrary::Arbitrary; +use enum_as_inner::EnumAsInner; +use rustc_hash::FxHashMap; +use serde::{de::VariantAccess, Deserialize, Serialize}; + +use crate::ContainerID; + +/// [LoroValue] is used to represents the state of CRDT at a given version. +/// +/// This struct is cheap to clone, the time complexity is O(1). +#[derive(Debug, Clone, EnumAsInner, Default)] +pub enum LoroValue { + #[default] + Null, + Bool(bool), + Double(f64), + I64(i64), + // i64? + Binary(LoroBinaryValue), + String(LoroStringValue), + List(LoroListValue), + // PERF We can use InternalString as key + Map(LoroMapValue), + Container(ContainerID), +} + +impl PartialEq for LoroValue { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (LoroValue::Null, LoroValue::Null) => true, + (LoroValue::Bool(a), LoroValue::Bool(b)) => a == b, + (LoroValue::Double(a), LoroValue::Double(b)) => a == b || (a.is_nan() && b.is_nan()), + (LoroValue::I64(a), LoroValue::I64(b)) => a == b, + (LoroValue::Binary(a), LoroValue::Binary(b)) => a == b, + (LoroValue::String(a), LoroValue::String(b)) => a == b, + (LoroValue::List(a), LoroValue::List(b)) => a == b, + (LoroValue::Map(a), LoroValue::Map(b)) => a == b, + (LoroValue::Container(a), LoroValue::Container(b)) => a == b, + _ => false, + } + } +} + +#[derive(Default, Debug, PartialEq, Clone, Arbitrary)] +pub struct LoroBinaryValue(Arc>); +#[derive(Default, Debug, PartialEq, Clone, Arbitrary)] +pub struct LoroStringValue(Arc); +#[derive(Default, Debug, PartialEq, Clone, Arbitrary)] +pub struct LoroListValue(Arc>); +#[derive(Default, Debug, PartialEq, Clone, Arbitrary)] +pub struct LoroMapValue(Arc>); + +impl From> for LoroBinaryValue { + fn from(value: Vec) -> Self { + LoroBinaryValue(Arc::new(value)) + } +} + +impl From for LoroStringValue { + fn from(value: String) -> Self { + LoroStringValue(Arc::new(value)) + } +} + +impl From<&str> for LoroStringValue { + fn from(value: &str) -> Self { + LoroStringValue(Arc::new(value.to_string())) + } +} + +impl From> for LoroListValue { + fn from(value: Vec) -> Self { + LoroListValue(Arc::new(value)) + } +} + +impl From> for LoroMapValue { + fn from(value: FxHashMap) -> Self { + LoroMapValue(Arc::new(value)) + } +} + +impl From> for LoroMapValue { + fn from(value: HashMap) -> Self { + LoroMapValue(Arc::new(FxHashMap::from_iter(value))) + } +} + +impl From> for LoroMapValue { + fn from(value: Vec<(String, LoroValue)>) -> Self { + LoroMapValue(Arc::new(FxHashMap::from_iter(value))) + } +} + +impl std::ops::Deref for LoroBinaryValue { + type Target = Vec; + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl std::ops::Deref for LoroStringValue { + type Target = String; + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl std::ops::Deref for LoroListValue { + type Target = Vec; + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl std::ops::Deref for LoroMapValue { + type Target = FxHashMap; + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl LoroBinaryValue { + pub fn make_mut(&mut self) -> &mut Vec { + Arc::make_mut(&mut self.0) + } +} + +impl LoroStringValue { + pub fn make_mut(&mut self) -> &mut String { + Arc::make_mut(&mut self.0) + } +} + +impl LoroListValue { + pub fn make_mut(&mut self) -> &mut Vec { + Arc::make_mut(&mut self.0) + } +} + +impl LoroMapValue { + pub fn make_mut(&mut self) -> &mut FxHashMap { + Arc::make_mut(&mut self.0) + } +} + +impl LoroBinaryValue { + pub fn unwrap(self) -> Vec { + match Arc::try_unwrap(self.0) { + Ok(v) => v, + Err(arc) => (*arc).clone(), + } + } +} + +impl LoroStringValue { + pub fn unwrap(self) -> String { + match Arc::try_unwrap(self.0) { + Ok(v) => v, + Err(arc) => (*arc).clone(), + } + } +} + +impl LoroListValue { + pub fn unwrap(self) -> Vec { + match Arc::try_unwrap(self.0) { + Ok(v) => v, + Err(arc) => (*arc).clone(), + } + } +} + +impl LoroMapValue { + pub fn unwrap(self) -> FxHashMap { + match Arc::try_unwrap(self.0) { + Ok(v) => v, + Err(arc) => (*arc).clone(), + } + } +} + +impl FromIterator for LoroListValue { + fn from_iter>(iter: T) -> Self { + LoroListValue(Arc::new(iter.into_iter().collect())) + } +} + +impl FromIterator<(String, LoroValue)> for LoroMapValue { + fn from_iter>(iter: T) -> Self { + let map: FxHashMap<_, _> = iter.into_iter().collect(); + LoroMapValue(Arc::new(map)) + } +} + +impl AsRef for LoroStringValue { + fn as_ref(&self) -> &str { + &self.0 + } +} + +impl AsRef<[LoroValue]> for LoroListValue { + fn as_ref(&self) -> &[LoroValue] { + &self.0 + } +} + +impl AsRef> for LoroMapValue { + fn as_ref(&self) -> &FxHashMap { + &self.0 + } +} + +const MAX_DEPTH: usize = 128; +impl<'a> arbitrary::Arbitrary<'a> for LoroValue { + fn arbitrary(u: &mut arbitrary::Unstructured<'a>) -> arbitrary::Result { + let value = match u.int_in_range(0..=7).unwrap() { + 0 => LoroValue::Null, + 1 => LoroValue::Bool(u.arbitrary()?), + 2 => LoroValue::Double(u.arbitrary()?), + 3 => LoroValue::I64(u.arbitrary()?), + 4 => LoroValue::Binary(LoroBinaryValue::arbitrary(u)?), + 5 => LoroValue::String(LoroStringValue::arbitrary(u)?), + 6 => LoroValue::List(LoroListValue::arbitrary(u)?), + 7 => LoroValue::Map(LoroMapValue::arbitrary(u)?), + _ => unreachable!(), + }; + + if value.get_depth() > MAX_DEPTH { + Err(arbitrary::Error::IncorrectFormat) + } else { + Ok(value) + } + } +} + +impl LoroValue { + pub fn get_by_key(&self, key: &str) -> Option<&LoroValue> { + match self { + LoroValue::Map(map) => map.get(key), + _ => None, + } + } + + pub fn get_by_index(&self, index: isize) -> Option<&LoroValue> { + match self { + LoroValue::List(list) => { + if index < 0 { + let offset = usize::try_from(index.checked_neg()?).ok()?; + list.get(list.len().checked_sub(offset)?) + } else { + list.get(usize::try_from(index).ok()?) + } + } + _ => None, + } + } + + pub fn is_false(&self) -> bool { + match self { + LoroValue::Bool(b) => !*b, + _ => false, + } + } + + pub fn get_depth(&self) -> usize { + let mut max_depth = 0; + let mut value_depth_pairs = vec![(self, 0)]; + while let Some((value, depth)) = value_depth_pairs.pop() { + match value { + LoroValue::List(arr) => { + for v in arr.iter() { + value_depth_pairs.push((v, depth + 1)); + } + max_depth = max_depth.max(depth + 1); + } + LoroValue::Map(map) => { + for (_, v) in map.iter() { + value_depth_pairs.push((v, depth + 1)); + } + + max_depth = max_depth.max(depth + 1); + } + _ => {} + } + } + + max_depth + } + + // TODO: add checks for too deep value, and return err if users + // try to insert such value into a container + pub fn is_too_deep(&self) -> bool { + self.get_depth() > MAX_DEPTH + } + + /// Visit the all list items or map's values + pub fn visit_children(&self, f: &mut dyn FnMut(&LoroValue)) { + match self { + LoroValue::List(list) => { + for v in list.iter() { + f(v); + } + } + LoroValue::Map(m) => { + for (_k, v) in m.iter() { + f(v) + } + } + _ => {} + } + } + + pub fn is_empty_collection(&self) -> bool { + match self { + LoroValue::Null => false, + LoroValue::Bool(_) => false, + LoroValue::Double(_) => false, + LoroValue::I64(_) => false, + LoroValue::Binary(loro_binary_value) => loro_binary_value.is_empty(), + LoroValue::String(loro_string_value) => loro_string_value.is_empty(), + LoroValue::List(loro_list_value) => loro_list_value.is_empty(), + LoroValue::Map(loro_map_value) => loro_map_value.is_empty(), + LoroValue::Container(_) => false, + } + } +} + +impl Index<&str> for LoroValue { + type Output = LoroValue; + + fn index(&self, index: &str) -> &Self::Output { + match self { + LoroValue::Map(map) => map.get(index).unwrap_or(&LoroValue::Null), + _ => &LoroValue::Null, + } + } +} + +impl Index for LoroValue { + type Output = LoroValue; + + fn index(&self, index: usize) -> &Self::Output { + match self { + LoroValue::List(list) => list.get(index).unwrap_or(&LoroValue::Null), + _ => &LoroValue::Null, + } + } +} + +impl TryFrom for bool { + type Error = &'static str; + + fn try_from(value: LoroValue) -> Result { + match value { + LoroValue::Bool(v) => Ok(v), + _ => Err("not a bool"), + } + } +} + +impl TryFrom for f64 { + type Error = &'static str; + + fn try_from(value: LoroValue) -> Result { + match value { + LoroValue::Double(v) => Ok(v), + _ => Err("not a double"), + } + } +} + +impl TryFrom for i32 { + type Error = &'static str; + + fn try_from(value: LoroValue) -> Result { + match value { + LoroValue::I64(v) => i32::try_from(v).map_err(|_| "i64 out of i32 range"), + _ => Err("not a i32"), + } + } +} + +impl TryFrom for Arc> { + type Error = &'static str; + + fn try_from(value: LoroValue) -> Result { + match value { + LoroValue::Binary(v) => Ok(Arc::clone(&v.0)), + _ => Err("not a binary"), + } + } +} + +impl TryFrom for Arc { + type Error = &'static str; + + fn try_from(value: LoroValue) -> Result { + match value { + LoroValue::String(v) => Ok(Arc::clone(&v.0)), + _ => Err("not a string"), + } + } +} + +impl TryFrom for Arc> { + type Error = &'static str; + + fn try_from(value: LoroValue) -> Result { + match value { + LoroValue::List(v) => Ok(Arc::clone(&v.0)), + _ => Err("not a list"), + } + } +} + +impl TryFrom for Arc> { + type Error = &'static str; + + fn try_from(value: LoroValue) -> Result { + match value { + LoroValue::Map(v) => Ok(Arc::clone(&v.0)), + _ => Err("not a map"), + } + } +} + +impl TryFrom for ContainerID { + type Error = &'static str; + + fn try_from(value: LoroValue) -> Result { + match value { + LoroValue::Container(v) => Ok(v), + _ => Err("not a container"), + } + } +} + +impl Hash for LoroValue { + fn hash(&self, state: &mut H) { + std::mem::discriminant(self).hash(state); + match self { + LoroValue::Null => {} + LoroValue::Bool(v) => { + state.write_u8(*v as u8); + } + LoroValue::Double(v) => { + let normalized = if v.is_nan() { + f64::NAN + } else if *v == 0.0 { + 0.0 + } else { + *v + }; + state.write_u64(normalized.to_bits()); + } + LoroValue::I64(v) => { + state.write_i64(*v); + } + LoroValue::Binary(v) => { + v.hash(state); + } + LoroValue::String(v) => { + v.hash(state); + } + LoroValue::List(v) => { + v.hash(state); + } + LoroValue::Map(v) => { + state.write_usize(v.len()); + let mut entries: Vec<_> = v.iter().collect(); + entries.sort_unstable_by_key(|(k, _)| *k); + for (k, v) in entries { + k.hash(state); + v.hash(state); + } + } + LoroValue::Container(v) => { + v.hash(state); + } + } + } +} + +impl Eq for LoroValue {} + +impl, M> From> for LoroValue { + fn from(map: HashMap) -> Self { + let mut new_map = FxHashMap::default(); + for (k, v) in map { + new_map.insert(k.into(), v); + } + + LoroValue::Map(new_map.into()) + } +} + +impl From> for LoroValue { + fn from(vec: Vec) -> Self { + LoroValue::Binary(vec.into()) + } +} + +impl From<&'_ [u8]> for LoroValue { + fn from(vec: &[u8]) -> Self { + LoroValue::Binary(vec.to_vec().into()) + } +} + +impl From<&'_ [u8; N]> for LoroValue { + fn from(vec: &[u8; N]) -> Self { + LoroValue::Binary(vec.to_vec().into()) + } +} + +impl From for LoroValue { + fn from(v: i32) -> Self { + LoroValue::I64(v as i64) + } +} + +impl From for LoroValue { + fn from(v: u32) -> Self { + LoroValue::I64(v as i64) + } +} + +impl From for LoroValue { + fn from(v: i64) -> Self { + LoroValue::I64(v) + } +} + +impl From for LoroValue { + fn from(v: u16) -> Self { + LoroValue::I64(v as i64) + } +} + +impl From for LoroValue { + fn from(v: i16) -> Self { + LoroValue::I64(v as i64) + } +} + +impl From for LoroValue { + fn from(v: f64) -> Self { + LoroValue::Double(v) + } +} + +impl From for LoroValue { + fn from(v: bool) -> Self { + LoroValue::Bool(v) + } +} + +impl> From> for LoroValue { + fn from(value: Vec) -> Self { + let vec: Vec = value.into_iter().map(|x| x.into()).collect(); + LoroValue::List(vec.into()) + } +} + +impl From<&str> for LoroValue { + fn from(v: &str) -> Self { + LoroValue::String(v.to_string().into()) + } +} + +impl From for LoroValue { + fn from(v: String) -> Self { + LoroValue::String(v.into()) + } +} + +impl<'a> From<&'a [LoroValue]> for LoroValue { + fn from(v: &'a [LoroValue]) -> Self { + LoroValue::List(v.to_vec().into()) + } +} + +impl From for LoroValue { + fn from(v: ContainerID) -> Self { + LoroValue::Container(v) + } +} + +#[cfg(feature = "wasm")] +pub mod wasm { + use js_sys::{Array, Object, Uint8Array}; + use rustc_hash::FxHashMap; + use wasm_bindgen::{__rt::IntoJsResult, JsCast, JsValue}; + + use crate::{ContainerID, LoroError, LoroValue}; + + pub fn convert(value: LoroValue) -> JsValue { + match value { + LoroValue::Null => JsValue::NULL, + LoroValue::Bool(b) => JsValue::from_bool(b), + LoroValue::Double(f) => JsValue::from_f64(f), + LoroValue::I64(i) => JsValue::from_f64(i as f64), + LoroValue::String(s) => JsValue::from_str(&s), + LoroValue::Binary(binary) => { + let binary = binary.unwrap(); + let arr = Uint8Array::new_with_length(binary.len() as u32); + for (i, v) in binary.into_iter().enumerate() { + arr.set_index(i as u32, v); + } + arr.into_js_result().unwrap() + } + LoroValue::List(list) => { + let list = list.unwrap(); + let arr = Array::new_with_length(list.len() as u32); + for (i, v) in list.into_iter().enumerate() { + arr.set(i as u32, convert(v)); + } + arr.into_js_result().unwrap() + } + LoroValue::Map(m) => { + let m = m.unwrap(); + let map = Object::new(); + for (k, v) in m.into_iter() { + let str: &str = &k; + js_sys::Reflect::set(&map, &JsValue::from_str(str), &convert(v)).unwrap(); + } + + map.into_js_result().unwrap() + } + LoroValue::Container(container_id) => JsValue::from(&container_id), + } + } + + impl From for JsValue { + fn from(value: LoroValue) -> Self { + convert(value) + } + } + + impl From for LoroValue { + fn from(js_value: JsValue) -> Self { + if js_value.is_null() || js_value.is_undefined() { + LoroValue::Null + } else if js_value.as_bool().is_some() { + LoroValue::Bool(js_value.as_bool().unwrap()) + } else if js_value.as_f64().is_some() { + let num = js_value.as_f64().unwrap(); + if num.fract() == 0.0 && num <= i64::MAX as f64 && num >= i64::MIN as f64 { + LoroValue::I64(num as i64) + } else { + LoroValue::Double(num) + } + } else if js_value.is_string() { + LoroValue::String(js_value.as_string().unwrap().into()) + } else if js_value.has_type::() { + let array = js_value.unchecked_into::(); + let mut list = Vec::new(); + for i in 0..array.length() { + list.push(LoroValue::from(array.get(i))); + } + + LoroValue::List(list.into()) + } else if js_value.is_instance_of::() { + let array = js_value.unchecked_into::(); + let mut binary = Vec::new(); + for i in 0..array.length() { + binary.push(array.get_index(i)); + } + + LoroValue::Binary(binary.into()) + } else if js_value.is_object() { + let object = js_value.unchecked_into::(); + let mut map = FxHashMap::default(); + for key in js_sys::Reflect::own_keys(&object).unwrap().iter() { + let key = key.as_string().unwrap(); + map.insert( + key.clone(), + LoroValue::from(js_sys::Reflect::get(&object, &key.into()).unwrap()), + ); + } + + LoroValue::Map(map.into()) + } else { + panic!("Fail to convert JsValue {js_value:?} to LoroValue ") + } + } + } + + impl From<&ContainerID> for JsValue { + fn from(id: &ContainerID) -> Self { + JsValue::from_str(id.to_string().as_str()) + } + } + + impl TryFrom for ContainerID { + type Error = LoroError; + + fn try_from(value: JsValue) -> Result { + if !value.is_string() { + return Err(LoroError::DecodeError( + "Given ContainerId is not string".into(), + )); + } + + let s = value.as_string().unwrap(); + ContainerID::try_from(s.as_str()).map_err(|_| { + LoroError::DecodeError( + format!("Given ContainerId is not a valid ContainerID: {s}").into(), + ) + }) + } + } +} + +const LORO_CONTAINER_ID_PREFIX: &str = "🦜:"; + +impl Serialize for LoroValue { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + if serializer.is_human_readable() { + // json type + match self { + LoroValue::Null => serializer.serialize_unit(), + LoroValue::Bool(b) => serializer.serialize_bool(*b), + LoroValue::Double(d) => serializer.serialize_f64(*d), + LoroValue::I64(i) => serializer.serialize_i64(*i), + LoroValue::String(s) => serializer.serialize_str(s), + LoroValue::Binary(b) => serializer.collect_seq(b.iter()), + LoroValue::List(l) => serializer.collect_seq(l.iter()), + LoroValue::Map(m) => serializer.collect_map(m.iter()), + LoroValue::Container(id) => { + serializer.serialize_str(&format!("{LORO_CONTAINER_ID_PREFIX}{id}")) + } + } + } else { + // binary type + match self { + LoroValue::Null => serializer.serialize_unit_variant("LoroValue", 0, "Null"), + LoroValue::Bool(b) => { + serializer.serialize_newtype_variant("LoroValue", 1, "Bool", b) + } + LoroValue::Double(d) => { + serializer.serialize_newtype_variant("LoroValue", 2, "Double", d) + } + LoroValue::I64(i) => serializer.serialize_newtype_variant("LoroValue", 3, "I32", i), + LoroValue::String(s) => { + serializer.serialize_newtype_variant("LoroValue", 4, "String", &**s) + } + + LoroValue::List(l) => { + serializer.serialize_newtype_variant("LoroValue", 5, "List", &**l) + } + LoroValue::Map(m) => { + serializer.serialize_newtype_variant("LoroValue", 6, "Map", &**m) + } + LoroValue::Container(id) => { + serializer.serialize_newtype_variant("LoroValue", 7, "Container", id) + } + LoroValue::Binary(b) => { + serializer.serialize_newtype_variant("LoroValue", 8, "Binary", &**b) + } + } + } + } +} + +impl<'de> Deserialize<'de> for LoroValue { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + if deserializer.is_human_readable() { + deserializer.deserialize_any(LoroValueVisitor) + } else { + deserializer.deserialize_enum( + "LoroValue", + &[ + "Null", + "Bool", + "Double", + "I32", + "String", + "List", + "Map", + "Container", + "Binary", + ], + LoroValueEnumVisitor, + ) + } + } +} + +struct LoroValueVisitor; + +impl<'de> serde::de::Visitor<'de> for LoroValueVisitor { + type Value = LoroValue; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("a LoroValue") + } + + fn visit_unit(self) -> Result + where + E: serde::de::Error, + { + Ok(LoroValue::Null) + } + + fn visit_bool(self, v: bool) -> Result + where + E: serde::de::Error, + { + Ok(LoroValue::Bool(v)) + } + + fn visit_i64(self, v: i64) -> Result + where + E: serde::de::Error, + { + Ok(LoroValue::I64(v)) + } + + fn visit_u64(self, v: u64) -> Result + where + E: serde::de::Error, + { + Ok(i64::try_from(v) + .map(LoroValue::I64) + .unwrap_or_else(|_| LoroValue::Double(v as f64))) + } + + fn visit_f64(self, v: f64) -> Result + where + E: serde::de::Error, + { + Ok(LoroValue::Double(v)) + } + + fn visit_str(self, v: &str) -> Result + where + E: serde::de::Error, + { + if let Some(id) = v.strip_prefix(LORO_CONTAINER_ID_PREFIX) { + return Ok(LoroValue::Container( + ContainerID::try_from(id) + .map_err(|_| serde::de::Error::custom("Invalid container id"))?, + )); + } + Ok(LoroValue::String(v.to_owned().into())) + } + + fn visit_string(self, v: String) -> Result + where + E: serde::de::Error, + { + if let Some(id) = v.strip_prefix(LORO_CONTAINER_ID_PREFIX) { + return Ok(LoroValue::Container( + ContainerID::try_from(id) + .map_err(|_| serde::de::Error::custom("Invalid container id"))?, + )); + } + + Ok(LoroValue::String(v.into())) + } + + fn visit_borrowed_bytes(self, v: &'de [u8]) -> Result + where + E: serde::de::Error, + { + let binary = Vec::from_iter(v.iter().copied()); + Ok(LoroValue::Binary(binary.into())) + } + + fn visit_bytes(self, v: &[u8]) -> Result + where + E: serde::de::Error, + { + let binary = Vec::from_iter(v.iter().copied()); + Ok(LoroValue::Binary(binary.into())) + } + + fn visit_seq(self, mut seq: A) -> Result + where + A: serde::de::SeqAccess<'de>, + { + let mut list = Vec::new(); + while let Some(value) = seq.next_element()? { + list.push(value); + } + Ok(LoroValue::List(list.into())) + } + + fn visit_map(self, mut map: A) -> Result + where + A: serde::de::MapAccess<'de>, + { + let mut ans: FxHashMap = FxHashMap::default(); + while let Some((key, value)) = map.next_entry::()? { + ans.insert(key, value); + } + + Ok(LoroValue::Map(ans.into())) + } +} + +#[derive(Deserialize)] +enum LoroValueFields { + Null, + Bool, + Double, + I32, + String, + List, + Map, + Container, + Binary, +} + +struct LoroValueEnumVisitor; +impl<'de> serde::de::Visitor<'de> for LoroValueEnumVisitor { + type Value = LoroValue; + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("a loro value") + } + + fn visit_enum(self, data: A) -> Result + where + A: serde::de::EnumAccess<'de>, + { + match data.variant()? { + (LoroValueFields::Null, v) => { + v.unit_variant()?; + Ok(LoroValue::Null) + } + (LoroValueFields::Bool, v) => v.newtype_variant().map(LoroValue::Bool), + (LoroValueFields::Double, v) => v.newtype_variant().map(LoroValue::Double), + (LoroValueFields::I32, v) => v.newtype_variant().map(LoroValue::I64), + (LoroValueFields::String, v) => v + .newtype_variant() + .map(|x: String| LoroValue::String(x.into())), + (LoroValueFields::List, v) => v + .newtype_variant() + .map(|x: Vec| LoroValue::List(x.into())), + (LoroValueFields::Map, v) => v + .newtype_variant() + .map(|x: FxHashMap| LoroValue::Map(x.into())), + (LoroValueFields::Container, v) => v.newtype_variant().map(LoroValue::Container), + (LoroValueFields::Binary, v) => v + .newtype_variant() + .map(|x: Vec| LoroValue::Binary(x.into())), + } + } +} + +pub fn to_value>(value: T) -> LoroValue { + value.into() +} + +#[cfg(feature = "serde_json")] +mod serde_json_impl { + use serde_json::{Number, Value}; + + use super::LoroValue; + + impl From for LoroValue { + fn from(value: Value) -> Self { + match value { + Value::Null => LoroValue::Null, + Value::Bool(b) => LoroValue::Bool(b), + Value::Number(n) => { + if let Some(i) = n.as_i64() { + LoroValue::I64(i) + } else { + LoroValue::Double(n.as_f64().unwrap()) + } + } + Value::String(s) => LoroValue::String(s.into()), + Value::Array(arr) => { + LoroValue::List(arr.into_iter().map(LoroValue::from).collect()) + } + Value::Object(obj) => LoroValue::Map( + obj.into_iter() + .map(|(k, v)| (k, LoroValue::from(v))) + .collect(), + ), + } + } + } + + use super::LORO_CONTAINER_ID_PREFIX; + impl From for Value { + fn from(value: LoroValue) -> Self { + match value { + LoroValue::Null => Value::Null, + LoroValue::Bool(b) => Value::Bool(b), + LoroValue::Double(d) => Number::from_f64(d).map_or(Value::Null, Value::Number), + LoroValue::I64(i) => Value::Number(Number::from(i)), + LoroValue::String(s) => Value::String(s.to_string()), + LoroValue::List(l) => Value::Array(l.iter().cloned().map(Value::from).collect()), + LoroValue::Map(m) => Value::Object( + m.iter() + .map(|(k, v)| (k.clone(), Value::from(v.clone()))) + .collect(), + ), + LoroValue::Container(id) => { + Value::String(format!("{LORO_CONTAINER_ID_PREFIX}{id}")) + } + LoroValue::Binary(b) => Value::Array(b.iter().copied().map(Value::from).collect()), + } + } + } +} diff --git a/3rdparty/loro/crates/loro-internal/.gitignore b/3rdparty/loro/crates/loro-internal/.gitignore new file mode 100644 index 0000000000..1b5d043584 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/.gitignore @@ -0,0 +1 @@ +*dhat-heap.json diff --git a/3rdparty/loro/crates/loro-internal/Cargo.toml b/3rdparty/loro/crates/loro-internal/Cargo.toml new file mode 100644 index 0000000000..0b755ca5f1 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/Cargo.toml @@ -0,0 +1,128 @@ +[package] +name = "loro-internal" +version = "1.13.7" +edition = "2021" +license = "MIT" +description = "Loro internal library. Do not use it directly as it's not stable." +documentation = "https://docs.rs/loro/" +homepage = "https://loro.dev" +repository = "https://github.com/loro-dev/loro/" +authors = ["Zixuan Chen", "Liang Zhao"] +categories = [] +keywords = ["crdt", "local-first"] + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +generic-btree = { version = "^0.10.7" } +smallvec = { workspace = true } +loro-delta = { path = "../delta", version = "1.13.0", package = "loro-delta" } +rle = { path = "../rle", version = "1.6.0", package = "loro-rle" } +loro-common = { path = "../loro-common", version = "1.13.1" } +fractional_index = { path = "../fractional_index", features = [ + "serde", +], version = "1.13.0", package = "loro_fractional_index" } +loro-kv-store = { path = "../kv-store", version = "1.13.7" } +rustc-hash = { workspace = true } +serde = { workspace = true } +thiserror = { workspace = true } +enum-as-inner = { workspace = true } +serde_json = { workspace = true } +bytes = { workspace = true } +xxhash-rust = { workspace = true } +serde_columnar = { workspace = true } +itertools = { workspace = true } +enum_dispatch = { workspace = true } +once_cell = { workspace = true } +arbitrary = { version = "1", optional = true } +postcard = { version = "1", default-features = false, features = ["use-std"] } +append-only-bytes = { version = "0.1.12", features = ["u32_range"] } +im = { version = "15.1.0", features = ["serde"] } +tabled = { version = "0.10.0", optional = true } +wasm-bindgen = { version = "=0.2.100", optional = true } +js-sys = { version = "0.3.60", optional = true } +num = "0.4.0" +rand = { version = "0.8.5" } +getrandom = "0.2.15" +leb128 = "0.2.5" +num-traits = "0.2" +either = "1" +md5 = "0.7.0" +arref = "0.1.0" +tracing = { version = "0.1" } +nonmax = "0.5.5" +ensure-cov = { workspace = true } +pretty_assertions = "1.4.1" +thread_local = "1.1.8" +parking_lot = "0.12.5" +pest = "2.8.3" +pest_derive = "2.8.3" + +[target.'cfg(loom)'.dependencies] +loom = { version = "0.7", features = ["checkpoint"] } + +[lints.rust] +unexpected_cfgs = { level = "warn", check-cfg = ['cfg(loom)'] } + +[target.'cfg(all(target_arch = "wasm32", not(target_os = "emscripten"), not(target_os = "wasi")))'.dependencies.getrandom] +version = "0.2.15" +features = ["js"] + +[target.'cfg(all(target_arch = "wasm32", not(target_os = "emscripten"), not(target_os = "wasi")))'.dependencies] +wasm-bindgen = "0.2.100" + +[dev-dependencies] +arbitrary = { version = "1" } +miniz_oxide = "0.7.1" +rand = { version = "0.8.5" } +tabled = "0.10.0" +ctor = "0.1.23" +zstd = "0.13.0" +criterion = { version = "0.7.0", features = ["html_reports"] } + +# See https://matklad.github.io/2021/02/27/delete-cargo-integration-tests.html +[lib] +doctest = false +bench = false + +[features] +# "wasm" will use utf-16 index by default when using text related stuff; But it'll use utf-8 encoding internally. +wasm = ["wasm-bindgen", "js-sys", "loro-common/wasm"] +# whether to use list slice instead of raw str in text container +test_utils = ["arbitrary", "tabled"] +# whether enable the counter container +counter = ["loro-common/counter"] +logging = ["loro-common/logging"] +jsonpath = [] + +[[bench]] +name = "text_r" +harness = false + +[[bench]] +name = "list" +harness = false + +[[bench]] +name = "encode" +harness = false + +[[bench]] +name = "pending" +harness = false + +[[bench]] +name = "event" +harness = false + +[[bench]] +name = "map" +harness = false + +[[bench]] +name = "tree" +harness = false + +[[bench]] +name = "jsonpath" +harness = false diff --git a/3rdparty/loro/crates/loro-internal/Encoding.md b/3rdparty/loro/crates/loro-internal/Encoding.md new file mode 100644 index 0000000000..2815951a68 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/Encoding.md @@ -0,0 +1,25 @@ +# Encoding Schema + +## Header + +The header has 22 bytes. + +- (0-4 bytes) Magic Bytes: The encoding starts with `loro` as magic bytes. +- (4-20 bytes) Checksum: MD5/xxhash checksum of the encoded data, including the header starting from 20th bytes. The checksum is encoded as a 16-byte array (for xxhash, the first 12 bytes are padded with 0). The `checksum` and `magic bytes` fields are trimmed when calculating the checksum. +- (20-21 bytes) Encoding Method (2 bytes, big endian): Multiple encoding methods are available for a specific encoding version. + +## Encode Mode: Updates + +In this approach, only ops, specifically their historical record, are encoded, while document states are excluded. + +Like Automerge's format, we employ columnar encoding for operations and changes. + +Previously, operations were ordered by their Operation ID (OpId) before columnar encoding. However, sorting operations based on their respective containers initially enhance compression potential. + +## Encode Mode: Snapshot + +This mode simultaneously captures document state and historical data. Upon importing a snapshot into a new document, initialization occurs directly from the snapshot, bypassing the need for CRDT-based recalculations. + +Unlike previous snapshot encoding methods, the current binary output in snapshot mode is compatible with the updates mode. This enhances the efficiency of importing snapshots into non-empty documents, where initialization via snapshot is infeasible. + +Additionally, when feasible, we leverage the sequence of operations to construct state snapshots. In CRDTs, deducing the specific ops constituting the current container state is feasible. These ops are tagged in relation to the container, facilitating direct state reconstruction from them. This approach, pioneered by Automerge, significantly improves compression efficiency. diff --git a/3rdparty/loro/crates/loro-internal/README.md b/3rdparty/loro/crates/loro-internal/README.md new file mode 100644 index 0000000000..d66d799e5f --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/README.md @@ -0,0 +1,5 @@ +# loro-internal + +This is the internal implementation of Loro. +Its interface subject to change. You should not use it as dependency directly. +Consider use `loro` crate instead. diff --git a/3rdparty/loro/crates/loro-internal/VERSION b/3rdparty/loro/crates/loro-internal/VERSION new file mode 100644 index 0000000000..348e808d1e --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/VERSION @@ -0,0 +1 @@ +1.13.7 \ No newline at end of file diff --git a/3rdparty/loro/crates/loro-internal/deno.json b/3rdparty/loro/crates/loro-internal/deno.json new file mode 100644 index 0000000000..6edf0b37df --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/deno.json @@ -0,0 +1,10 @@ +{ + "tasks": { + "test": "cargo nextest run --features test_utils", + "deny": "cargo deny check", + "fuzz": "cargo +nightly fuzz run", + "mem": "deno run -A ./scripts/run_mem.ts", + "flame": "CARGO_PROFILE_RELEASE_DEBUG=true cargo flamegraph --example encoding_refactored --root", + "bench": "cargo bench --features test_utils" + } +} diff --git a/3rdparty/loro/crates/loro-internal/deno.lock b/3rdparty/loro/crates/loro-internal/deno.lock new file mode 100644 index 0000000000..5a95611b38 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/deno.lock @@ -0,0 +1,134 @@ +{ + "version": "3", + "redirects": { + "https://deno.land/std/fmt/printf.ts": "https://deno.land/std@0.105.0/fmt/printf.ts", + "https://deno.land/std/path/mod.ts": "https://deno.land/std@0.105.0/path/mod.ts", + "https://deno.land/std/testing/asserts.ts": "https://deno.land/std@0.105.0/testing/asserts.ts", + "https://x.nest.land/std@0.73.0/path/mod.ts": "https://lra6z45nakk5lnu3yjchp7tftsdnwwikwr65ocha5eojfnlgu4sa.arweave.net/XEHs860CldW2m8JEd_5lnIbbWQq0fdcI4OkckrVmpyQ/path/mod.ts" + }, + "remote": { + "https://deno.land/std@0.105.0/_util/assert.ts": "2f868145a042a11d5ad0a3c748dcf580add8a0dbc0e876eaa0026303a5488f58", + "https://deno.land/std@0.105.0/_util/os.ts": "dfb186cc4e968c770ab6cc3288bd65f4871be03b93beecae57d657232ecffcac", + "https://deno.land/std@0.105.0/fmt/colors.ts": "d2f8355f00a74404668fc5a1e4a92983ce1a9b0a6ac1d40efbd681cb8f519586", + "https://deno.land/std@0.105.0/fmt/printf.ts": "7ec612e9b89958b8f7710129f74f502327aad285a9e48ee5297f5882fbc3a078", + "https://deno.land/std@0.105.0/path/_constants.ts": "1247fee4a79b70c89f23499691ef169b41b6ccf01887a0abd131009c5581b853", + "https://deno.land/std@0.105.0/path/_interface.ts": "1fa73b02aaa24867e481a48492b44f2598cd9dfa513c7b34001437007d3642e4", + "https://deno.land/std@0.105.0/path/_util.ts": "2e06a3b9e79beaf62687196bd4b60a4c391d862cfa007a20fc3a39f778ba073b", + "https://deno.land/std@0.105.0/path/common.ts": "eaf03d08b569e8a87e674e4e265e099f237472b6fd135b3cbeae5827035ea14a", + "https://deno.land/std@0.105.0/path/glob.ts": "3b84af55c53febacf6afe214c095624b22a56b6f57d7312157479cc783a0de65", + "https://deno.land/std@0.105.0/path/mod.ts": "4465dc494f271b02569edbb4a18d727063b5dbd6ed84283ff906260970a15d12", + "https://deno.land/std@0.105.0/path/posix.ts": "b81974c768d298f8dcd2c720229639b3803ca4a241fa9a355c762fa2bc5ef0c1", + "https://deno.land/std@0.105.0/path/separator.ts": "8fdcf289b1b76fd726a508f57d3370ca029ae6976fcde5044007f062e643ff1c", + "https://deno.land/std@0.105.0/path/win32.ts": "f4a3d4a3f2c9fe894da046d5eac48b5e789a0ebec5152b2c0985efe96a9f7ae1", + "https://deno.land/std@0.105.0/testing/_diff.ts": "5d3693155f561d1a5443ac751ac70aab9f5d67b4819a621d4b96b8a1a1c89620", + "https://deno.land/std@0.105.0/testing/asserts.ts": "e4311d45d956459d4423bc267208fe154b5294989da2ed93257b6a85cae0427e", + "https://deno.land/std@0.106.0/_util/assert.ts": "2f868145a042a11d5ad0a3c748dcf580add8a0dbc0e876eaa0026303a5488f58", + "https://deno.land/std@0.106.0/_util/os.ts": "dfb186cc4e968c770ab6cc3288bd65f4871be03b93beecae57d657232ecffcac", + "https://deno.land/std@0.106.0/fs/_util.ts": "f2ce811350236ea8c28450ed822a5f42a0892316515b1cd61321dec13569c56b", + "https://deno.land/std@0.106.0/fs/copy.ts": "631bbafbfe6cba282158abc8aeb7e8251cc69a7ec28ce12878ea1b75fec2add4", + "https://deno.land/std@0.106.0/fs/empty_dir.ts": "5f08b263dd064dc7917c4bbeb13de0f5505a664b9cdfe312fa86e7518cfaeb84", + "https://deno.land/std@0.106.0/fs/ensure_dir.ts": "b7c103dc41a3d1dbbb522bf183c519c37065fdc234831a4a0f7d671b1ed5fea7", + "https://deno.land/std@0.106.0/fs/ensure_file.ts": "c06031af24368e80c330897e4b8e9109efc8602ffabc8f3e2306be07529e1d13", + "https://deno.land/std@0.106.0/fs/ensure_link.ts": "26e54363508b822afd87a3f6e873bbbcd6b5993dd638f8170758c16262a75065", + "https://deno.land/std@0.106.0/fs/ensure_symlink.ts": "c07b6d19ef58b6f5c671ffa942e7f9be50315f4f78e2f9f511626fd2e13beccc", + "https://deno.land/std@0.106.0/fs/eol.ts": "afaebaaac36f48c423b920c836551997715672b80a0fee9aa7667c181a94f2df", + "https://deno.land/std@0.106.0/fs/exists.ts": "b0d2e31654819cc2a8d37df45d6b14686c0cc1d802e9ff09e902a63e98b85a00", + "https://deno.land/std@0.106.0/fs/expand_glob.ts": "73e7b13f01097b04ed782b3d63863379b718417417758ba622e282b1e5300b91", + "https://deno.land/std@0.106.0/fs/mod.ts": "26eee4b52a8c516e37d464094b080ff6822883e7f01ff0ba0a72b8dcd54b9927", + "https://deno.land/std@0.106.0/fs/move.ts": "4623058e39bbbeb3ad30aeff9c974c55d2d574ad7c480295c12b04c244686a99", + "https://deno.land/std@0.106.0/fs/walk.ts": "b91c655c60d048035f9cae0e6177991ab3245e786e3ab7d20a5b60012edf2126", + "https://deno.land/std@0.106.0/path/_constants.ts": "1247fee4a79b70c89f23499691ef169b41b6ccf01887a0abd131009c5581b853", + "https://deno.land/std@0.106.0/path/_interface.ts": "1fa73b02aaa24867e481a48492b44f2598cd9dfa513c7b34001437007d3642e4", + "https://deno.land/std@0.106.0/path/_util.ts": "2e06a3b9e79beaf62687196bd4b60a4c391d862cfa007a20fc3a39f778ba073b", + "https://deno.land/std@0.106.0/path/common.ts": "eaf03d08b569e8a87e674e4e265e099f237472b6fd135b3cbeae5827035ea14a", + "https://deno.land/std@0.106.0/path/glob.ts": "3b84af55c53febacf6afe214c095624b22a56b6f57d7312157479cc783a0de65", + "https://deno.land/std@0.106.0/path/mod.ts": "4465dc494f271b02569edbb4a18d727063b5dbd6ed84283ff906260970a15d12", + "https://deno.land/std@0.106.0/path/posix.ts": "b81974c768d298f8dcd2c720229639b3803ca4a241fa9a355c762fa2bc5ef0c1", + "https://deno.land/std@0.106.0/path/separator.ts": "8fdcf289b1b76fd726a508f57d3370ca029ae6976fcde5044007f062e643ff1c", + "https://deno.land/std@0.106.0/path/win32.ts": "f4a3d4a3f2c9fe894da046d5eac48b5e789a0ebec5152b2c0985efe96a9f7ae1", + "https://deno.land/std@0.198.0/_util/os.ts": "d932f56d41e4f6a6093d56044e29ce637f8dcc43c5a90af43504a889cf1775e3", + "https://deno.land/std@0.198.0/assert/assert.ts": "9a97dad6d98c238938e7540736b826440ad8c1c1e54430ca4c4e623e585607ee", + "https://deno.land/std@0.198.0/assert/assertion_error.ts": "4d0bde9b374dfbcbe8ac23f54f567b77024fb67dbb1906a852d67fe050d42f56", + "https://deno.land/std@0.198.0/path/_basename.ts": "057d420c9049821f983f784fd87fa73ac471901fb628920b67972b0f44319343", + "https://deno.land/std@0.198.0/path/_constants.ts": "e49961f6f4f48039c0dfed3c3f93e963ca3d92791c9d478ac5b43183413136e0", + "https://deno.land/std@0.198.0/path/_dirname.ts": "355e297236b2218600aee7a5301b937204c62e12da9db4b0b044993d9e658395", + "https://deno.land/std@0.198.0/path/_extname.ts": "eaaa5aae1acf1f03254d681bd6a8ce42a9cb5b7ff2213a9d4740e8ab31283664", + "https://deno.land/std@0.198.0/path/_format.ts": "4a99270d6810f082e614309164fad75d6f1a483b68eed97c830a506cc589f8b4", + "https://deno.land/std@0.198.0/path/_from_file_url.ts": "7e4e5626089785adddb061f1b9f4932d6b21c7df778e7449531a11e32048245c", + "https://deno.land/std@0.198.0/path/_interface.ts": "6471159dfbbc357e03882c2266d21ef9afdb1e4aa771b0545e90db58a0ba314b", + "https://deno.land/std@0.198.0/path/_is_absolute.ts": "05dac10b5e93c63198b92e3687baa2be178df5321c527dc555266c0f4f51558c", + "https://deno.land/std@0.198.0/path/_join.ts": "fd78555bc34d5f188918fc7018dfe8fe2df5bbad94a3b30a433666c03934d77f", + "https://deno.land/std@0.198.0/path/_normalize.ts": "a19ec8706b2707f9dd974662a5cd89fad438e62ab1857e08b314a8eb49a34d81", + "https://deno.land/std@0.198.0/path/_parse.ts": "0f9b0ff43682dd9964eb1c4398610c4e165d8db9d3ac9d594220217adf480cfa", + "https://deno.land/std@0.198.0/path/_relative.ts": "27bdeffb5311a47d85be26d37ad1969979359f7636c5cd9fcf05dcd0d5099dc5", + "https://deno.land/std@0.198.0/path/_resolve.ts": "3bf0287d62488cad08c3c219a9708c4a4c658c65d7b4400fd99afdc3ba10a64d", + "https://deno.land/std@0.198.0/path/_to_file_url.ts": "739bfda583598790b2e77ce227f2bb618f6ebdb939788cea47555b43970ec58c", + "https://deno.land/std@0.198.0/path/_to_namespaced_path.ts": "0d5f4caa2ed98ef7a8786286df6af804b50e38859ae897b5b5b4c8c5930a75c8", + "https://deno.land/std@0.198.0/path/_util.ts": "4e191b1bac6b3bf0c31aab42e5ca2e01a86ab5a0d2e08b75acf8585047a86221", + "https://deno.land/std@0.198.0/path/basename.ts": "6f08fbb90dbfcf320765b3abb01f995b1723f75e2534acfd5380e202c802a3aa", + "https://deno.land/std@0.198.0/path/common.ts": "ee7505ab01fd22de3963b64e46cff31f40de34f9f8de1fff6a1bd2fe79380000", + "https://deno.land/std@0.198.0/path/dirname.ts": "098996822a31b4c46e1eb52a19540d3c6f9f54b772fc8a197939eeabc29fca2f", + "https://deno.land/std@0.198.0/path/extname.ts": "9b83c62fd16505739541f7a3ab447d8972da39dbf668d47af2f93206c2480893", + "https://deno.land/std@0.198.0/path/format.ts": "cb22f95cc7853d590b87708cc9441785e760d711188facff3d225305a8213aca", + "https://deno.land/std@0.198.0/path/from_file_url.ts": "a6221cfc928928ec4d9786d767dfac98fa2ab746af0786446c9834a07b98817e", + "https://deno.land/std@0.198.0/path/glob.ts": "d479e0a695621c94d3fd7fe7abd4f9499caf32a8de13f25073451c6ef420a4e1", + "https://deno.land/std@0.198.0/path/is_absolute.ts": "6b3d36352eb7fa29edb53f9e7b09b1aeb022a3c5465764f6cc5b8c41f9736197", + "https://deno.land/std@0.198.0/path/join.ts": "4a2867ff2f3c81ffc9eb3d56dade16db6f8bd3854f269306d23dad4115089c84", + "https://deno.land/std@0.198.0/path/mod.ts": "7765507696cb321994cdacfc19ee3ba61e8e3ebf4bd98fa75a276cf5dc18ce2a", + "https://deno.land/std@0.198.0/path/normalize.ts": "7d992cd262b2deefa842d93a8ba2ed51f3949ba595b1d07f627ac2cddbc74808", + "https://deno.land/std@0.198.0/path/parse.ts": "031fe488b3497fb8312fc1dc3c3d6c2d80707edd9c661e18ee9fd20f95edf322", + "https://deno.land/std@0.198.0/path/posix.ts": "0a1c1952d132323a88736d03e92bd236f3ed5f9f079e5823fae07c8d978ee61b", + "https://deno.land/std@0.198.0/path/relative.ts": "7db80c5035016174267da16321a742d76e875215c317859a383b12f413c6f5d6", + "https://deno.land/std@0.198.0/path/resolve.ts": "103b62207726a27f28177f397008545804ecb20aaf00623af1f622b18cd80b9f", + "https://deno.land/std@0.198.0/path/separator.ts": "0fb679739d0d1d7bf45b68dacfb4ec7563597a902edbaf3c59b50d5bcadd93b1", + "https://deno.land/std@0.198.0/path/to_file_url.ts": "dd32f7a01bbf3b15b5df46796659984b372973d9b2d7d59bcf0eb990763a0cb5", + "https://deno.land/std@0.198.0/path/to_namespaced_path.ts": "4e643ab729bf49ccdc166ad48d2de262ff462938fcf2a44a4425588f4a0bd690", + "https://deno.land/std@0.198.0/path/win32.ts": "8b3f80ef7a462511d5e8020ff490edcaa0a0d118f1b1e9da50e2916bdd73f9dd", + "https://deno.land/x/cliui@v7.0.4-deno/build/lib/index.d.ts": "4f04923352ce24027ad6fe25c85249e0b0cc5c29fe18d62024d004c59d9e41ee", + "https://deno.land/x/cliui@v7.0.4-deno/build/lib/index.js": "fb6030c7b12602a4fca4d81de3ddafa301ba84fd9df73c53de6f3bdda7b482d5", + "https://deno.land/x/cliui@v7.0.4-deno/build/lib/string-utils.js": "b3eb9d2e054a43a3064af17332fb1839a7dadb205c5371af4789616afb1a117f", + "https://deno.land/x/cliui@v7.0.4-deno/deno.ts": "d07bc3338661f8011e3a5fd215061d17a52107a5383c29f40ce0c1ecb8bb8cc3", + "https://deno.land/x/dirname@1.1.2/types.ts": "c1ed1667545bc4b1d69bdb2fc26a5fa8edae3a56e3081209c16a408a322a2319", + "https://deno.land/x/escalade@v3.0.3/sync.ts": "493bc66563292c5c10c4a75a467a5933f24dad67d74b0f5a87e7b988fe97c104", + "https://deno.land/x/y18n@v5.0.0-deno/build/lib/index.d.ts": "11f40d97041eb271cc1a1c7b296c6e7a068d4843759575e7416f0d14ebf8239c", + "https://deno.land/x/y18n@v5.0.0-deno/build/lib/index.js": "92c4624714aa508d33c6d21c0b0ffa072369a8b306e5f8c7727662f570bbd026", + "https://deno.land/x/y18n@v5.0.0-deno/deno.ts": "80997f0709a0b43d29931e2b33946f2bbc32b13fd82f80a5409628455427e28d", + "https://deno.land/x/y18n@v5.0.0-deno/lib/platform-shims/deno.ts": "8fa2c96ac03734966260cfd2c5bc240e41725c913e5b64a0297aede09f52b39d", + "https://deno.land/x/yargs@v17.1.1-deno/build/lib/argsert.js": "eb085555452eac3ff300935994a42f35d16e04cf698cb775cb5ad4f5653c0627", + "https://deno.land/x/yargs@v17.1.1-deno/build/lib/command.js": "499c95cecd5e93f627e0b5ce66a193c9a595adc10fbafe0581a9725e38324dee", + "https://deno.land/x/yargs@v17.1.1-deno/build/lib/completion-templates.js": "f84823b1daa0ed0189e4f823f6a4fd29ad58de6a05771004918368fd62bb2b3f", + "https://deno.land/x/yargs@v17.1.1-deno/build/lib/completion.js": "c91772b89907ebf1a462804305d12d3b9deade75cd1b319c06831ac0bf5abd27", + "https://deno.land/x/yargs@v17.1.1-deno/build/lib/middleware.js": "cef3f017d5ff61c340c65b8422f5ab9600ba381aa656df634d1a3edf0f967527", + "https://deno.land/x/yargs@v17.1.1-deno/build/lib/parse-command.js": "327242c0afae207b7aefa13133439e3b321d7db4229febc5b7bd5285770ac7f7", + "https://deno.land/x/yargs@v17.1.1-deno/build/lib/typings/common-types.js": "9618b81a86acb88a61fd9988e9bc3ec21c5250d94fc2231ba7d898e71500789d", + "https://deno.land/x/yargs@v17.1.1-deno/build/lib/usage.js": "61071feb99ac220f1b27036406ae8e4f9ee606b373a5f3bcb60042c7bcfbd0d8", + "https://deno.land/x/yargs@v17.1.1-deno/build/lib/utils/apply-extends.js": "64640dce92669705abead3bdbe2c46c8318c8623843a55e4726fb3c55ff9dd1d", + "https://deno.land/x/yargs@v17.1.1-deno/build/lib/utils/is-promise.js": "be45baa3090c5106dd4e442cceef6b357a268783a2ee28ec10fe131a8cd8db72", + "https://deno.land/x/yargs@v17.1.1-deno/build/lib/utils/levenshtein.js": "d8638efc3376b5f794b1c8df6ef4f3d484b29d919127c7fdc242400e3cfded91", + "https://deno.land/x/yargs@v17.1.1-deno/build/lib/utils/maybe-async-result.js": "31cf4026279e14c87d16faa14ac758f35c8cc5795d29393c5ce07120f5a3caf6", + "https://deno.land/x/yargs@v17.1.1-deno/build/lib/utils/obj-filter.js": "5523fb2288d1e86ed48c460e176770b49587554df4ae2405b468c093786b040b", + "https://deno.land/x/yargs@v17.1.1-deno/build/lib/utils/set-blocking.js": "6fa8ffc3299f456e42902736bae35fbc1f2dc96b3905a02ba9629f5bd9f80af1", + "https://deno.land/x/yargs@v17.1.1-deno/build/lib/utils/which-module.js": "9267633b2c9f8990b2c699101b641e59ae59932e0dee5270613c0508bfa13c5d", + "https://deno.land/x/yargs@v17.1.1-deno/build/lib/validation.js": "3dc366de2eb23bc9457ed3e120b69db9d801251bef3dc19f93e4c0380ac0198c", + "https://deno.land/x/yargs@v17.1.1-deno/build/lib/yargs-factory.js": "a3e629d7d063b5ac007b18a0d8e9ad2ca72ca4d702c5c46822fbbdfdd6c512df", + "https://deno.land/x/yargs@v17.1.1-deno/build/lib/yerror.js": "1d9dead374fe06c8f13f2e4adafc002b8a15682b7185abf29638f1be96fd9dfc", + "https://deno.land/x/yargs@v17.1.1-deno/deno-types.ts": "62f5c61899c6da491890c8c84fd9580cfbfa2a83f5a70f6dc74727bbfb148623", + "https://deno.land/x/yargs@v17.1.1-deno/deno.ts": "f3df0bfd08ba367ec36dc59ef6cab1a391ace49ad44387ec5fe5d76289af08af", + "https://deno.land/x/yargs@v17.1.1-deno/lib/platform-shims/deno.ts": "b5a48b40d5c64fe66f5a77f87ebaf4413eea828ccd8159feeac370b3eef9a356", + "https://deno.land/x/yargs_parser@v20.2.4-deno/build/lib/string-utils.js": "12fc056b23703bc370aae5b179dc5abee53fca277abc30eaf76f78d2546d6413", + "https://deno.land/x/yargs_parser@v20.2.4-deno/build/lib/tokenize-arg-string.js": "7e0875b11795b8e217386e45f14b24a6e501ebbc62e15aa469aa8829d4d0ee61", + "https://deno.land/x/yargs_parser@v20.2.4-deno/build/lib/yargs-parser-types.d.ts": "434deb76c6632b3b6cbc4c6f153f8aca04e06055ae9c6b24b40218cbc42688d9", + "https://deno.land/x/yargs_parser@v20.2.4-deno/build/lib/yargs-parser.js": "453200a7dfbb002e605d8009b7dad30f2b1d93665e046ab89c073a4fe63dfd48", + "https://deno.land/x/yargs_parser@v20.2.4-deno/deno.ts": "ad53c0c82c3982c4fc5be9472384b259e0a32ce1f7ae0f68de7b2445df5642fc", + "https://lra6z45nakk5lnu3yjchp7tftsdnwwikwr65ocha5eojfnlgu4sa.arweave.net/XEHs860CldW2m8JEd_5lnIbbWQq0fdcI4OkckrVmpyQ/_util/assert.ts": "e1f76e77c5ccb5a8e0dbbbe6cce3a56d2556c8cb5a9a8802fc9565af72462149", + "https://lra6z45nakk5lnu3yjchp7tftsdnwwikwr65ocha5eojfnlgu4sa.arweave.net/XEHs860CldW2m8JEd_5lnIbbWQq0fdcI4OkckrVmpyQ/path/_constants.ts": "aba480c4a2c098b6374fdd5951fea13ecc8aaaf8b8aa4dae1871baa50243d676", + "https://lra6z45nakk5lnu3yjchp7tftsdnwwikwr65ocha5eojfnlgu4sa.arweave.net/XEHs860CldW2m8JEd_5lnIbbWQq0fdcI4OkckrVmpyQ/path/_interface.ts": "67b276380d297a7cedc3c17f7a0bf122edcfc96a3e1f69de06f379d85ba0e2c0", + "https://lra6z45nakk5lnu3yjchp7tftsdnwwikwr65ocha5eojfnlgu4sa.arweave.net/XEHs860CldW2m8JEd_5lnIbbWQq0fdcI4OkckrVmpyQ/path/_util.ts": "7820a788b35c26dfc27ff329df12507fc0553ae92727009597046f6cf856b4fa", + "https://lra6z45nakk5lnu3yjchp7tftsdnwwikwr65ocha5eojfnlgu4sa.arweave.net/XEHs860CldW2m8JEd_5lnIbbWQq0fdcI4OkckrVmpyQ/path/common.ts": "e4ec66a7416d56f60331b66e27a8a4f08c7b1cf48e350271cb69754a01cf5c04", + "https://lra6z45nakk5lnu3yjchp7tftsdnwwikwr65ocha5eojfnlgu4sa.arweave.net/XEHs860CldW2m8JEd_5lnIbbWQq0fdcI4OkckrVmpyQ/path/glob.ts": "54fea22f0f47dee07399d22e18a34a84eaf4b8a3771bbed96db43bbec71ef237", + "https://lra6z45nakk5lnu3yjchp7tftsdnwwikwr65ocha5eojfnlgu4sa.arweave.net/XEHs860CldW2m8JEd_5lnIbbWQq0fdcI4OkckrVmpyQ/path/mod.ts": "6de8885c2534757097818e302becd1cefcbc4c28ac022cc279e612ee04e8cfd1", + "https://lra6z45nakk5lnu3yjchp7tftsdnwwikwr65ocha5eojfnlgu4sa.arweave.net/XEHs860CldW2m8JEd_5lnIbbWQq0fdcI4OkckrVmpyQ/path/posix.ts": "7c2539b5d9e22b47f02ccc5b6ef11e3975a5c60f49d2241271eaa5936e6082a0", + "https://lra6z45nakk5lnu3yjchp7tftsdnwwikwr65ocha5eojfnlgu4sa.arweave.net/XEHs860CldW2m8JEd_5lnIbbWQq0fdcI4OkckrVmpyQ/path/separator.ts": "9dd15d46ff84a16e13554f56af7fee1f85f8d0f379efbbe60ac066a60561f036", + "https://lra6z45nakk5lnu3yjchp7tftsdnwwikwr65ocha5eojfnlgu4sa.arweave.net/XEHs860CldW2m8JEd_5lnIbbWQq0fdcI4OkckrVmpyQ/path/win32.ts": "47114c941681ecbabab4ec355cb41d525fb5a14175cc47a5c76fdc5eaee2685a" + } +} diff --git a/3rdparty/loro/crates/loro-internal/fuzz/.gitignore b/3rdparty/loro/crates/loro-internal/fuzz/.gitignore new file mode 100644 index 0000000000..1a45eee776 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/fuzz/.gitignore @@ -0,0 +1,4 @@ +target +corpus +artifacts +coverage diff --git a/3rdparty/loro/crates/loro-internal/fuzz/Cargo.lock b/3rdparty/loro/crates/loro-internal/fuzz/Cargo.lock new file mode 100644 index 0000000000..648738abb0 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/fuzz/Cargo.lock @@ -0,0 +1,1102 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "append-only-bytes" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac436601d6bdde674a0d7fb593e829ffe7b3387c351b356dd20e2d40f5bf3ee5" + +[[package]] +name = "arbitrary" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2d098ff73c1ca148721f37baad5ea6a465a13f9573aba8641fbbbae8164a54e" +dependencies = [ + "derive_arbitrary", +] + +[[package]] +name = "arrayvec" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" + +[[package]] +name = "arref" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ccd462b64c3c72f1be8305905a85d85403d768e8690c9b8bd3b9009a5761679" + +[[package]] +name = "atomic-polyfill" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3ff7eb3f316534d83a8a2c3d1674ace8a5a71198eba31e2e2b597833f699b28" +dependencies = [ + "critical-section", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitmaps" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031043d04099746d8db04daf1fa424b2bc8bd69d92b25962dcde24da39ab64a2" +dependencies = [ + "typenum", +] + +[[package]] +name = "bitmaps" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d084b0137aaa901caf9f1e8b21daa6aa24d41cd806e111335541eff9683bd6" + +[[package]] +name = "bytecount" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c676a478f63e9fa2dd5368a42f28bba0d6c560b775f38583c8bbaa7fcd67c9c" + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + +[[package]] +name = "bytes" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50" + +[[package]] +name = "cc" +version = "1.0.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9f73505338f7d905b19d18738976aae232eb46b8efc15554ffc56deb5d9ebe4" +dependencies = [ + "jobserver", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "cobs" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67ba02a97a2bd10f4b59b25c7973101c79642302776489e030cd13cdab09ed15" + +[[package]] +name = "critical-section" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6548a0ad5d2549e111e1f6a11a6c2e2d00ce6a3dafe22948d67c2b443f775e52" + +[[package]] +name = "darling" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0209d94da627ab5605dcccf08bb18afa5009cfbef48d8a8b7d7bdbc79be25c5e" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "177e3443818124b357d8e76f53be906d60937f0d3a90773a664fa63fa253e621" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.25", +] + +[[package]] +name = "darling_macro" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.25", +] + +[[package]] +name = "derive_arbitrary" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e0efad4403bfc52dc201159c4b842a246a14b98c64b55dfd0f2d89729dfeb8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.25", +] + +[[package]] +name = "either" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797" + +[[package]] +name = "embedded-io" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" + +[[package]] +name = "enum-as-inner" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9720bba047d567ffc8a3cba48bf19126600e249ab7f128e9233e6376976a116" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 1.0.105", +] + +[[package]] +name = "enum-as-inner" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ffccbb6966c05b32ef8fbac435df276c4ae4d3dc55a8cd0eb9745e6c12f546a" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.25", +] + +[[package]] +name = "enum_dispatch" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11f36e95862220b211a6e2aa5eca09b4fa391b13cd52ceb8035a24bf65a79de2" +dependencies = [ + "once_cell", + "proc-macro2", + "quote", + "syn 1.0.105", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + +[[package]] +name = "generic-btree" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "210507e6dec78bb1304e52a174bd99efdd83894219bf20d656a066a0ce2fedc5" +dependencies = [ + "arref", + "fxhash", + "heapless 0.7.16", + "itertools 0.11.0", + "loro-thunderdome", + "proc-macro2", +] + +[[package]] +name = "getrandom" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "hash32" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c35f58762feb77d74ebe43bdbc3210f09be9fe6742234d573bacc26ed92b67" +dependencies = [ + "byteorder", +] + +[[package]] +name = "hash32" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d60b12902ba28e2730cd37e95b8c9223af2808df9e902d4df49588d1470606" +dependencies = [ + "byteorder", +] + +[[package]] +name = "heapless" +version = "0.7.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db04bc24a18b9ea980628ecf00e6c0264f3c1426dac36c00cb49b6fbad8b0743" +dependencies = [ + "atomic-polyfill", + "hash32 0.2.1", + "rustc_version", + "serde", + "spin", + "stable_deref_trait", +] + +[[package]] +name = "heapless" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bfb9eb618601c89945a70e254898da93b13be0388091d42117462b265bb3fad" +dependencies = [ + "hash32 0.3.1", + "stable_deref_trait", +] + +[[package]] +name = "heck" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "im" +version = "15.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0acd33ff0285af998aaf9b57342af478078f53492322fafc47450e09397e0e9" +dependencies = [ + "bitmaps 2.1.0", + "rand_core", + "rand_xoshiro", + "sized-chunks", + "typenum", + "version_check", +] + +[[package]] +name = "imbl" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc3be8d8cd36f33a46b1849f31f837c44d9fa87223baee3b4bd96b8f11df81eb" +dependencies = [ + "bitmaps 3.2.1", + "imbl-sized-chunks", + "rand_core", + "rand_xoshiro", + "version_check", +] + +[[package]] +name = "imbl-sized-chunks" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "144006fb58ed787dcae3f54575ff4349755b00ccc99f4b4873860b654be1ed63" +dependencies = [ + "bitmaps 3.2.1", +] + +[[package]] +name = "itertools" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc" + +[[package]] +name = "jobserver" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "068b1ee6743e4d11fb9c6a1e6064b3693a1b600e7f5f5988047d98b3dc9fb90b" +dependencies = [ + "libc", +] + +[[package]] +name = "leb128" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" + +[[package]] +name = "libc" +version = "0.2.147" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" + +[[package]] +name = "libfuzzer-sys" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8fff891139ee62800da71b7fd5b508d570b9ad95e614a53c6f453ca08366038" +dependencies = [ + "arbitrary", + "cc", + "once_cell", +] + +[[package]] +name = "lock_api" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "loro-common" +version = "0.16.2" +dependencies = [ + "arbitrary", + "enum-as-inner 0.6.0", + "fxhash", + "leb128", + "loro-rle", + "nonmax", + "serde", + "serde_columnar", + "string_cache", + "thiserror", +] + +[[package]] +name = "loro-delta" +version = "0.16.2" +dependencies = [ + "arrayvec", + "enum-as-inner 0.5.1", + "generic-btree", + "heapless 0.8.0", + "tracing", +] + +[[package]] +name = "loro-internal" +version = "0.16.2" +dependencies = [ + "append-only-bytes", + "arbitrary", + "arref", + "bytes", + "either", + "enum-as-inner 0.5.1", + "enum_dispatch", + "fxhash", + "generic-btree", + "getrandom", + "im", + "itertools 0.12.1", + "leb128", + "loro-common", + "loro-delta", + "loro-rle", + "loro_fractional_index", + "md5", + "num", + "num-derive", + "num-traits", + "once_cell", + "postcard", + "rand", + "serde", + "serde_columnar", + "serde_json", + "smallvec", + "tabled", + "thiserror", + "tracing", +] + +[[package]] +name = "loro-internal-fuzz" +version = "0.0.0" +dependencies = [ + "libfuzzer-sys", + "loro-internal", +] + +[[package]] +name = "loro-rle" +version = "0.16.2" +dependencies = [ + "append-only-bytes", + "arref", + "enum-as-inner 0.6.0", + "fxhash", + "num", + "smallvec", +] + +[[package]] +name = "loro-thunderdome" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f3d053a135388e6b1df14e8af1212af5064746e9b87a06a345a7a779ee9695a" + +[[package]] +name = "loro_fractional_index" +version = "0.16.2" +dependencies = [ + "imbl", + "rand", + "serde", + "smallvec", +] + +[[package]] +name = "md5" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" + +[[package]] +name = "new_debug_unreachable" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4a24736216ec316047a1fc4252e27dabb04218aa4a3f37c6e7ddbf1f9782b54" + +[[package]] +name = "nonmax" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "610a5acd306ec67f907abe5567859a3c693fb9886eb1f012ab8f2a47bef3db51" + +[[package]] +name = "num" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43db66d1170d347f9a065114077f7dccb00c1b9478c89384490a3425279a4606" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ae39348c8bc5fbd7f40c727a9925f03517afd2ab27d46702108b6a7e5414c19" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-derive" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "876a53fff98e03a936a674b29568b0e605f06b29372c2489ff4de23f1949743d" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.105", +] + +[[package]] +name = "num-integer" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0" +dependencies = [ + "autocfg", + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" + +[[package]] +name = "papergrid" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1526bb6aa9f10ec339fb10360f22c57edf81d5678d0278e93bc12a47ffbe4b01" +dependencies = [ + "bytecount", + "fnv", + "unicode-width", +] + +[[package]] +name = "parking_lot" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ff9f3fef3968a3ec5945535ed654cb38ff72d7495a25619e2247fb15a2ed9ba" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-sys", +] + +[[package]] +name = "phf_shared" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" + +[[package]] +name = "postcard" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a55c51ee6c0db07e68448e336cf8ea4131a620edefebf9893e759b2d793420f8" +dependencies = [ + "cobs", + "embedded-io", + "heapless 0.7.16", + "serde", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "precomputed-hash" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn 1.0.105", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + +[[package]] +name = "proc-macro2" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "573015e8ab27661678357f27dc26460738fd2b6c86e46f386fde94cb5d913105" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rand_xoshiro" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa" +dependencies = [ + "rand_core", +] + +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags", +] + +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver", +] + +[[package]] +name = "ryu" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09" + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "semver" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e25dfac463d778e353db5be2449d1cce89bd6fd23c9f1ea21310ce6e5a1b29c4" + +[[package]] +name = "serde" +version = "1.0.171" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30e27d1e4fd7659406c492fd6cfaf2066ba8773de45ca75e855590f856dc34a9" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_columnar" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3748cbf2b43a15ee9627881cabd7820d50508781cdebd3bb54cea49215d367a1" +dependencies = [ + "itertools 0.11.0", + "postcard", + "serde", + "serde_columnar_derive", + "thiserror", +] + +[[package]] +name = "serde_columnar_derive" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e5eaacabbc55a397ffbb1ee32523f40f86fdefea8a8d9db19630d8b7c00edd1" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.25", +] + +[[package]] +name = "serde_derive" +version = "1.0.171" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389894603bd18c46fa56231694f8d827779c0951a667087194cf9de94ed24682" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.25", +] + +[[package]] +name = "serde_json" +version = "1.0.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020ff22c755c2ed3f8cf162dbb41a7268d934702f3ed3631656ea597e08fc3db" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "siphasher" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" + +[[package]] +name = "sized-chunks" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16d69225bde7a69b235da73377861095455d298f2b970996eec25ddbb42b3d1e" +dependencies = [ + "bitmaps 2.1.0", + "typenum", +] + +[[package]] +name = "smallvec" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" +dependencies = [ + "serde", +] + +[[package]] +name = "spin" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f6002a767bff9e83f8eeecf883ecb8011875a21ae8da43bffb817a57e78cc09" +dependencies = [ + "lock_api", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + +[[package]] +name = "string_cache" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f91138e76242f575eb1d3b38b4f1362f10d3a43f47d182a5b359af488a02293b" +dependencies = [ + "new_debug_unreachable", + "once_cell", + "parking_lot", + "phf_shared", + "precomputed-hash", + "serde", +] + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "syn" +version = "1.0.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b9b43d45702de4c839cb9b51d9f529c5dd26a4aff255b42b1ebc03e88ee908" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15e3fc8c0c74267e2df136e5e5fb656a464158aa57624053375eb9c8c6e25ae2" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tabled" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56c3ee73732ffceaea7b8f6b719ce3bb17f253fa27461ffeaf568ebd0cdb4b85" +dependencies = [ + "papergrid", + "tabled_derive", + "unicode-width", +] + +[[package]] +name = "tabled_derive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "beca1b4eaceb4f2755df858b88d9b9315b7ccfd1ffd0d7a48a52602301f01a57" +dependencies = [ + "heck", + "proc-macro-error", + "proc-macro2", + "quote", + "syn 1.0.105", +] + +[[package]] +name = "thiserror" +version = "1.0.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a35fc5b8971143ca348fa6df4f024d4d55264f3468c71ad1c2f365b0a4d58c42" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "463fe12d7993d3b327787537ce8dd4dfa058de32fc2b195ef3cde03dc4771e8f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.25", +] + +[[package]] +name = "tracing" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.25", +] + +[[package]] +name = "tracing-core" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +dependencies = [ + "once_cell", +] + +[[package]] +name = "typenum" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" + +[[package]] +name = "unicode-ident" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3" + +[[package]] +name = "unicode-width" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "windows-sys" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4" + +[[package]] +name = "windows_i686_gnu" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7" + +[[package]] +name = "windows_i686_msvc" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf7b1b21b5362cbc318f686150e5bcea75ecedc74dd157d874d754a2ca44b0ed" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5" diff --git a/3rdparty/loro/crates/loro-internal/fuzz/Cargo.toml b/3rdparty/loro/crates/loro-internal/fuzz/Cargo.toml new file mode 100644 index 0000000000..264a9167bd --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/fuzz/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "loro-internal-fuzz" +version = "1.2.7" +authors = ["Automatically generated"] +publish = false +edition = "2018" + +[package.metadata] +cargo-fuzz = true + +[dependencies] +libfuzzer-sys = "0.4" + +[dependencies.loro-internal] +path = ".." +features = ["test_utils"] + +# Prevent this from interfering with workspaces +[workspace] +members = ["."] + +[[bin]] +name = "import" +path = "fuzz_targets/import.rs" +test = false +doc = false + +[[bin]] +name = "alloc_tree" +path = "fuzz_targets/alloc_tree.rs" +test = false +doc = false diff --git a/3rdparty/loro/crates/loro-internal/fuzz/fuzz_targets/alloc_tree.rs b/3rdparty/loro/crates/loro-internal/fuzz/fuzz_targets/alloc_tree.rs new file mode 100644 index 0000000000..93f3932af6 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/fuzz/fuzz_targets/alloc_tree.rs @@ -0,0 +1,10 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; +use loro_internal::dag::{fuzz_alloc_tree, Interaction}; + +fuzz_target!(|data: Vec| { + fuzz_alloc_tree(10, data.clone()); + fuzz_alloc_tree(5, data.clone()); + fuzz_alloc_tree(2, data); +}); diff --git a/3rdparty/loro/crates/loro-internal/fuzz/fuzz_targets/import.rs b/3rdparty/loro/crates/loro-internal/fuzz/fuzz_targets/import.rs new file mode 100644 index 0000000000..db0663e4e6 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/fuzz/fuzz_targets/import.rs @@ -0,0 +1,91 @@ +#![no_main] +use libfuzzer_sys::fuzz_target; +use loro_internal::version::Frontiers; +use loro_internal::{LoroDoc, LoroError, LoroValue, TreeParentId, VersionVector}; + +struct DocSnapshot { + value: LoroValue, + oplog_vv: VersionVector, + oplog_frontiers: Frontiers, + state_frontiers: Frontiers, +} + +fn seed_doc() -> LoroDoc { + let doc = LoroDoc::new_auto_commit(); + doc.set_peer_id(0xF00D).unwrap(); + + let map = doc.get_map("map"); + map.insert("seed", "value").unwrap(); + map.insert("number", 42).unwrap(); + + let list = doc.get_list("list"); + list.push("head").unwrap(); + list.push(7).unwrap(); + + let text = doc.get_text("text"); + text.insert_unicode(0, "initial text").unwrap(); + + let tree = doc.get_tree("tree"); + let node = tree.create(TreeParentId::Root).unwrap(); + tree.get_meta(node).unwrap().insert("kind", "seed").unwrap(); + + doc.commit_then_renew(); + doc +} + +fn snapshot(doc: &LoroDoc) -> DocSnapshot { + DocSnapshot { + value: doc.get_deep_value(), + oplog_vv: doc.oplog_vv(), + oplog_frontiers: doc.oplog_frontiers(), + state_frontiers: doc.state_frontiers(), + } +} + +fn assert_unchanged(doc: &LoroDoc, before: &DocSnapshot) { + assert_eq!(doc.get_deep_value(), before.value); + assert_eq!(doc.oplog_vv(), before.oplog_vv); + assert_eq!(doc.oplog_frontiers(), before.oplog_frontiers); + assert_eq!(doc.state_frontiers(), before.state_frontiers); +} + +fn is_prefix_import_error(error: &LoroError) -> bool { + matches!(error, LoroError::ImportUpdatesThatDependsOnOutdatedVersion) +} + +fn fuzz_binary_import(data: &[u8]) { + let doc = seed_doc(); + let before = snapshot(&doc); + + if let Err(error) = doc.import(data) { + if !is_prefix_import_error(&error) { + assert_unchanged(&doc, &before); + } + } +} + +fn fuzz_import_blob_meta(data: &[u8]) { + let _ = LoroDoc::decode_import_blob_meta(data, false); + let _ = LoroDoc::decode_import_blob_meta(data, true); +} + +fn fuzz_json_import(data: &[u8]) { + let Ok(json) = std::str::from_utf8(data) else { + return; + }; + + let doc = seed_doc(); + let before = snapshot(&doc); + + if let Err(error) = doc.import_json_updates(json) { + if !is_prefix_import_error(&error) { + assert_unchanged(&doc, &before); + } + } +} + +fuzz_target!(|data: &[u8]| { + fuzz_binary_import(data); + fuzz_import_blob_meta(data); + fuzz_json_import(data); +}); diff --git a/3rdparty/loro/crates/loro-internal/scripts/fuzz.ts b/3rdparty/loro/crates/loro-internal/scripts/fuzz.ts new file mode 100644 index 0000000000..86f7f1005b --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/scripts/fuzz.ts @@ -0,0 +1,36 @@ +import * as path from "https://deno.land/std@0.105.0/path/mod.ts"; +const __dirname = path.dirname(path.fromFileUrl(import.meta.url)); +import { resolve } from "https://deno.land/std@0.198.0/path/mod.ts"; + +const validTargets = Array.from( + Deno.readDirSync(resolve(__dirname, "../fuzz/fuzz_targets")), +).map((x) => x.name.replace(/.rs$/, "")); + +const targets = + Deno.args.length === 0 + ? validTargets + : Deno.args.filter((x) => validTargets.includes(x)); + +const promises = []; +for (const target of targets) { + const cmd = [ + "cargo", + "+nightly", + "fuzz", + "run", + target, + "--", + "-max_total_time=1", + ]; + console.log("🔨" + cmd.join(" ")); + promises.push( + Deno.run({ + cmd, + stdout: "inherit", + stderr: "inherit", + cwd: resolve(__dirname, ".."), + }).status(), + ); +} + +await Promise.allSettled(promises); diff --git a/3rdparty/loro/crates/loro-internal/scripts/mem.ts b/3rdparty/loro/crates/loro-internal/scripts/mem.ts new file mode 100644 index 0000000000..384ba2b558 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/scripts/mem.ts @@ -0,0 +1,49 @@ +import * as path from "https://deno.land/std@0.105.0/path/mod.ts"; +const __dirname = path.dirname(path.fromFileUrl(import.meta.url)); +import { resolve } from "https://deno.land/std@0.105.0/path/mod.ts"; + +export const Tasks = [ + "100_concurrent", + "200_concurrent", + "automerge", + "10_actor_sync_1000_actions", + "20_actor_sync_1000_actions", + "10_actor_sync_2000_actions", +]; + +export interface Result { + task: string; + maxBytes: number; + endBytes: number; +} + +// run `cargo run --example mem -r -- ${task}` +export async function run(task: string): Promise { + const cmd = `cargo run --example mem -r -- ${task}`; + const process = Deno.run({ + cmd: cmd.split(" "), + cwd: resolve(__dirname, ".."), + stdout: "piped", + stderr: "piped", + }); + + const output = new TextDecoder().decode(await process.stderrOutput()); + try { + // extract "2,555,555" from `dhat: At t-gmax: 2,555,555 bytes` + const maxBytes = parseInt( + output.match(/dhat: At t-gmax:\s+((\d+,?)+) bytes/)![1].replace(/,/g, ""), + ); + const endBytes = parseInt( + output.match(/dhat: At t-end:\s+((\d+,?)+) bytes/)![1].replace(/,/g, ""), + ); + return { + task, + maxBytes, + endBytes, + }; + } catch (e) { + console.error(e); + console.log(output); + throw e; + } +} diff --git a/3rdparty/loro/crates/loro-internal/scripts/run_mem.ts b/3rdparty/loro/crates/loro-internal/scripts/run_mem.ts new file mode 100644 index 0000000000..c1476b9394 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/scripts/run_mem.ts @@ -0,0 +1,9 @@ +import { run, Tasks } from "./mem.ts"; + +const output = []; +for (const task of Tasks) { + const result = await run(task); + output.push(result); +} + +console.log(output); diff --git a/3rdparty/loro/crates/loro-internal/src/allocation/bfs.rs b/3rdparty/loro/crates/loro-internal/src/allocation/bfs.rs new file mode 100644 index 0000000000..47fae0818b --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/allocation/bfs.rs @@ -0,0 +1,386 @@ +#![allow(dead_code)] +use crate::{ + dag::{Dag, DagNode}, + id::ID, + version::Frontiers, +}; + +use rustc_hash::FxHashSet; +use std::collections::BinaryHeap; + +#[derive(Debug, PartialEq, Eq)] +struct SortBase { + id: ID, + lamport: u32, +} + +impl PartialOrd for SortBase { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for SortBase { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.lamport.cmp(&other.lamport) + } +} + +pub struct BfsBody { + queue: BinaryHeap, + visited: FxHashSet, +} + +pub fn calc_critical_version_bfs>( + graph: &D, + start_list: &Frontiers, +) -> Vec { + let mut runner = BfsBody::new(); + runner.run(graph, start_list) +} + +impl BfsBody { + fn new() -> Self { + Self { + queue: BinaryHeap::new(), + visited: FxHashSet::default(), + } + } + + fn run>(&mut self, graph: &D, start_list: &Frontiers) -> Vec { + let mut start_end_set: FxHashSet = start_list.iter().collect(); + for start in start_list.iter() { + self.queue.push(SortBase { + id: start, + lamport: graph.get(start).unwrap().lamport(), + }); + } + let mut result: Vec = Vec::new(); + while let Some(SortBase { id, lamport: _ }) = self.queue.pop() { + if self.queue.is_empty() { + result.push(id); + } + let node = graph.get(id).unwrap(); + if node.deps().is_empty() { + start_end_set.insert(id); + } else { + for to_id in node.deps().iter() { + if self.visited.contains(&to_id) { + continue; + } + self.visited.insert(to_id); + self.queue.push(SortBase { + id: to_id, + lamport: graph.get(to_id).unwrap().lamport(), + }); + } + } + } + result + .iter() + .filter(|id| !start_end_set.contains(id)) + .cloned() + .collect() + } +} + +#[cfg(test)] +mod additional_tests { + use std::collections::BTreeMap; + + use loro_common::{HasId, HasIdSpan}; + use rle::{HasLength, Sliceable}; + + use super::*; + use crate::{ + change::Lamport, + span::{HasLamport, HasLamportSpan}, + version::VersionVector, + }; + + #[derive(Clone, Debug)] + struct TestNode { + id: ID, + lamport: Lamport, + deps: Frontiers, + } + + impl DagNode for TestNode { + fn deps(&self) -> &Frontiers { + &self.deps + } + } + + impl HasId for TestNode { + fn id_start(&self) -> ID { + self.id + } + } + + impl HasLamport for TestNode { + fn lamport(&self) -> Lamport { + self.lamport + } + } + + impl HasLength for TestNode { + fn content_len(&self) -> usize { + 1 + } + } + + impl Sliceable for TestNode { + fn slice(&self, _from: usize, _to: usize) -> Self { + self.clone() + } + } + + #[derive(Debug)] + struct TestDag { + nodes: BTreeMap, + vv: VersionVector, + frontier: Frontiers, + } + + impl TestDag { + fn new(nodes: impl IntoIterator, frontier: Frontiers) -> Self { + let mut vv = VersionVector::default(); + let nodes = nodes + .into_iter() + .map(|node| { + vv.set_end(node.id_end()); + (node.id_start(), node) + }) + .collect(); + Self { + nodes, + vv, + frontier, + } + } + } + + impl Dag for TestDag { + type Node = TestNode; + + fn get(&self, id: ID) -> Option { + self.nodes.get(&id).cloned() + } + + fn frontier(&self) -> &Frontiers { + &self.frontier + } + + fn vv(&self) -> &VersionVector { + &self.vv + } + + fn contains(&self, id: ID) -> bool { + self.nodes.contains_key(&id) + } + } + + fn node(peer: u64, counter: i32, lamport: Lamport, deps: Frontiers) -> TestNode { + TestNode { + id: ID::new(peer, counter), + lamport, + deps, + } + } + + #[test] + fn sort_base_prioritizes_larger_lamport_in_binary_heap() { + let mut heap = BinaryHeap::new(); + heap.push(SortBase { + id: ID::new(1, 0), + lamport: 10, + }); + heap.push(SortBase { + id: ID::new(2, 0), + lamport: 20, + }); + + assert_eq!(heap.pop().unwrap().id, ID::new(2, 0)); + assert_eq!(heap.pop().unwrap().id, ID::new(1, 0)); + } + + #[test] + fn bfs_critical_versions_include_linear_cut_points() { + let root = node(1, 0, 0, Frontiers::default()); + let middle = node(1, 1, 1, root.id.into()); + let head = node(1, 2, 2, middle.id.into()); + let dag = TestDag::new( + vec![root.clone(), middle.clone(), head.clone()], + head.id.into(), + ); + + let critical = calc_critical_version_bfs(&dag, &head.id.into()); + assert_eq!(critical, vec![middle.id]); + } + + #[test] + fn bfs_critical_versions_skip_start_and_leaf_nodes_in_diamond_graphs() { + let root = node(1, 0, 0, Frontiers::default()); + let left = node(2, 0, 1, root.id.into()); + let right = node(3, 0, 2, root.id.into()); + let merge = node(4, 0, 3, Frontiers::from([left.id, right.id])); + let dag = TestDag::new( + vec![root.clone(), left.clone(), right.clone(), merge.clone()], + merge.id.into(), + ); + + let critical = calc_critical_version_bfs(&dag, &merge.id.into()); + assert!(critical.is_empty()); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::{ + cmp::Ordering, + collections::{HashMap, HashSet}, + sync::Arc, + }; + + use crate::{ + change::Lamport, + id::{Counter, PeerID}, + span::{HasId, HasLamport}, + }; + use rle::{HasLength, Sliceable}; + + #[derive(Debug, Clone, PartialEq, Eq)] + struct TestNode { + id: ID, + lamport: Lamport, + len: usize, + deps: Arc, + } + + impl TestNode { + fn new(id: ID, lamport: Lamport, deps: Frontiers) -> Self { + Self { + id, + lamport, + len: 1, + deps: Arc::new(deps), + } + } + } + + impl DagNode for TestNode { + fn deps(&self) -> &Frontiers { + &self.deps + } + } + + impl Sliceable for TestNode { + fn slice(&self, _from: usize, _to: usize) -> Self { + self.clone() + } + } + + impl HasLamport for TestNode { + fn lamport(&self) -> Lamport { + self.lamport + } + } + + impl HasId for TestNode { + fn id_start(&self) -> ID { + self.id + } + } + + impl HasLength for TestNode { + fn content_len(&self) -> usize { + self.len + } + } + + #[derive(Debug)] + struct TestDag { + nodes: HashMap>, + version_vec: crate::version::VersionVector, + } + + impl TestDag { + fn new(nodes: Vec) -> Self { + let mut map: HashMap> = HashMap::new(); + let mut vv = crate::version::VersionVector::new(); + for node in nodes { + vv.insert(node.id.peer, node.id.counter + node.len as Counter); + map.entry(node.id.peer).or_default().push(node); + } + for nodes in map.values_mut() { + nodes.sort_by(|a, b| match a.id.counter.cmp(&b.id.counter) { + Ordering::Equal => a.len.cmp(&b.len), + other => other, + }); + } + Self { + nodes: map, + version_vec: vv, + } + } + } + + impl Dag for TestDag { + type Node = TestNode; + + fn get(&self, id: ID) -> Option { + let arr = self.nodes.get(&id.peer)?; + arr.binary_search_by(|node| { + if node.id.counter > id.counter { + Ordering::Greater + } else if node.id.counter + node.len as i32 <= id.counter { + Ordering::Less + } else { + Ordering::Equal + } + }) + .ok() + .map(|idx| arr[idx].clone()) + } + + fn frontier(&self) -> &Frontiers { + panic!("frontier is not used in bfs tests") + } + + fn vv(&self) -> &crate::version::VersionVector { + &self.version_vec + } + + fn contains(&self, id: ID) -> bool { + self.version_vec.includes_id(id) + } + } + + fn id(peer: PeerID, counter: Counter) -> ID { + ID::new(peer, counter) + } + + fn frontier(ids: &[ID]) -> Frontiers { + let mut frontier = Frontiers::new(); + for id in ids { + frontier.push(*id); + } + frontier + } + + #[test] + fn dedupes_shared_dependencies_and_filters_terminal_nodes() { + let graph = TestDag::new(vec![ + TestNode::new(id(1, 0), 10, frontier(&[id(2, 0), id(3, 0)])), + TestNode::new(id(2, 0), 8, frontier(&[id(4, 0)])), + TestNode::new(id(3, 0), 7, frontier(&[id(4, 0)])), + TestNode::new(id(4, 0), 1, Frontiers::new()), + ]); + + let result = calc_critical_version_bfs(&graph, &frontier(&[id(1, 0)])); + + assert!(result.is_empty()); + assert_eq!(graph.get(id(4, 0)).unwrap().deps().len(), 0); + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/allocation/dfs.rs b/3rdparty/loro/crates/loro-internal/src/allocation/dfs.rs new file mode 100644 index 0000000000..da41c5ca71 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/allocation/dfs.rs @@ -0,0 +1,416 @@ +use std::collections::HashSet; + +use crate::{ + dag::{Dag, DagNode}, + id::ID, + version::Frontiers, +}; + +fn get_all_points>(graph: &D, points: &mut HashSet, current: &ID) { + points.insert(*current); + for to_id in graph.get(*current).unwrap().deps().iter() { + get_all_points(graph, points, &to_id); + } +} + +pub fn get_end_list>(graph: &D, start_list: &Frontiers) -> Frontiers { + let mut end_set: HashSet = HashSet::new(); + for start_id in start_list.iter() { + end_dfs(graph, &start_id, &mut end_set); + } + end_set.into_iter().collect() +} + +fn end_dfs>(graph: &D, current: &ID, end_set: &mut HashSet) { + let binding = graph.get(*current).unwrap(); + let deps = binding.deps(); + if deps.is_empty() { + end_set.insert(*current); + } + for to_id in deps.iter() { + end_dfs(graph, &to_id, end_set); + } +} + +pub fn calc_critical_version_dfs>( + graph: &D, + start_list: &Frontiers, + end_list: &Frontiers, +) -> Vec { + let mut result: Vec = vec![]; + let mut points: HashSet = HashSet::new(); + let start_list_set: HashSet = HashSet::from_iter(start_list.iter()); + let end_list_set: HashSet = HashSet::from_iter(end_list.iter()); + for start_id in start_list.iter() { + get_all_points(graph, &mut points, &start_id); + } + for escape in points { + let mut flag = false; + for start_id in start_list.iter() { + if dfs(graph, &start_id, &escape, &end_list_set) { + flag = true; + break; + } + } + if flag { + continue; + } + if !end_list_set.contains(&escape) && !start_list_set.contains(&escape) { + result.push(escape); + } + } + result +} + +fn dfs>( + graph: &D, + current: &ID, + escape: &ID, + end_list_set: &HashSet, +) -> bool { + if current == escape { + return false; + } + if end_list_set.contains(current) { + return true; + } + for to_id in graph.get(*current).unwrap().deps().iter() { + if dfs(graph, &to_id, escape, end_list_set) { + return true; + } + } + false +} + +#[cfg(test)] +mod additional_tests { + use std::collections::BTreeMap; + + use loro_common::{HasId, HasIdSpan}; + use rle::{HasLength, Sliceable}; + + use super::*; + use crate::{ + change::Lamport, + span::{HasLamport, HasLamportSpan}, + version::VersionVector, + }; + + #[derive(Clone, Debug)] + struct TestNode { + id: ID, + lamport: Lamport, + deps: Frontiers, + } + + impl DagNode for TestNode { + fn deps(&self) -> &Frontiers { + &self.deps + } + } + + impl HasId for TestNode { + fn id_start(&self) -> ID { + self.id + } + } + + impl HasLamport for TestNode { + fn lamport(&self) -> Lamport { + self.lamport + } + } + + impl HasLength for TestNode { + fn content_len(&self) -> usize { + 1 + } + } + + impl Sliceable for TestNode { + fn slice(&self, _from: usize, _to: usize) -> Self { + self.clone() + } + } + + #[derive(Debug)] + struct TestDag { + nodes: BTreeMap, + vv: VersionVector, + frontier: Frontiers, + } + + impl TestDag { + fn new(nodes: impl IntoIterator, frontier: Frontiers) -> Self { + let mut vv = VersionVector::default(); + let nodes = nodes + .into_iter() + .map(|node| { + vv.set_end(node.id_end()); + (node.id_start(), node) + }) + .collect(); + Self { + nodes, + vv, + frontier, + } + } + } + + impl Dag for TestDag { + type Node = TestNode; + + fn get(&self, id: ID) -> Option { + self.nodes.get(&id).cloned() + } + + fn frontier(&self) -> &Frontiers { + &self.frontier + } + + fn vv(&self) -> &VersionVector { + &self.vv + } + + fn contains(&self, id: ID) -> bool { + self.nodes.contains_key(&id) + } + } + + fn node(peer: u64, counter: i32, lamport: Lamport, deps: Frontiers) -> TestNode { + TestNode { + id: ID::new(peer, counter), + lamport, + deps, + } + } + + #[test] + fn end_list_collects_dependency_leaves_from_all_start_frontiers() { + let a = node(1, 0, 0, Frontiers::default()); + let b = node(1, 1, 1, a.id.into()); + let c = node(2, 0, 2, a.id.into()); + let d = node(3, 0, 3, Frontiers::from([b.id, c.id])); + let e = node(4, 0, 4, c.id.into()); + let dag = TestDag::new( + vec![a.clone(), b.clone(), c.clone(), d.clone(), e.clone()], + Frontiers::from([d.id, e.id]), + ); + + let ends = get_end_list(&dag, &Frontiers::from([d.id, e.id])); + assert_eq!(ends, Frontiers::from(a.id)); + } + + #[test] + fn dfs_critical_versions_include_linear_cut_points() { + let root = node(1, 0, 0, Frontiers::default()); + let middle = node(1, 1, 1, root.id.into()); + let head = node(1, 2, 2, middle.id.into()); + let dag = TestDag::new( + vec![root.clone(), middle.clone(), head.clone()], + head.id.into(), + ); + + let critical = calc_critical_version_dfs(&dag, &head.id.into(), &root.id.into()); + assert_eq!(critical, vec![middle.id]); + } + + #[test] + fn dfs_critical_versions_exclude_diamond_branches_with_alternate_paths() { + let root = node(1, 0, 0, Frontiers::default()); + let left = node(2, 0, 1, root.id.into()); + let right = node(3, 0, 2, root.id.into()); + let merge = node(4, 0, 3, Frontiers::from([left.id, right.id])); + let dag = TestDag::new( + vec![root.clone(), left.clone(), right.clone(), merge.clone()], + merge.id.into(), + ); + + let critical = calc_critical_version_dfs(&dag, &merge.id.into(), &root.id.into()); + assert!(critical.is_empty()); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::{ + cmp::Ordering, + collections::{HashMap, HashSet}, + sync::Arc, + }; + + use crate::{ + change::Lamport, + id::{Counter, PeerID}, + span::{HasId, HasLamport}, + }; + use rle::{HasLength, Sliceable}; + + #[derive(Debug, Clone, PartialEq, Eq)] + struct TestNode { + id: ID, + lamport: Lamport, + len: usize, + deps: Arc, + } + + impl TestNode { + fn new(id: ID, lamport: Lamport, deps: Frontiers) -> Self { + Self { + id, + lamport, + len: 1, + deps: Arc::new(deps), + } + } + } + + impl DagNode for TestNode { + fn deps(&self) -> &Frontiers { + &self.deps + } + } + + impl Sliceable for TestNode { + fn slice(&self, _from: usize, _to: usize) -> Self { + self.clone() + } + } + + impl HasLamport for TestNode { + fn lamport(&self) -> Lamport { + self.lamport + } + } + + impl HasId for TestNode { + fn id_start(&self) -> ID { + self.id + } + } + + impl HasLength for TestNode { + fn content_len(&self) -> usize { + self.len + } + } + + #[derive(Debug)] + struct TestDag { + nodes: HashMap>, + version_vec: crate::version::VersionVector, + } + + impl TestDag { + fn new(nodes: Vec) -> Self { + let mut map: HashMap> = HashMap::new(); + let mut vv = crate::version::VersionVector::new(); + for node in nodes { + vv.insert(node.id.peer, node.id.counter + node.len as Counter); + map.entry(node.id.peer).or_default().push(node); + } + for nodes in map.values_mut() { + nodes.sort_by(|a, b| match a.id.counter.cmp(&b.id.counter) { + Ordering::Equal => a.len.cmp(&b.len), + other => other, + }); + } + Self { + nodes: map, + version_vec: vv, + } + } + } + + impl Dag for TestDag { + type Node = TestNode; + + fn get(&self, id: ID) -> Option { + let arr = self.nodes.get(&id.peer)?; + arr.binary_search_by(|node| { + if node.id.counter > id.counter { + Ordering::Greater + } else if node.id.counter + node.len as i32 <= id.counter { + Ordering::Less + } else { + Ordering::Equal + } + }) + .ok() + .map(|idx| arr[idx].clone()) + } + + fn frontier(&self) -> &Frontiers { + panic!("frontier is not used in dfs tests") + } + + fn vv(&self) -> &crate::version::VersionVector { + &self.version_vec + } + + fn contains(&self, id: ID) -> bool { + self.version_vec.includes_id(id) + } + } + + fn id(peer: PeerID, counter: Counter) -> ID { + ID::new(peer, counter) + } + + fn frontier(ids: &[ID]) -> Frontiers { + let mut frontier = Frontiers::new(); + for id in ids { + frontier.push(*id); + } + frontier + } + + fn as_set(ids: Vec) -> HashSet { + ids.into_iter().collect() + } + + #[test] + fn get_end_list_collects_all_leaf_nodes_reachable_from_start() { + let graph = TestDag::new(vec![ + TestNode::new(id(1, 0), 10, frontier(&[id(2, 0), id(3, 0)])), + TestNode::new(id(2, 0), 7, Frontiers::new()), + TestNode::new(id(3, 0), 8, Frontiers::new()), + ]); + + let ends = get_end_list(&graph, &frontier(&[id(1, 0)])); + + assert_eq!(ends.len(), 2); + assert!(ends.contains(&id(2, 0))); + assert!(ends.contains(&id(3, 0))); + assert!(!ends.contains(&id(1, 0))); + } + + #[test] + fn calc_critical_version_dfs_returns_non_start_nodes_when_no_end_is_present() { + let graph = TestDag::new(vec![ + TestNode::new(id(1, 0), 10, frontier(&[id(2, 0), id(3, 0)])), + TestNode::new(id(2, 0), 7, Frontiers::new()), + TestNode::new(id(3, 0), 8, Frontiers::new()), + ]); + + let result = calc_critical_version_dfs(&graph, &frontier(&[id(1, 0)]), &Frontiers::new()); + + assert_eq!(as_set(result), as_set(vec![id(2, 0), id(3, 0)])); + } + + #[test] + fn calc_critical_version_dfs_skips_candidates_when_an_end_is_on_every_start_path() { + let graph = TestDag::new(vec![ + TestNode::new(id(1, 0), 10, frontier(&[id(2, 0), id(3, 0)])), + TestNode::new(id(2, 0), 7, Frontiers::new()), + TestNode::new(id(3, 0), 8, Frontiers::new()), + ]); + + let result = + calc_critical_version_dfs(&graph, &frontier(&[id(1, 0)]), &frontier(&[id(2, 0)])); + + assert!(result.is_empty()); + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/allocation/mod.rs b/3rdparty/loro/crates/loro-internal/src/allocation/mod.rs new file mode 100644 index 0000000000..be34ae5e27 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/allocation/mod.rs @@ -0,0 +1,10 @@ +#![allow(dead_code)] +#![allow(unused)] + +mod bfs; +pub(crate) use bfs::calc_critical_version_bfs as calc_critical_version; + +// Only for testing +mod dfs; +pub(crate) use dfs::calc_critical_version_dfs; +pub(crate) use dfs::get_end_list; diff --git a/3rdparty/loro/crates/loro-internal/src/arena.rs b/3rdparty/loro/crates/loro-internal/src/arena.rs new file mode 100644 index 0000000000..b38877c3d2 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/arena.rs @@ -0,0 +1,856 @@ +mod str_arena; +use self::str_arena::{StrArena, StrArenaCheckpoint}; +use crate::sync::{Mutex, MutexGuard, RwLock, RwLockWriteGuard}; +use crate::{ + change::Lamport, + container::{ + idx::ContainerIdx, + list::list_op::{InnerListOp, ListOp}, + map::MapSet, + ContainerID, + }, + id::Counter, + op::{InnerContent, ListSlice, Op, RawOp, RawOpContent, SliceRange}, + LoroValue, +}; +use append_only_bytes::BytesSlice; +use loro_common::PeerID; +use rustc_hash::FxHashMap; +use std::fmt; +use std::{ + num::NonZeroU16, + ops::{Range, RangeBounds}, + sync::Arc, +}; + +pub(crate) struct LoadAllFlag; +type ParentResolver = dyn Fn(ContainerID) -> Option + Send + Sync + 'static; + +#[derive(Default)] +struct ArenaContainers { + container_idx_to_id: Vec, + /// Cached container depth. `None` means unknown or waiting on an unknown parent depth. + /// Use `get_depth()` for authoritative reads; direct access is only a cache fast path. + depth: Vec>, + container_id_to_idx: FxHashMap, + /// The parent of each container. + parents: FxHashMap>, + /// All retention roots: top-level user roots **and** mergeable cids. Used by + /// alive-container / shallow-snapshot retention walks that must see both. + root_c_idx: Vec, + /// Subset of `root_c_idx` containing only top-level (non-mergeable) roots. This is the + /// list user-facing APIs enumerate (`preferred_root_containers`, `get_value`, + /// `get_deep_value`, jsonpath, ...). Keeping it pre-filtered means those paths do + /// not pay a per-mergeable `is_mergeable()` parse on every call. + top_level_root_c_idx: Vec, + /// Optional resolver used when querying parent for a container that has not been registered yet. + /// If set, `get_parent` will try this resolver to lazily fetch and register the parent. + /// + /// Locking: the resolver may read the state KV store. Code that loads from KV must snapshot + /// KV data and release the KV lock before taking the arena lock. + parent_resolver: Option>, +} + +#[derive(Default)] +struct InnerSharedArena { + // Container metadata is a single consistency domain. Keep it under one + // mutex so container id/index/parent/depth updates cannot acquire locks in + // inconsistent orders. + containers: RwLock, + values: Mutex>, + str: Arc>, +} + +impl fmt::Debug for InnerSharedArena { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("InnerSharedArena") + .field("containers", &">") + .field("values", &">") + .field("str", &">>") + .finish() + } +} + +/// This is shared between [OpLog] and [AppState]. +/// +#[derive(Debug, Clone)] +pub struct SharedArena { + inner: Arc, +} + +pub(crate) struct SharedArenaRollback { + container_len: usize, + root_len: usize, + top_level_root_len: usize, + values_len: usize, + str: StrArenaCheckpoint, +} + +#[derive(Debug)] +pub struct StrAllocResult { + /// unicode start + pub start: usize, + /// unicode end + pub end: usize, +} + +impl ArenaContainers { + /// Add a freshly-registered cid to the retention-root tracking vectors. + /// + /// Centralizes the `root_c_idx ⊇ top_level_root_c_idx` invariant: `root_c_idx` is the set of + /// retention roots (top-level user roots **and** mergeable cids) that seed the alive-walk for + /// shallow snapshot; `top_level_root_c_idx` is the user-visible subset enumerated by + /// `preferred_root_containers` etc. Every push to either vector should go through this method, + /// so the invariant is maintained by construction rather than by remembering to push to two + /// vectors at every call site. + fn push_root(&mut self, idx: ContainerIdx, is_mergeable: bool) { + self.root_c_idx.push(idx); + if !is_mergeable { + self.top_level_root_c_idx.push(idx); + } + } + + fn register_container(&mut self, id: &ContainerID) -> ContainerIdx { + if let Some(&idx) = self.container_id_to_idx.get(id) { + return idx; + } + + let idx = self.container_idx_to_id.len(); + self.container_idx_to_id.push(id.clone()); + let idx = ContainerIdx::from_index_and_type(idx as u32, id.container_type()); + self.container_id_to_idx.insert(id.clone(), idx); + // Resolve the cid's kind once. `is_mergeable` is non-trivial for mergeable names, + // so we avoid calling it twice on the same id. + let mergeable_parts = if id.is_root() { + id.parse_mergeable() + } else { + None + }; + match (id.is_root(), mergeable_parts) { + (true, None) => { + self.push_root(idx, false); + self.parents.insert(idx, None); + self.depth.push(NonZeroU16::new(1)); + } + (true, Some((parent_id, _key, _kind))) => { + // Mergeable Roots are retention roots AND logical children. Push to `root_c_idx` + // so shallow snapshot's `retain_keys` does not GC the loser of a concurrent-kind + // conflict (whose state is reachable only by its deterministic cid, not through + // the parent marker); then `set_parent` for path/event resolution. They are + // deliberately absent from `top_level_root_c_idx` so user-facing root + // enumeration does not have to filter them out. + self.push_root(idx, true); + self.depth.push(None); + let parent_idx = self.register_container(&parent_id); + self.set_parent(idx, Some(parent_idx)); + } + (false, _) => { + self.depth.push(None); + } + } + idx + } + + fn set_parent(&mut self, child: ContainerIdx, parent: Option) { + self.parents.insert(child, parent); + + match parent { + Some(p) => { + // Keep parent linking side-effect free. Calling `get_depth()` here may invoke the + // lazy parent resolver, which can acquire the state KV lock. Import/load paths may + // be assembling parent edges from KV snapshots, so resolving depth here would make + // arena -> KV and KV -> arena lock orders coexist. + if let Some(d) = self.depth[p.to_index() as usize] { + self.depth[child.to_index() as usize] = NonZeroU16::new(d.get() + 1); + } else { + self.depth[child.to_index() as usize] = None; + } + } + None => { + self.depth[child.to_index() as usize] = NonZeroU16::new(1); + } + } + } + + fn get_depth(&mut self, target: ContainerIdx) -> Option { + if let Some(d) = self.depth[target.to_index() as usize] { + return Some(d); + } + + let parent: Option = if let Some(p) = self.parents.get(&target) { + *p + } else { + let id = self + .container_idx_to_id + .get(target.to_index() as usize) + .unwrap() + .clone(); + if id.is_root() && !id.is_mergeable() { + None + } else { + // Mergeable Roots get registered via the main `register_container` path, which + // already wires their parent edge — they should never reach here with a missing + // `parents` entry. But for ordinary children whose parent isn't in the arena + // yet, fall back to the resolver. + let resolver = self.parent_resolver.clone()?; + let parent_id = resolver(id)?; + // Route through `register_container` so a freshly-discovered parent (especially + // a mergeable cid) lands in `root_c_idx`, gets its own parent edge wired up, + // and has `parents` initialized — instead of being half-registered with just + // `idx_to_id` / `id_to_idx` / `depth` and silently missing from retention walks. + let parent_idx = + if let Some(idx) = self.container_id_to_idx.get(&parent_id).copied() { + // Already in the arena. For a top-level root that was hand-registered + // somewhere else without a `parents` entry, ensure it has one. + if parent_id.is_root() && !parent_id.is_mergeable() { + self.parents.entry(idx).or_insert(None); + if self.depth[idx.to_index() as usize].is_none() { + self.depth[idx.to_index() as usize] = NonZeroU16::new(1); + } + } + idx + } else { + self.register_container(&parent_id) + }; + + Some(parent_idx) + } + }; + + let d = match parent { + Some(p) => NonZeroU16::new(self.get_depth(p)?.get() + 1), + None => NonZeroU16::new(1), + }; + self.depth[target.to_index() as usize] = d; + d + } + + fn container_id(&self, idx: ContainerIdx) -> Option { + self.container_idx_to_id + .get(idx.to_index() as usize) + .cloned() + } +} + +pub(crate) struct ArenaGuards<'a> { + containers: RwLockWriteGuard<'a, ArenaContainers>, +} + +impl ArenaGuards<'_> { + pub fn register_container(&mut self, id: &ContainerID) -> ContainerIdx { + self.containers.register_container(id) + } + + pub fn set_parent(&mut self, child: ContainerIdx, parent: Option) { + self.containers.set_parent(child, parent); + } +} + +impl SharedArena { + #[allow(clippy::new_without_default)] + pub fn new() -> Self { + Self { + inner: Arc::new(InnerSharedArena::default()), + } + } + + pub fn fork(&self) -> Self { + Self { + inner: Arc::new(InnerSharedArena { + containers: RwLock::new({ + let containers = self.inner.containers.read(); + ArenaContainers { + container_idx_to_id: containers.container_idx_to_id.clone(), + depth: containers.depth.clone(), + container_id_to_idx: containers.container_id_to_idx.clone(), + parents: containers.parents.clone(), + root_c_idx: containers.root_c_idx.clone(), + top_level_root_c_idx: containers.top_level_root_c_idx.clone(), + parent_resolver: containers.parent_resolver.clone(), + } + }), + values: Mutex::new(self.inner.values.lock().clone()), + str: self.inner.str.clone(), + }), + } + } + + pub(crate) fn checkpoint_for_rollback(&self) -> SharedArenaRollback { + let containers = self.inner.containers.read(); + let container_len = containers.container_idx_to_id.len(); + let root_len = containers.root_c_idx.len(); + let top_level_root_len = containers.top_level_root_c_idx.len(); + drop(containers); + let values_len = self.inner.values.lock().len(); + let str = self.inner.str.lock().checkpoint(); + SharedArenaRollback { + container_len, + root_len, + top_level_root_len, + values_len, + str, + } + } + + pub(crate) fn rollback(&self, checkpoint: SharedArenaRollback) { + let mut containers = self.inner.containers.write(); + let removed_ids = containers + .container_idx_to_id + .split_off(checkpoint.container_len); + for id in removed_ids { + containers.container_id_to_idx.remove(&id); + } + containers.depth.truncate(checkpoint.container_len); + containers.root_c_idx.truncate(checkpoint.root_len); + containers + .top_level_root_c_idx + .truncate(checkpoint.top_level_root_len); + containers.parents.retain(|child, parent| { + let child_is_kept = (child.to_index() as usize) < checkpoint.container_len; + let parent_is_kept = parent + .map(|p| (p.to_index() as usize) < checkpoint.container_len) + .unwrap_or(true); + child_is_kept && parent_is_kept + }); + drop(containers); + + self.inner.values.lock().truncate(checkpoint.values_len); + self.inner.str.lock().rollback(checkpoint.str); + } + + pub(crate) fn with_guards(&self, f: impl FnOnce(&mut ArenaGuards)) { + let mut guards = self.get_arena_guards(); + f(&mut guards); + } + + fn get_arena_guards(&self) -> ArenaGuards<'_> { + ArenaGuards { + containers: self.inner.containers.write(), + } + } + + pub fn register_container(&self, id: &ContainerID) -> ContainerIdx { + self.inner.containers.write().register_container(id) + } + + pub fn get_container_id(&self, idx: ContainerIdx) -> Option { + self.inner.containers.read().container_id(idx) + } + + /// Fast map from `ContainerID` to `ContainerIdx` for containers already registered + /// in the arena. + /// + /// Important: This is not an existence check. Absence here does not imply that a + /// container does not exist, since registration can be lazy and containers may + /// be persisted only in the state KV store until first use. + /// + /// For existence-aware lookup that consults persisted state and performs lazy + /// registration, prefer `DocState::resolve_idx`. + pub fn id_to_idx(&self, id: &ContainerID) -> Option { + self.inner + .containers + .read() + .container_id_to_idx + .get(id) + .copied() + } + + #[inline] + pub fn idx_to_id(&self, id: ContainerIdx) -> Option { + self.inner.containers.read().container_id(id) + } + + #[inline] + pub fn with_idx_to_id(&self, f: impl FnOnce(&Vec) -> R) -> R { + let containers = self.inner.containers.read(); + f(&containers.container_idx_to_id) + } + + pub fn alloc_str(&self, str: &str) -> StrAllocResult { + let mut text_lock = self.inner.str.lock(); + _alloc_str(&mut text_lock, str) + } + + /// return slice and unicode index + pub fn alloc_str_with_slice(&self, str: &str) -> (BytesSlice, StrAllocResult) { + let mut text_lock = self.inner.str.lock(); + _alloc_str_with_slice(&mut text_lock, str) + } + + /// alloc str without extra info + pub fn alloc_str_fast(&self, bytes: &[u8]) { + let mut text_lock = self.inner.str.lock(); + text_lock.alloc(std::str::from_utf8(bytes).unwrap()); + } + + #[inline] + pub fn utf16_len(&self) -> usize { + self.inner.str.lock().len_utf16() + } + + #[inline] + pub fn alloc_value(&self, value: LoroValue) -> usize { + let mut values_lock = self.inner.values.lock(); + _alloc_value(&mut values_lock, value) + } + + #[inline] + pub fn alloc_values(&self, values: impl Iterator) -> std::ops::Range { + let mut values_lock = self.inner.values.lock(); + _alloc_values(&mut values_lock, values) + } + + #[inline] + pub fn set_parent(&self, child: ContainerIdx, parent: Option) { + self.inner.containers.write().set_parent(child, parent); + } + + pub fn log_hierarchy(&self) { + if cfg!(debug_assertions) { + let containers = self.inner.containers.read(); + for (c, p) in containers.parents.iter() { + tracing::info!( + "container {:?} {:?} {:?}", + c, + containers.container_id(*c), + p.and_then(|x| containers.container_id(x)) + ); + } + } + } + + pub fn log_all_containers(&self) { + let containers = self.inner.containers.read(); + containers.container_id_to_idx.iter().for_each(|(id, idx)| { + tracing::info!("container {:?} {:?}", id, idx); + }); + containers + .container_idx_to_id + .iter() + .enumerate() + .for_each(|(i, id)| { + tracing::info!("container {} {:?}", i, id); + }); + } + + pub fn get_parent(&self, child: ContainerIdx) -> Option { + let (child_id, resolver) = { + let containers = self.inner.containers.read(); + let child_id = containers.container_id(child).unwrap(); + if child_id.is_root() && !child_id.is_mergeable() { + // TODO: PERF: we can speed this up by use a special bit in ContainerIdx to indicate + // whether the target is a root container + return None; + } + + // Try fast path first + if let Some(p) = containers.parents.get(&child).copied() { + return p; + } + + // Fallback: try to resolve parent lazily via the resolver if provided. + (child_id, containers.parent_resolver.clone()) + }; + if let Some(resolver) = resolver { + if let Some(parent_id) = resolver(child_id.clone()) { + let parent_idx = self.register_container(&parent_id); + self.set_parent(child, Some(parent_idx)); + return Some(parent_idx); + } + } + + panic!("InternalError: Parent is not registered") + } + + /// Call `f` on each ancestor of `container`, including `container` itself. + /// + /// f(ContainerIdx, is_first) + pub fn with_ancestors(&self, container: ContainerIdx, mut f: impl FnMut(ContainerIdx, bool)) { + let mut container = Some(container); + let mut is_first = true; + while let Some(c) = container { + f(c, is_first); + is_first = false; + container = self.get_parent(c) + } + } + + #[inline] + pub fn slice_by_unicode(&self, range: impl RangeBounds) -> BytesSlice { + self.inner.str.lock().slice_by_unicode(range) + } + + #[inline] + pub fn slice_by_utf8(&self, range: impl RangeBounds) -> BytesSlice { + self.inner.str.lock().slice_bytes(range) + } + + #[inline] + pub fn slice_str_by_unicode_range(&self, range: Range) -> String { + let mut s = self.inner.str.lock(); + let s: &mut StrArena = &mut s; + let mut ans = String::with_capacity(range.len()); + ans.push_str(s.slice_str_by_unicode(range)); + ans + } + + #[inline] + pub fn with_text_slice(&self, range: Range, mut f: impl FnMut(&str)) { + f(self.inner.str.lock().slice_str_by_unicode(range)) + } + + #[inline] + pub fn get_value(&self, idx: usize) -> Option { + self.inner.values.lock().get(idx).cloned() + } + + #[inline] + pub fn get_values(&self, range: Range) -> Vec { + (self.inner.values.lock()[range]).to_vec() + } + + pub fn convert_single_op( + &self, + container: &ContainerID, + peer: PeerID, + counter: Counter, + lamport: Lamport, + content: RawOpContent, + ) -> Op { + let container = self.register_container(container); + self.inner_convert_op(content, peer, counter, lamport, container) + } + + pub fn can_import_snapshot(&self) -> bool { + let str_empty = self.inner.str.lock().is_empty(); + let values_empty = self.inner.values.lock().is_empty(); + str_empty && values_empty + } + + fn inner_convert_op( + &self, + content: RawOpContent<'_>, + _peer: PeerID, + counter: i32, + _lamport: Lamport, + container: ContainerIdx, + ) -> Op { + match content { + crate::op::RawOpContent::Map(MapSet { key, value }) => Op { + counter, + container, + content: crate::op::InnerContent::Map(MapSet { key, value }), + }, + crate::op::RawOpContent::List(list) => match list { + ListOp::Insert { slice, pos } => match slice { + ListSlice::RawData(values) => { + let range = self.alloc_values(values.iter().cloned()); + Op { + counter, + container, + content: crate::op::InnerContent::List(InnerListOp::Insert { + slice: SliceRange::from(range.start as u32..range.end as u32), + pos, + }), + } + } + ListSlice::RawStr { str, unicode_len } => { + let (slice, info) = self.alloc_str_with_slice(&str); + Op { + counter, + container, + content: crate::op::InnerContent::List(InnerListOp::InsertText { + slice, + unicode_start: info.start as u32, + unicode_len: unicode_len as u32, + pos: pos as u32, + }), + } + } + }, + ListOp::Delete(span) => Op { + counter, + container, + content: crate::op::InnerContent::List(InnerListOp::Delete(span)), + }, + ListOp::StyleStart { + start, + end, + info, + key, + value, + } => Op { + counter, + container, + content: InnerContent::List(InnerListOp::StyleStart { + start, + end, + key, + info, + value, + }), + }, + ListOp::StyleEnd => Op { + counter, + container, + content: InnerContent::List(InnerListOp::StyleEnd), + }, + ListOp::Move { + from, + to, + elem_id: from_id, + } => Op { + counter, + container, + content: InnerContent::List(InnerListOp::Move { + from, + to, + elem_id: from_id, + }), + }, + ListOp::Set { elem_id, value } => Op { + counter, + container, + content: InnerContent::List(InnerListOp::Set { elem_id, value }), + }, + }, + crate::op::RawOpContent::Tree(tree) => Op { + counter, + container, + content: crate::op::InnerContent::Tree(tree.clone()), + }, + #[cfg(feature = "counter")] + crate::op::RawOpContent::Counter(c) => Op { + counter, + container, + content: crate::op::InnerContent::Future(crate::op::FutureInnerContent::Counter(c)), + }, + crate::op::RawOpContent::Unknown { prop, value } => Op { + counter, + container, + content: crate::op::InnerContent::Future(crate::op::FutureInnerContent::Unknown { + prop, + value: Box::new(value), + }), + }, + } + } + + #[inline] + pub fn convert_raw_op(&self, op: &RawOp) -> Op { + self.inner_convert_op( + op.content.clone(), + op.id.peer, + op.id.counter, + op.lamport, + op.container, + ) + } + + #[inline] + pub fn export_containers(&self) -> Vec { + self.inner.containers.read().container_idx_to_id.clone() + } + + pub fn export_parents(&self) -> Vec> { + let containers = self.inner.containers.read(); + containers + .container_idx_to_id + .iter() + .enumerate() + .map(|(x, id)| { + let idx = ContainerIdx::from_index_and_type(x as u32, id.container_type()); + let parent_idx = containers.parents.get(&idx)?; + *parent_idx + }) + .collect() + } + + /// Returns all the possible root containers of the docs + /// + /// We need to load all the cached kv in DocState before we can ensure all root contains are covered. + /// So we need the flag type here. + /// + /// This includes mergeable cids (which are also retention roots). Callers that only want + /// user-visible top-level roots should use [`Self::top_level_root_containers`]. + #[inline] + pub(crate) fn root_containers(&self, _f: LoadAllFlag) -> Vec { + self.inner.containers.read().root_c_idx.clone() + } + + /// Returns only the user-visible top-level root containers (excludes mergeable cids). + /// + /// Used by `preferred_root_containers`, `get_value` / `get_deep_value`, jsonpath, etc. — + /// any path that enumerates the doc's top-level roots. Pre-filtering at registration time + /// keeps these calls O(top_level_roots) instead of O(top_level_roots + mergeable_cids). + #[inline] + pub(crate) fn top_level_root_containers(&self, _f: LoadAllFlag) -> Vec { + self.inner.containers.read().top_level_root_c_idx.clone() + } + + // TODO: this can return a u16 directly now, since the depths are always valid + pub(crate) fn get_depth(&self, container: ContainerIdx) -> Option { + self.inner.containers.write().get_depth(container) + } + + pub(crate) fn iter_value_slice( + &self, + range: Range, + ) -> impl Iterator + '_ { + let values = self.inner.values.lock(); + range + .into_iter() + .map(move |i| values.get(i).unwrap().clone()) + } + + #[allow(unused)] + pub(crate) fn log_all_values(&self) { + let values = self.inner.values.lock(); + for (i, v) in values.iter().enumerate() { + loro_common::debug!("value {} {:?}", i, v); + } + } +} + +fn _alloc_str_with_slice( + text_lock: &mut MutexGuard<'_, StrArena>, + str: &str, +) -> (BytesSlice, StrAllocResult) { + let start = text_lock.len_bytes(); + let ans = _alloc_str(text_lock, str); + (text_lock.slice_bytes(start..), ans) +} + +fn _alloc_values( + values_lock: &mut MutexGuard<'_, Vec>, + values: impl Iterator, +) -> Range { + values_lock.reserve(values.size_hint().0); + let start = values_lock.len(); + for value in values { + values_lock.push(value); + } + + start..values_lock.len() +} + +fn _alloc_value(values_lock: &mut MutexGuard<'_, Vec>, value: LoroValue) -> usize { + values_lock.push(value); + values_lock.len() - 1 +} + +fn _alloc_str(text_lock: &mut MutexGuard<'_, StrArena>, str: &str) -> StrAllocResult { + let start = text_lock.len_unicode(); + text_lock.alloc(str); + StrAllocResult { + start, + end: text_lock.len_unicode(), + } +} + +fn _slice_str(range: Range, s: &mut StrArena) -> String { + let mut ans = String::with_capacity(range.len()); + ans.push_str(s.slice_str_by_unicode(range)); + ans +} + +impl SharedArena { + /// Register or clear a resolver to lazily determine a container's parent when missing. + /// + /// - The resolver receives the child `ContainerIdx` and returns an optional `ContainerID` of its parent. + /// - If the resolver returns `Some`, `SharedArena` will register the parent in the arena and link it. + /// - If the resolver is `None` or returns `None`, `get_parent` will panic for non-root containers as before. + pub fn set_parent_resolver(&self, resolver: Option) + where + F: Fn(ContainerID) -> Option + Send + Sync + 'static, + { + self.inner.containers.write().parent_resolver = + resolver.map(|f| Arc::new(f) as Arc); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use loro_common::ContainerType; + use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }; + + /// When a non-mergeable child is registered, then later asks for its depth and the parent + /// resolver yields a mergeable cid, the mergeable parent must be registered through the + /// retention-root path (push to `root_c_idx`, recursively link its own parent) — exactly the + /// same way the main `register_container` path would handle a freshly-encountered mergeable + /// cid. Without this, shallow snapshot's alive-walk would silently GC the mergeable parent's + /// state. + #[test] + fn get_depth_resolver_registers_mergeable_parent_as_retention_root() { + let arena = SharedArena::new(); + + let top_root = ContainerID::new_root("state", ContainerType::Map); + let mergeable_parent = ContainerID::new_mergeable(&top_root, "profile", ContainerType::Map); + let child = ContainerID::new_normal(loro_common::ID::new(1, 0), ContainerType::List); + + let mergeable_parent_for_resolver = mergeable_parent.clone(); + let child_for_resolver = child.clone(); + arena.set_parent_resolver(Some(move |q: ContainerID| { + if q == child_for_resolver { + Some(mergeable_parent_for_resolver.clone()) + } else { + None + } + })); + + let child_idx = arena.register_container(&child); + let _ = arena.get_depth(child_idx); + + let flag = LoadAllFlag; + let mergeable_idx = arena + .id_to_idx(&mergeable_parent) + .expect("resolver should have registered the mergeable parent"); + assert!( + arena.root_containers(flag).contains(&mergeable_idx), + "mergeable parent discovered via resolver must be a retention root" + ); + + let flag = LoadAllFlag; + assert!( + !arena + .top_level_root_containers(flag) + .contains(&mergeable_idx), + "mergeable parent must not appear in user-facing top-level enumeration" + ); + + let flag = LoadAllFlag; + let top_root_idx = arena + .id_to_idx(&top_root) + .expect("mergeable parent's own grandparent must also be registered"); + assert!( + arena + .top_level_root_containers(flag) + .contains(&top_root_idx), + "the mergeable parent's grandparent (a top-level root) must be in the top-level list" + ); + } + + #[test] + fn set_parent_does_not_resolve_missing_parent_depth() { + let arena = SharedArena::new(); + let parent = ContainerID::new_normal(loro_common::ID::new(1, 0), ContainerType::Map); + let child = ContainerID::new_mergeable(&parent, "field", ContainerType::Text); + let resolver_called = Arc::new(AtomicBool::new(false)); + let called = resolver_called.clone(); + arena.set_parent_resolver(Some(move |_| { + called.store(true, Ordering::SeqCst); + None + })); + + arena.register_container(&child); + + assert!( + !resolver_called.load(Ordering::SeqCst), + "registering a mergeable child must not invoke the lazy parent resolver" + ); + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/arena/str_arena.rs b/3rdparty/loro/crates/loro-internal/src/arena/str_arena.rs new file mode 100644 index 0000000000..db7de2a4b8 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/arena/str_arena.rs @@ -0,0 +1,248 @@ +use std::ops::{Bound, RangeBounds}; + +use append_only_bytes::{AppendOnlyBytes, BytesSlice}; + +use crate::container::richtext::richtext_state::unicode_to_utf8_index; +const INDEX_INTERVAL: u32 = 128; + +#[derive(Default, Debug, Clone)] +pub(crate) struct StrArena { + bytes: AppendOnlyBytes, + unicode_indexes: Vec, + len: Index, +} + +#[derive(Debug, Clone)] +pub(crate) struct StrArenaCheckpoint { + bytes_len: usize, + unicode_indexes_len: usize, + len: Index, +} + +#[derive(Debug, Default, Clone, Copy)] +struct Index { + bytes: u32, + utf16: u32, + unicode: u32, +} + +impl StrArena { + #[inline] + pub fn is_empty(&self) -> bool { + self.len.bytes == 0 + } + + #[inline] + #[allow(dead_code)] + pub fn len_bytes(&self) -> usize { + self.len.bytes as usize + } + + #[inline] + pub fn len_utf16(&self) -> usize { + self.len.utf16 as usize + } + + #[inline] + pub fn len_unicode(&self) -> usize { + self.len.unicode as usize + } + + pub fn alloc(&mut self, input: &str) { + let mut utf16 = 0; + let mut unicode_len = 0; + let mut last_save_index = 0; + for (byte_index, c) in input.char_indices() { + let byte_index = byte_index + c.len_utf8(); + utf16 += c.len_utf16() as u32; + unicode_len += 1; + if byte_index - last_save_index > INDEX_INTERVAL as usize { + self._alloc(&input[last_save_index..byte_index], utf16, unicode_len); + last_save_index = byte_index; + utf16 = 0; + unicode_len = 0; + } + } + + if last_save_index != input.len() { + self._alloc(&input[last_save_index..], utf16, unicode_len); + } + } + + fn _alloc(&mut self, input: &str, utf16: u32, unicode_len: i32) { + let s = input; + self.len.bytes += s.len() as u32; + self.len.utf16 += utf16; + self.len.unicode += unicode_len as u32; + self.bytes.push_str(s); + let cur_len = self.len; + + let index = &mut self.unicode_indexes; + if index.is_empty() { + index.push(Index { + bytes: 0, + utf16: 0, + unicode: 0, + }); + } + + let last = index.last().unwrap(); + if cur_len.bytes - last.bytes > INDEX_INTERVAL { + self.unicode_indexes.push(cur_len); + } + } + + #[inline] + pub fn slice_by_unicode(&mut self, range: impl RangeBounds) -> BytesSlice { + let (start, end) = self.unicode_range_to_utf8_range(range); + self.bytes.slice(start..end) + } + + #[inline] + pub fn slice_str_by_unicode(&mut self, range: impl RangeBounds) -> &str { + let (start, end) = self.unicode_range_to_utf8_range(range); + // SAFETY: we know that the range is valid + unsafe { std::str::from_utf8_unchecked(&self.bytes[start..end]) } + } + + fn unicode_range_to_utf8_range(&mut self, range: impl RangeBounds) -> (usize, usize) { + if self.is_empty() { + return (0, 0); + } + + let start = match range.start_bound() { + Bound::Included(&i) => { + unicode_to_byte_index(&self.unicode_indexes, i as u32, &self.bytes) + } + Bound::Excluded(&_i) => unreachable!(), + Bound::Unbounded => 0, + }; + + let end = match range.end_bound() { + Bound::Included(&i) => { + unicode_to_byte_index(&self.unicode_indexes, i as u32 + 1, &self.bytes) + } + Bound::Excluded(&i) => { + unicode_to_byte_index(&self.unicode_indexes, i as u32, &self.bytes) + } + Bound::Unbounded => self.len.bytes as usize, + }; + + (start, end) + } + + #[inline] + pub fn slice_bytes(&self, range: impl RangeBounds) -> BytesSlice { + self.bytes.slice(range) + } + + #[inline] + pub(crate) fn checkpoint(&self) -> StrArenaCheckpoint { + StrArenaCheckpoint { + bytes_len: self.bytes.len(), + unicode_indexes_len: self.unicode_indexes.len(), + len: self.len, + } + } + + pub(crate) fn rollback(&mut self, checkpoint: StrArenaCheckpoint) { + if checkpoint.bytes_len == 0 { + *self = Self::default(); + return; + } + + if checkpoint.bytes_len < self.bytes.len() { + let mut bytes = AppendOnlyBytes::with_capacity(checkpoint.bytes_len); + bytes.push_slice(&self.bytes[..checkpoint.bytes_len]); + self.bytes = bytes; + } + self.unicode_indexes + .truncate(checkpoint.unicode_indexes_len); + self.len = checkpoint.len; + } +} + +fn unicode_to_byte_index(index: &[Index], unicode_index: u32, bytes: &AppendOnlyBytes) -> usize { + let i = match index.binary_search_by_key(&unicode_index, |x| x.unicode) { + Ok(i) => i, + Err(i) => i - 1, + }; + + let index = index[i]; + if index.unicode == unicode_index { + return index.bytes as usize; + } + + // SAFETY: we know that the index must be valid, because we record and calculate the valid index + let s = unsafe { std::str::from_utf8_unchecked(&bytes[index.bytes as usize..]) }; + unicode_to_utf8_index(s, (unicode_index - index.unicode) as usize).unwrap() + + index.bytes as usize +} + +#[cfg(test)] +mod test { + use std::ops::Deref; + + use super::*; + + #[test] + fn test() { + let mut arena = StrArena::default(); + arena.alloc("Hello"); + let slice = arena.slice_by_unicode(0..5); + assert_eq!(slice.deref(), b"Hello"); + arena.alloc("World"); + let slice = arena.slice_by_unicode(5..10); + assert_eq!(slice.deref(), b"World"); + } + + #[test] + fn parse_unicode_correctly() { + let mut arena = StrArena::default(); + arena.alloc("Hello"); + arena.alloc("World"); + arena.alloc("你好"); + arena.alloc("世界"); + let slice = arena.slice_by_unicode(0..4); + assert_eq!(slice.deref(), b"Hell"); + + let slice = arena.slice_by_unicode(4..8); + assert_eq!(slice.deref(), b"oWor"); + + let slice = arena.slice_by_unicode(8..10); + assert_eq!(slice.deref(), b"ld"); + + let slice = arena.slice_by_unicode(10..12); + assert_eq!(slice.deref(), "你好".as_bytes()); + + let slice = arena.slice_by_unicode(12..14); + assert_eq!(slice.deref(), "世界".as_bytes()); + } + + #[test] + fn slice_long_unicode_correctly() { + let mut arena = StrArena::default(); + let src = "一二34567八九零"; + for s in std::iter::repeat_n(src, 100) { + arena.alloc(s); + } + + let slice = arena.slice_by_unicode(110..120); + assert_eq!(slice.deref(), src.as_bytes()); + let slice = arena.slice_by_unicode(111..121); + assert_eq!(slice.deref(), "二34567八九零一".as_bytes()); + } + + #[test] + fn rollback_truncates_bytes_without_losing_prefix() { + let mut arena = StrArena::default(); + arena.alloc("hello"); + let checkpoint = arena.checkpoint(); + arena.alloc(" transient"); + arena.rollback(checkpoint); + arena.alloc("!"); + + let slice = arena.slice_by_unicode(..); + assert_eq!(slice.deref(), b"hello!"); + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/awareness.rs b/3rdparty/loro/crates/loro-internal/src/awareness.rs new file mode 100644 index 0000000000..581b2783ba --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/awareness.rs @@ -0,0 +1,873 @@ +//! Ephemeral presence utilities. +//! +//! `EphemeralStore` is the recommended API: a timestamped, last-write-wins key-value +//! store for transient presence data (cursors, selections, etc.). It supports: +//! - Per-key timeouts: expired entries are skipped by `encode`/`encode_all` and removed +//! by `remove_outdated`. +//! - Import/export via `encode`/`apply` for syncing between peers. +//! - Subscriptions for both local updates (raw bytes to send) and merged updates. +//! +//! The legacy `Awareness` type remains for backward compatibility but is deprecated in +//! favor of `EphemeralStore`. +use crate::sync::Mutex; +use std::sync::atomic::AtomicI64; +use std::sync::Arc; + +use loro_common::{LoroValue, PeerID}; +use rustc_hash::FxHashMap; +use serde::de::{DeserializeSeed, EnumAccess, MapAccess, SeqAccess, VariantAccess}; +use serde::{Deserialize, Deserializer, Serialize}; + +use crate::change::{get_sys_timestamp, Timestamp}; +use crate::{SubscriberSetWithQueue, Subscription}; + +/// `Awareness` is a structure that tracks the ephemeral state of peers. +/// +/// It can be used to synchronize cursor positions, selections, and the names of the peers. +/// +/// The state of a specific peer is expected to be removed after a specified timeout. Use +/// `remove_outdated` to eliminate outdated states. +#[derive(Debug, Clone)] +#[deprecated(since = "1.4.6", note = "Use `EphemeralStore` instead.")] +pub struct Awareness { + peer: PeerID, + peers: FxHashMap, + timeout: i64, +} + +#[derive(Debug, Clone)] +pub struct PeerInfo { + pub state: LoroValue, + pub counter: i32, + // This field is generated locally + pub timestamp: i64, +} + +#[derive(Serialize, Deserialize)] +struct EncodedPeerInfo { + peer: PeerID, + counter: i32, + record: LoroValue, +} + +#[derive(Deserialize)] +struct DecodedPeerInfo { + peer: PeerID, + counter: i32, + #[serde(deserialize_with = "deserialize_depth_limited_loro_value")] + record: LoroValue, +} + +#[allow(deprecated)] +impl Awareness { + pub fn new(peer: PeerID, timeout: i64) -> Awareness { + Awareness { + peer, + timeout, + peers: FxHashMap::default(), + } + } + + pub fn encode(&self, peers: &[PeerID]) -> Vec { + let mut peers_info = Vec::new(); + let now = get_sys_timestamp() as Timestamp; + for peer in peers { + if let Some(peer_info) = self.peers.get(peer) { + if now - peer_info.timestamp > self.timeout { + continue; + } + + let encoded_peer_info = EncodedPeerInfo { + peer: *peer, + record: peer_info.state.clone(), + counter: peer_info.counter, + }; + peers_info.push(encoded_peer_info); + } + } + + postcard::to_allocvec(&peers_info).unwrap() + } + + pub fn encode_all(&self) -> Vec { + let mut peers_info = Vec::new(); + let now = get_sys_timestamp() as Timestamp; + for (peer, peer_info) in self.peers.iter() { + if now - peer_info.timestamp > self.timeout { + continue; + } + + let encoded_peer_info = EncodedPeerInfo { + peer: *peer, + record: peer_info.state.clone(), + counter: peer_info.counter, + }; + peers_info.push(encoded_peer_info); + } + + postcard::to_allocvec(&peers_info).unwrap() + } + + /// Try to apply encoded updates imported from another peer/process. + /// + /// Returns (updated, added). + pub fn try_apply( + &mut self, + encoded_peers_info: &[u8], + ) -> Result<(Vec, Vec), Box> { + let peers_info: Vec = postcard::from_bytes(encoded_peers_info) + .map_err(|err| format!("Failed to decode awareness data: {err}").into_boxed_str())?; + let mut changed_peers = Vec::new(); + let mut added_peers = Vec::new(); + let now = get_sys_timestamp() as Timestamp; + for peer_info in peers_info { + match self.peers.get(&peer_info.peer) { + Some(x) if x.counter >= peer_info.counter || peer_info.peer == self.peer => { + // do nothing + } + _ => { + let old = self.peers.insert( + peer_info.peer, + PeerInfo { + counter: peer_info.counter, + state: peer_info.record, + timestamp: now, + }, + ); + if old.is_some() { + changed_peers.push(peer_info.peer); + } else { + added_peers.push(peer_info.peer); + } + } + } + } + + Ok((changed_peers, added_peers)) + } + + /// Returns (updated, added) + pub fn apply(&mut self, encoded_peers_info: &[u8]) -> (Vec, Vec) { + self.try_apply(encoded_peers_info) + .expect("Failed to decode awareness data") + } + + pub fn set_local_state(&mut self, value: impl Into) { + self._set_local_state(value.into()); + } + + fn _set_local_state(&mut self, value: LoroValue) { + let peer = self.peers.entry(self.peer).or_insert_with(|| PeerInfo { + state: Default::default(), + counter: 0, + timestamp: 0, + }); + + peer.state = value; + peer.counter += 1; + peer.timestamp = get_sys_timestamp() as Timestamp; + } + + pub fn get_local_state(&self) -> Option { + self.peers.get(&self.peer).map(|x| x.state.clone()) + } + + pub fn remove_outdated(&mut self) -> Vec { + let now = get_sys_timestamp() as Timestamp; + let mut removed = Vec::new(); + self.peers.retain(|id, v| { + if now - v.timestamp > self.timeout { + removed.push(*id); + false + } else { + true + } + }); + + removed + } + + pub fn get_all_states(&self) -> &FxHashMap { + &self.peers + } + + pub fn peer(&self) -> PeerID { + self.peer + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EphemeralEventTrigger { + Local, + Import, + Timeout, +} + +#[derive(Debug, Clone)] +pub struct EphemeralStoreEvent { + pub by: EphemeralEventTrigger, + pub added: Arc>, + pub updated: Arc>, + pub removed: Arc>, +} + +pub type LocalEphemeralCallback = Box) -> bool + Send + Sync + 'static>; +pub type EphemeralSubscriber = Box bool + Send + Sync + 'static>; + +/// `EphemeralStore` is a structure that tracks ephemeral key-value state across peers. +/// +/// - Use it for syncing lightweight presence/state like cursors, selections, and UI hints. +/// - Each key uses timestamp-based LWW (Last-Write-Wins) conflict resolution. +/// - Timeout unit: milliseconds. +/// - After timeout: entries are considered expired. They are omitted from +/// `encode`/`encode_all`, and calling [`remove_outdated`] will purge them and +/// notify subscribers with `EphemeralEventTrigger::Timeout`. +/// +/// In Rust, you are responsible for periodically calling [`remove_outdated`] +/// if you want timed-out entries to be removed and corresponding events to be +/// emitted. In the WASM/TS wrapper, a timer runs automatically while the store +/// is non-empty. +/// +/// See: https://loro.dev/docs/tutorial/ephemeral +/// +/// # Example +/// +/// ```rust +/// use loro_internal::awareness::EphemeralStore; +/// +/// let mut store = EphemeralStore::new(1000); +/// store.set("key", "value"); +/// let encoded = store.encode("key"); +/// let mut store2 = EphemeralStore::new(1000); +/// store.subscribe_local_updates(Box::new(|data| { +/// println!("local update: {:?}", data); +/// true +/// })); +/// store2.apply(&encoded); +/// assert_eq!(store2.get("key"), Some("value".into())); +/// ``` +#[derive(Debug, Clone)] +pub struct EphemeralStore { + inner: Arc, +} + +impl EphemeralStore { + /// Create a new `EphemeralStore`. + /// + /// - `timeout`: inactivity timeout in milliseconds. If a key does not + /// receive updates within this duration, it is considered expired. It + /// will be skipped by `encode`/`encode_all`, and removed when + /// [`remove_outdated`] is called (triggering a `Timeout` event to + /// subscribers). + pub fn new(timeout: i64) -> Self { + Self { + inner: Arc::new(EphemeralStoreInner::new(timeout)), + } + } + + /// Encode the latest value of `key`. + /// + /// Expired keys (past timeout) are omitted and produce an empty payload. + pub fn encode(&self, key: &str) -> Vec { + self.inner.encode(key) + } + + /// Encode all non-expired keys. + /// + /// Entries that have exceeded the timeout are not included in the payload. + pub fn encode_all(&self) -> Vec { + self.inner.encode_all() + } + + /// Apply encoded updates imported from another peer/process. + /// + /// Subscribers receive an event with `by = Import` and the lists of + /// `added`/`updated`/`removed` keys as appropriate. + pub fn apply(&self, data: &[u8]) -> Result<(), Box> { + self.inner.apply(data) + } + + pub fn set(&self, key: &str, value: impl Into) { + self.inner.set(key, value) + } + + pub fn delete(&self, key: &str) { + self.inner.delete(key) + } + + pub fn get(&self, key: &str) -> Option { + self.inner.get(key) + } + + /// Remove entries whose last update timestamp has exceeded the timeout. + /// + /// If any keys are removed, subscribers receive an event with + /// `by = Timeout` and the `removed` keys. This does not run automatically + /// in Rust; call it on your own schedule. + pub fn remove_outdated(&self) { + self.inner.remove_outdated() + } + + pub fn get_all_states(&self) -> FxHashMap { + self.inner.get_all_states() + } + + pub fn keys(&self) -> Vec { + self.inner.keys() + } + + /// Subscribe to local ephemeral/awareness updates. + /// + /// The callback receives encoded update bytes whenever local ephemeral state changes. + /// This is useful for syncing awareness information like cursor positions, selections, + /// or presence data to other peers in real-time. + /// + /// **Auto-unsubscription**: If the callback returns `false`, the subscription will be + /// automatically removed, providing a convenient way to implement one-time or conditional + /// subscriptions in Rust. + /// + /// # Parameters + /// - `callback`: Function that receives `&Vec` (encoded ephemeral updates) and returns `bool` + /// - Return `true` to keep the subscription active + /// - Return `false` to automatically unsubscribe + /// + /// # Example + /// ```rust + /// use loro_internal::awareness::EphemeralStore; + /// use std::sync::{Arc, Mutex}; + /// + /// let store = EphemeralStore::new(30000); + /// let update_count = Arc::new(Mutex::new(0)); + /// let count_clone = update_count.clone(); + /// + /// // Subscribe and collect first 3 updates, then auto-unsubscribe + /// let sub = store.subscribe_local_updates(Box::new(move |bytes| { + /// println!("Received {} bytes of ephemeral data", bytes.len()); + /// let mut count = count_clone.lock().unwrap(); + /// *count += 1; + /// *count < 3 // Auto-unsubscribe after 3 updates + /// })); + /// + /// store.set("cursor", 42); + /// ``` + pub fn subscribe_local_updates(&self, callback: LocalEphemeralCallback) -> Subscription { + self.inner.subscribe_local_updates(callback) + } + + pub fn subscribe(&self, callback: EphemeralSubscriber) -> Subscription { + self.inner.subscribe(callback) + } +} + +struct EphemeralStoreInner { + states: Mutex>, + local_subs: SubscriberSetWithQueue<(), LocalEphemeralCallback, Vec>, + subscribers: SubscriberSetWithQueue<(), EphemeralSubscriber, EphemeralStoreEvent>, + timeout: AtomicI64, +} + +impl std::fmt::Debug for EphemeralStoreInner { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "AwarenessV2 {{ states: {:?}, timeout: {:?} }}", + self.states, self.timeout + ) + } +} + +#[derive(Serialize, Deserialize)] +struct EncodedState<'a> { + #[serde(borrow)] + key: &'a str, + value: Option, + timestamp: i64, +} + +#[derive(Deserialize)] +struct DecodedState<'a> { + #[serde(borrow)] + key: &'a str, + #[serde(deserialize_with = "deserialize_optional_depth_limited_loro_value")] + value: Option, + timestamp: i64, +} + +const EPHEMERAL_VALUE_MAX_DEPTH: usize = 512; + +fn deserialize_optional_depth_limited_loro_value<'de, D>( + deserializer: D, +) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + Option::::deserialize(deserializer) + .map(|value| value.map(DepthLimitedLoroValue::into_inner)) +} + +fn deserialize_depth_limited_loro_value<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + LoroValueSeed { + remaining_depth: EPHEMERAL_VALUE_MAX_DEPTH, + } + .deserialize(deserializer) +} + +struct DepthLimitedLoroValue(LoroValue); + +impl DepthLimitedLoroValue { + fn into_inner(self) -> LoroValue { + self.0 + } +} + +impl<'de> Deserialize<'de> for DepthLimitedLoroValue { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + deserialize_depth_limited_loro_value(deserializer).map(Self) + } +} + +#[derive(Clone, Copy)] +struct LoroValueSeed { + remaining_depth: usize, +} + +impl LoroValueSeed { + fn nested(self) -> Result + where + E: serde::de::Error, + { + let remaining_depth = self + .remaining_depth + .checked_sub(1) + .ok_or_else(|| E::custom("LoroValue nesting depth exceeded"))?; + Ok(Self { remaining_depth }) + } +} + +impl<'de> DeserializeSeed<'de> for LoroValueSeed { + type Value = LoroValue; + + fn deserialize(self, deserializer: D) -> Result + where + D: Deserializer<'de>, + { + if deserializer.is_human_readable() { + return Err(serde::de::Error::custom( + "human-readable LoroValue is not supported by the awareness decoder", + )); + } + + deserializer.deserialize_enum( + "LoroValue", + &[ + "Null", + "Bool", + "Double", + "I32", + "String", + "List", + "Map", + "Container", + "Binary", + ], + LoroValueSeedVisitor { seed: self }, + ) + } +} + +struct LoroValueSeedVisitor { + seed: LoroValueSeed, +} + +impl<'de> serde::de::Visitor<'de> for LoroValueSeedVisitor { + type Value = LoroValue; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("a depth-limited LoroValue") + } + + fn visit_enum(self, data: A) -> Result + where + A: EnumAccess<'de>, + { + match data.variant()? { + (DepthLimitedLoroValueField::Null, v) => { + v.unit_variant()?; + Ok(LoroValue::Null) + } + (DepthLimitedLoroValueField::Bool, v) => v.newtype_variant().map(LoroValue::Bool), + (DepthLimitedLoroValueField::Double, v) => v.newtype_variant().map(LoroValue::Double), + (DepthLimitedLoroValueField::I32, v) => v.newtype_variant().map(LoroValue::I64), + (DepthLimitedLoroValueField::String, v) => v + .newtype_variant() + .map(|value: String| LoroValue::String(value.into())), + (DepthLimitedLoroValueField::List, v) => v + .newtype_variant_seed(LoroListSeed { + value_seed: self.seed.nested()?, + }) + .map(|value| LoroValue::List(value.into())), + (DepthLimitedLoroValueField::Map, v) => v + .newtype_variant_seed(LoroMapSeed { + value_seed: self.seed.nested()?, + }) + .map(|value| LoroValue::Map(value.into())), + (DepthLimitedLoroValueField::Container, v) => { + v.newtype_variant().map(LoroValue::Container) + } + (DepthLimitedLoroValueField::Binary, v) => v + .newtype_variant() + .map(|value: Vec| LoroValue::Binary(value.into())), + } + } +} + +#[derive(Deserialize)] +enum DepthLimitedLoroValueField { + Null, + Bool, + Double, + I32, + String, + List, + Map, + Container, + Binary, +} + +struct LoroListSeed { + value_seed: LoroValueSeed, +} + +impl<'de> DeserializeSeed<'de> for LoroListSeed { + type Value = Vec; + + fn deserialize(self, deserializer: D) -> Result + where + D: Deserializer<'de>, + { + deserializer.deserialize_seq(LoroListVisitor { + value_seed: self.value_seed, + }) + } +} + +struct LoroListVisitor { + value_seed: LoroValueSeed, +} + +impl<'de> serde::de::Visitor<'de> for LoroListVisitor { + type Value = Vec; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("a depth-limited LoroValue list") + } + + fn visit_seq(self, mut seq: A) -> Result + where + A: SeqAccess<'de>, + { + let mut list = Vec::new(); + while let Some(value) = seq.next_element_seed(self.value_seed)? { + list.push(value); + } + Ok(list) + } +} + +struct LoroMapSeed { + value_seed: LoroValueSeed, +} + +impl<'de> DeserializeSeed<'de> for LoroMapSeed { + type Value = FxHashMap; + + fn deserialize(self, deserializer: D) -> Result + where + D: Deserializer<'de>, + { + deserializer.deserialize_map(LoroMapVisitor { + value_seed: self.value_seed, + }) + } +} + +struct LoroMapVisitor { + value_seed: LoroValueSeed, +} + +impl<'de> serde::de::Visitor<'de> for LoroMapVisitor { + type Value = FxHashMap; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("a depth-limited LoroValue map") + } + + fn visit_map(self, mut map: A) -> Result + where + A: MapAccess<'de>, + { + let mut value = FxHashMap::default(); + while let Some(key) = map.next_key()? { + let entry = map.next_value_seed(self.value_seed)?; + value.insert(key, entry); + } + Ok(value) + } +} + +#[derive(Debug, Clone)] +struct State { + state: Option, + timestamp: i64, +} + +impl EphemeralStoreInner { + pub fn new(timeout: i64) -> EphemeralStoreInner { + EphemeralStoreInner { + timeout: AtomicI64::new(timeout), + states: Mutex::new(FxHashMap::default()), + local_subs: SubscriberSetWithQueue::new(), + subscribers: SubscriberSetWithQueue::new(), + } + } + + pub fn encode(&self, key: &str) -> Vec { + let mut peers_info = Vec::new(); + let now = get_sys_timestamp() as Timestamp; + let states = self.states.lock(); + if let Some(peer_state) = states.get(key) { + if now - peer_state.timestamp > self.timeout.load(std::sync::atomic::Ordering::Relaxed) + { + return vec![]; + } + let encoded_peer_info = EncodedState { + key, + value: peer_state.state.clone(), + timestamp: peer_state.timestamp, + }; + peers_info.push(encoded_peer_info); + } + + postcard::to_allocvec(&peers_info).unwrap() + } + + pub fn encode_all(&self) -> Vec { + let mut peers_info = Vec::new(); + let now = get_sys_timestamp() as Timestamp; + let states = self.states.lock(); + for (key, peer_state) in states.iter() { + if now - peer_state.timestamp > self.timeout.load(std::sync::atomic::Ordering::Relaxed) + { + continue; + } + let encoded_peer_info = EncodedState { + key, + value: peer_state.state.clone(), + timestamp: peer_state.timestamp, + }; + peers_info.push(encoded_peer_info); + } + postcard::to_allocvec(&peers_info).unwrap() + } + + pub fn apply(&self, data: &[u8]) -> Result<(), Box> { + let peers_info = match postcard::from_bytes::>(data) { + Ok(ans) => ans, + Err(err) => return Err(format!("Failed to decode data: {}", err).into()), + }; + + let mut updated_keys = Vec::new(); + let mut added_keys = Vec::new(); + let mut removed_keys = Vec::new(); + let now = get_sys_timestamp() as Timestamp; + let timeout = self.timeout.load(std::sync::atomic::Ordering::Relaxed); + let mut states = self.states.lock(); + for DecodedState { + key, + value: record, + timestamp, + } in peers_info + { + if now - timestamp > timeout { + continue; + } + + match states.get_mut(key) { + Some(peer_info) if peer_info.timestamp >= timestamp => { + // do nothing + } + _ => { + let old = states.insert( + key.to_string(), + State { + state: record.clone(), + timestamp, + }, + ); + match (old, record) { + (Some(_), Some(_)) => updated_keys.push(key.to_string()), + (None, Some(_)) => added_keys.push(key.to_string()), + (Some(_), None) => removed_keys.push(key.to_string()), + (None, None) => {} + } + } + } + } + + drop(states); + if !self.subscribers.inner().is_empty() { + self.subscribers.emit( + &(), + EphemeralStoreEvent { + by: EphemeralEventTrigger::Import, + added: Arc::new(added_keys), + updated: Arc::new(updated_keys), + removed: Arc::new(removed_keys), + }, + ); + } + + Ok(()) + } + + pub fn set(&self, key: &str, value: impl Into) { + self._set_local_state(key, Some(value.into())); + } + + pub fn delete(&self, key: &str) { + self._set_local_state(key, None); + } + + pub fn get(&self, key: &str) -> Option { + let states = self.states.lock(); + states.get(key).and_then(|x| x.state.clone()) + } + + pub fn remove_outdated(&self) { + let now = get_sys_timestamp() as Timestamp; + let mut removed = Vec::new(); + let mut states = self.states.lock(); + states.retain(|key, state| { + if now - state.timestamp > self.timeout.load(std::sync::atomic::Ordering::Relaxed) { + if state.state.is_some() { + removed.push(key.clone()); + } + false + } else { + true + } + }); + drop(states); + if !self.subscribers.inner().is_empty() { + self.subscribers.emit( + &(), + EphemeralStoreEvent { + by: EphemeralEventTrigger::Timeout, + added: Arc::new(Vec::new()), + updated: Arc::new(Vec::new()), + removed: Arc::new(removed), + }, + ); + } + } + + pub fn get_all_states(&self) -> FxHashMap { + let states = self.states.lock(); + states + .iter() + .filter(|(_, v)| v.state.is_some()) + .map(|(k, v)| (k.clone(), v.state.clone().unwrap())) + .collect() + } + + pub fn keys(&self) -> Vec { + let states = self.states.lock(); + states + .keys() + .filter(|&k| states.get(k).unwrap().state.is_some()) + .map(|s| s.to_string()) + .collect() + } + + /// Subscribe to local ephemeral state updates. + /// + /// The callback receives encoded update bytes whenever the local peer's ephemeral state + /// is modified. This enables real-time synchronization of awareness data like cursor + /// positions, user presence, or temporary collaborative state. + /// + /// **Auto-unsubscription**: If the callback returns `false`, the subscription will be + /// automatically removed, making it easy to implement conditional or temporary subscriptions. + /// + /// # Parameters + /// - `callback`: Function that receives `&Vec` and returns `bool` + /// - Return `true` to maintain the subscription + /// - Return `false` to automatically unsubscribe + /// + /// # Returns + /// A `Subscription` that can be used to manually unsubscribe if needed. + pub fn subscribe_local_updates(&self, callback: LocalEphemeralCallback) -> Subscription { + let (sub, activate) = self.local_subs.inner().insert((), callback); + activate(); + sub + } + + pub fn subscribe(&self, callback: EphemeralSubscriber) -> Subscription { + let (sub, activate) = self.subscribers.inner().insert((), callback); + activate(); + sub + } + + fn _set_local_state(&self, key: &str, value: Option) { + let is_delete = value.is_none(); + let mut states = self.states.lock(); + let old = states.insert( + key.to_string(), + State { + state: value, + timestamp: get_sys_timestamp() as Timestamp, + }, + ); + + drop(states); + if !self.local_subs.inner().is_empty() { + self.local_subs.emit(&(), self.encode(key)); + } + if !self.subscribers.inner().is_empty() { + if old.is_some() { + self.subscribers.emit( + &(), + EphemeralStoreEvent { + by: EphemeralEventTrigger::Local, + added: Arc::new(Vec::new()), + updated: if !is_delete { + Arc::new(vec![key.to_string()]) + } else { + Arc::new(Vec::new()) + }, + removed: if !is_delete { + Arc::new(Vec::new()) + } else { + Arc::new(vec![key.to_string()]) + }, + }, + ); + } else if !is_delete { + self.subscribers.emit( + &(), + EphemeralStoreEvent { + by: EphemeralEventTrigger::Local, + added: Arc::new(vec![key.to_string()]), + updated: Arc::new(Vec::new()), + removed: Arc::new(Vec::new()), + }, + ); + } + } + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/change.rs b/3rdparty/loro/crates/loro-internal/src/change.rs new file mode 100644 index 0000000000..72edc4a70c --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/change.rs @@ -0,0 +1,329 @@ +//! [Change]s are merged ops. +//! +//! Every [Change] has deps on other [Change]s. All [Change]s in the document thus form a DAG. +//! +//! Note: `dep` can only point to the end of the other [Change]. This is the invariant of [Change]s. + +use crate::{ + dag::DagNode, + estimated_size::EstimatedSize, + id::{Counter, ID}, + op::Op, + span::{HasId, HasLamport}, + version::Frontiers, +}; +use loro_common::{HasCounter, HasCounterSpan, PeerID}; +use num::traits::AsPrimitive; +use rle::{HasIndex, HasLength, Mergable, RleVec, Sliceable}; +use smallvec::SmallVec; + +pub type Timestamp = i64; +pub type Lamport = u32; + +/// A `Change` contains a list of [Op]s. +/// +/// When undo/redo we should always undo/redo a whole [Change]. +// PERF change slice and getting length is kinda slow I guess +#[derive(Debug, Clone, PartialEq)] +pub struct Change { + /// id of the first op in the change + pub(crate) id: ID, + /// Lamport timestamp of the change. It can be calculated from deps + pub(crate) lamport: Lamport, + pub(crate) deps: Frontiers, + /// [Unix time](https://en.wikipedia.org/wiki/Unix_time) + /// It is the number of seconds that have elapsed since 00:00:00 UTC on 1 January 1970. + pub(crate) timestamp: Timestamp, + pub(crate) commit_msg: Option>, + pub(crate) ops: RleVec<[O; 1]>, +} + +pub(crate) struct ChangeRef<'a, O = Op> { + pub(crate) id: &'a ID, + pub(crate) lamport: &'a Lamport, + pub(crate) deps: &'a Frontiers, + pub(crate) timestamp: &'a Timestamp, + pub(crate) commit_msg: &'a Option>, + pub(crate) ops: &'a RleVec<[O; 1]>, +} + +impl<'a, O> ChangeRef<'a, O> { + pub fn from_change(change: &'a Change) -> Self { + Self { + id: &change.id, + lamport: &change.lamport, + deps: &change.deps, + timestamp: &change.timestamp, + commit_msg: &change.commit_msg, + ops: &change.ops, + } + } +} + +impl Change { + pub fn new( + ops: RleVec<[O; 1]>, + deps: Frontiers, + id: ID, + lamport: Lamport, + timestamp: Timestamp, + ) -> Self { + Change { + ops, + deps, + id, + lamport, + timestamp, + commit_msg: None, + } + } + + #[inline] + pub fn ops(&self) -> &RleVec<[O; 1]> { + &self.ops + } + + #[inline] + pub fn deps(&self) -> &Frontiers { + &self.deps + } + + #[inline] + pub fn peer(&self) -> PeerID { + self.id.peer + } + + #[inline] + pub fn lamport(&self) -> Lamport { + self.lamport + } + + #[inline] + pub fn timestamp(&self) -> Timestamp { + self.timestamp + } + + #[inline] + pub fn id(&self) -> ID { + self.id + } + + #[inline] + pub fn deps_on_self(&self) -> bool { + if let Some(id) = self.deps.as_single() { + id.peer == self.id.peer + } else { + false + } + } + + pub fn message(&self) -> Option<&Arc> { + self.commit_msg.as_ref() + } +} + +impl EstimatedSize for Change { + /// Estimate the storage size of the change in bytes + #[inline] + fn estimate_storage_size(&self) -> usize { + let id_size = 2; + let lamport_size = 1; + let timestamp_size = 1; + let deps_size = (self.deps.len().max(1) - 1) * 4; + let ops_size = self + .ops + .iter() + .map(|op| op.estimate_storage_size()) + .sum::(); + id_size + lamport_size + timestamp_size + ops_size + deps_size + } +} + +impl HasIndex for Change { + type Int = Counter; + + fn get_start_index(&self) -> Self::Int { + self.id.counter + } +} + +impl HasId for Change { + fn id_start(&self) -> ID { + self.id + } +} + +impl HasCounter for Change { + fn ctr_start(&self) -> Counter { + self.id.counter + } +} + +impl HasLamport for Change { + fn lamport(&self) -> Lamport { + self.lamport + } +} + +impl Mergable for Change { + fn is_mergable(&self, _other: &Self, _conf: &()) -> bool + where + Self: Sized, + { + false + } + + fn merge(&mut self, _other: &Self, _conf: &()) + where + Self: Sized, + { + unreachable!() + } +} + +impl Change { + pub fn len(&self) -> usize { + self.ops.span().as_() + } + + pub fn is_empty(&self) -> bool { + self.ops.is_empty() + } +} + +use std::{fmt::Debug, sync::Arc}; +impl HasLength for Change { + fn content_len(&self) -> usize { + self.ops.span().as_() + } +} + +impl Sliceable for Change { + // TODO: feels slow, need to confirm whether this affects performance + fn slice(&self, from: usize, to: usize) -> Self { + assert!(from < to); + assert!(to <= self.atom_len()); + let from_counter = self.id.counter + from as Counter; + let to_counter = self.id.counter + to as Counter; + let ops = { + if from >= to { + RleVec::new() + } else { + let mut ans: SmallVec<[_; 1]> = SmallVec::new(); + let mut start_index = 0; + if self.ops.len() >= 8 { + let result = self + .ops + .binary_search_by(|op| op.ctr_end().cmp(&from_counter)); + start_index = match result { + Ok(i) => i, + Err(i) => i, + }; + } + + for i in start_index..self.ops.len() { + let op = &self.ops[i]; + if op.ctr_start() >= to_counter { + break; + } + if op.ctr_end() <= from_counter { + continue; + } + + let start_offset = + ((from_counter - op.ctr_start()).max(0) as usize).min(op.atom_len()); + let end_offset = + ((to_counter - op.ctr_start()).max(0) as usize).min(op.atom_len()); + assert_ne!(start_offset, end_offset); + ans.push(op.slice(start_offset, end_offset)) + } + + RleVec::from(ans) + } + }; + assert_eq!(ops.first().unwrap().ctr_start(), from_counter); + assert_eq!(ops.last().unwrap().ctr_end(), to_counter); + Self { + ops, + deps: if from > 0 { + Frontiers::from_id(self.id.inc(from as Counter - 1)) + } else { + self.deps.clone() + }, + id: self.id.inc(from as Counter), + lamport: self.lamport + from as Lamport, + timestamp: self.timestamp, + commit_msg: self.commit_msg.clone(), + } + } +} + +impl DagNode for Change { + fn deps(&self) -> &Frontiers { + &self.deps + } +} + +impl Change { + pub fn can_merge_right(&self, other: &Self, merge_interval: i64) -> bool { + if other.id.peer == self.id.peer + && other.id.counter == self.id.counter + self.content_len() as Counter + && other.deps.len() == 1 + && other.deps.as_single().unwrap().peer == self.id.peer + && other.timestamp - self.timestamp <= merge_interval + && self.commit_msg == other.commit_msg + { + debug_assert!(other.timestamp >= self.timestamp); + debug_assert!(other.lamport == self.lamport + self.len() as Lamport); + true + } else { + false + } + } +} + +/// [Unix time](https://en.wikipedia.org/wiki/Unix_time) +/// It is the number of milliseconds that have elapsed since 00:00:00 UTC on 1 January 1970. +#[cfg(any( + not(target_arch = "wasm32"), + target_os = "emscripten", + target_os = "wasi" +))] +pub(crate) fn get_sys_timestamp() -> f64 { + use std::time::{SystemTime, UNIX_EPOCH}; + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_millis() + .as_() +} + +/// [Unix time](https://en.wikipedia.org/wiki/Unix_time) +/// It is the number of milliseconds that have elapsed since 00:00:00 UTC on 1 January 1970. +#[cfg(all( + target_arch = "wasm32", + not(target_os = "emscripten"), + not(target_os = "wasi") +))] +pub fn get_sys_timestamp() -> f64 { + use wasm_bindgen::prelude::wasm_bindgen; + #[wasm_bindgen] + extern "C" { + // Use `js_namespace` here to bind `console.log(..)` instead of just + // `log(..)` + #[wasm_bindgen(js_namespace = Date)] + pub fn now() -> f64; + } + + now() +} + +#[cfg(test)] +mod test { + use super::*; + #[test] + fn size_of_change() { + let size = std::mem::size_of::(); + println!("{}", size); + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/change_meta.rs b/3rdparty/loro/crates/loro-internal/src/change_meta.rs new file mode 100644 index 0000000000..93e9cbd9e6 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/change_meta.rs @@ -0,0 +1,98 @@ +use std::{cmp::Ordering, sync::Arc}; + +use loro_common::HasLamport; +use rle::HasLength; + +use crate::{ + change::{Change, Lamport, Timestamp}, + id::ID, + version::Frontiers, +}; + +/// `Change` is a grouped continuous operations that share the same id, timestamp, commit message. +/// +/// - The id of the `Change` is the id of its first op. +/// - The second op's id is `{ peer: change.id.peer, counter: change.id.counter + 1 }` +/// +/// The same applies on `Lamport`: +/// +/// - The lamport of the `Change` is the lamport of its first op. +/// - The second op's lamport is `change.lamport + 1` +/// +/// The length of the `Change` is how many operations it contains +#[derive(Debug, Clone)] +pub struct ChangeMeta { + /// Lamport timestamp of the Change + pub lamport: Lamport, + /// The first Op id of the Change + pub id: ID, + /// [Unix time](https://en.wikipedia.org/wiki/Unix_time) + /// It is the number of seconds that have elapsed since 00:00:00 UTC on 1 January 1970. + pub timestamp: Timestamp, + /// The commit message of the change + pub message: Option>, + /// The dependencies of the first op of the change + pub deps: Frontiers, + /// The total op num inside this change + pub len: usize, +} + +impl PartialEq for ChangeMeta { + fn eq(&self, other: &Self) -> bool { + self.lamport == other.lamport && self.id == other.id + } +} + +impl Eq for ChangeMeta {} + +impl PartialOrd for ChangeMeta { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for ChangeMeta { + fn cmp(&self, other: &Self) -> Ordering { + (self.lamport + self.len as Lamport) + .cmp(&(other.lamport + other.len as Lamport)) + .then(self.id.peer.cmp(&other.id.peer)) + } +} + +impl HasLamport for ChangeMeta { + fn lamport(&self) -> loro_common::Lamport { + self.lamport + } +} + +impl HasLength for ChangeMeta { + fn content_len(&self) -> usize { + self.len + } +} + +impl ChangeMeta { + pub fn from_change(c: &Change) -> Self { + Self { + id: c.id(), + lamport: c.lamport(), + timestamp: c.timestamp(), + message: c.message().cloned(), + deps: c.deps().clone(), + len: c.len(), + } + } + + /// Get the commit message in &str + pub fn message(&self) -> &str { + match self.message.as_ref() { + Some(m) => m, + None => "", + } + } + + /// Get the commit timestamp in seconds since Unix epoch. + pub fn timestamp(&self) -> crate::change::Timestamp { + self.timestamp + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/configure.rs b/3rdparty/loro/crates/loro-internal/src/configure.rs new file mode 100644 index 0000000000..e4fe5e8612 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/configure.rs @@ -0,0 +1,291 @@ +use crate::sync::{Mutex, RwLock}; +use loro_common::ContainerID; +use rustc_hash::FxHashSet; + +pub use crate::container::richtext::config::{StyleConfig, StyleConfigMap}; +use crate::LoroDoc; +use std::sync::atomic::{AtomicBool, AtomicI64}; +use std::sync::Arc; + +#[derive(Clone, Debug)] +pub struct Configure { + pub(crate) text_style_config: Arc>, + record_timestamp: Arc, + pub(crate) merge_interval_in_s: Arc, + pub(crate) editable_detached_mode: Arc, + pub(crate) deleted_root_containers: Arc>>, + pub(crate) hide_empty_root_containers: Arc, +} + +impl LoroDoc { + pub(crate) fn set_config(&self, config: &Configure) { + self.config_text_style(config.text_style_config.read().clone()); + self.set_record_timestamp(config.record_timestamp()); + self.set_change_merge_interval(config.merge_interval()); + self.set_detached_editing(config.detached_editing()); + } +} + +impl Default for Configure { + fn default() -> Self { + Self { + text_style_config: Arc::new(RwLock::new(StyleConfigMap::default_rich_text_config())), + record_timestamp: Arc::new(AtomicBool::new(false)), + editable_detached_mode: Arc::new(AtomicBool::new(false)), + merge_interval_in_s: Arc::new(AtomicI64::new(1000)), + deleted_root_containers: Arc::new(Mutex::new(Default::default())), + hide_empty_root_containers: Arc::new(AtomicBool::new(false)), + } + } +} + +impl Configure { + pub fn fork(&self) -> Self { + Self { + text_style_config: Arc::new(RwLock::new(self.text_style_config.read().clone())), + record_timestamp: Arc::new(AtomicBool::new( + self.record_timestamp + .load(std::sync::atomic::Ordering::Relaxed), + )), + merge_interval_in_s: Arc::new(AtomicI64::new( + self.merge_interval_in_s + .load(std::sync::atomic::Ordering::Relaxed), + )), + editable_detached_mode: Arc::new(AtomicBool::new( + self.editable_detached_mode + .load(std::sync::atomic::Ordering::Relaxed), + )), + deleted_root_containers: Arc::new(Mutex::new( + self.deleted_root_containers.lock().clone(), + )), + hide_empty_root_containers: Arc::new(AtomicBool::new( + self.hide_empty_root_containers + .load(std::sync::atomic::Ordering::Relaxed), + )), + } + } + + pub fn text_style_config(&self) -> &Arc> { + &self.text_style_config + } + + pub fn record_timestamp(&self) -> bool { + self.record_timestamp + .load(std::sync::atomic::Ordering::Relaxed) + } + + pub fn set_record_timestamp(&self, record: bool) { + self.record_timestamp + .store(record, std::sync::atomic::Ordering::Relaxed); + } + + pub fn detached_editing(&self) -> bool { + self.editable_detached_mode + .load(std::sync::atomic::Ordering::Relaxed) + } + + pub fn set_detached_editing(&self, mode: bool) { + self.editable_detached_mode + .store(mode, std::sync::atomic::Ordering::Relaxed); + } + + pub fn merge_interval(&self) -> i64 { + self.merge_interval_in_s + .load(std::sync::atomic::Ordering::Relaxed) + } + + pub fn set_merge_interval(&self, interval: i64) { + self.merge_interval_in_s + .store(interval, std::sync::atomic::Ordering::Relaxed); + } + + pub fn set_hide_empty_root_containers(&self, hide: bool) { + self.hide_empty_root_containers + .store(hide, std::sync::atomic::Ordering::Relaxed); + } +} + +#[derive(Debug)] +pub struct DefaultRandom; + +#[cfg(test)] +use std::sync::atomic::AtomicU64; +#[cfg(test)] +static mut TEST_RANDOM: AtomicU64 = AtomicU64::new(0); + +impl SecureRandomGenerator for DefaultRandom { + fn fill_byte(&self, dest: &mut [u8]) { + #[cfg(not(test))] + getrandom::getrandom(dest).unwrap(); + + #[cfg(test)] + // SAFETY: this is only used in test + unsafe { + #[allow(static_mut_refs)] + let bytes = TEST_RANDOM + .fetch_add(1, std::sync::atomic::Ordering::Release) + .to_le_bytes(); + dest.copy_from_slice(&bytes[..dest.len()]); + } + } +} + +pub trait SecureRandomGenerator: Send + Sync { + fn fill_byte(&self, dest: &mut [u8]); + fn next_u64(&self) -> u64 { + let mut buf = [0u8; 8]; + self.fill_byte(&mut buf); + u64::from_le_bytes(buf) + } + + fn next_u32(&self) -> u32 { + let mut buf = [0u8; 4]; + self.fill_byte(&mut buf); + u32::from_le_bytes(buf) + } + + fn next_i64(&self) -> i64 { + let mut buf = [0u8; 8]; + self.fill_byte(&mut buf); + i64::from_le_bytes(buf) + } + + fn next_i32(&self) -> i32 { + let mut buf = [0u8; 4]; + self.fill_byte(&mut buf); + i32::from_le_bytes(buf) + } +} + +#[cfg(test)] +mod tests { + use std::sync::{atomic::Ordering, Mutex, OnceLock}; + + use loro_common::{ContainerID, ContainerType, InternalString}; + + use crate::container::richtext::ExpandType; + + use super::*; + + fn random_lock() -> &'static Mutex<()> { + static LOCK: OnceLock> = OnceLock::new(); + LOCK.get_or_init(|| Mutex::new(())) + } + + #[test] + fn configure_default_values_and_setters_match_the_public_contract() { + let config = Configure::default(); + + assert!(!config.record_timestamp()); + assert!(!config.detached_editing()); + assert_eq!(config.merge_interval(), 1000); + assert!(!config.hide_empty_root_containers.load(Ordering::Relaxed)); + assert!(config.deleted_root_containers.lock().is_empty()); + + let styles = config.text_style_config.read(); + assert_eq!( + styles.get(&InternalString::from("bold")), + Some(StyleConfig { + expand: ExpandType::After, + }) + ); + assert_eq!( + styles.get(&InternalString::from("italic")), + Some(StyleConfig { + expand: ExpandType::After, + }) + ); + assert_eq!( + styles.get(&InternalString::from("link")), + Some(StyleConfig { + expand: ExpandType::None, + }) + ); + assert_eq!(styles.get(&InternalString::from("missing")), None); + + config.set_record_timestamp(true); + config.set_detached_editing(true); + config.set_merge_interval(42); + config.set_hide_empty_root_containers(true); + + assert!(config.record_timestamp()); + assert!(config.detached_editing()); + assert_eq!(config.merge_interval(), 42); + assert!(config.hide_empty_root_containers.load(Ordering::Relaxed)); + } + + #[test] + fn configure_fork_copies_current_state_and_then_diverges() { + let config = Configure::default(); + config.set_record_timestamp(true); + config.set_detached_editing(true); + config.set_merge_interval(25); + config.set_hide_empty_root_containers(true); + config + .deleted_root_containers + .lock() + .insert(ContainerID::Root { + name: InternalString::from("root"), + container_type: ContainerType::Map, + }); + config.text_style_config.write().insert( + InternalString::from("custom"), + StyleConfig { + expand: ExpandType::None, + }, + ); + + let forked = config.fork(); + + assert!(forked.record_timestamp()); + assert!(forked.detached_editing()); + assert_eq!(forked.merge_interval(), 25); + assert!(forked.hide_empty_root_containers.load(Ordering::Relaxed)); + assert_eq!(forked.deleted_root_containers.lock().len(), 1); + assert_eq!( + forked + .text_style_config + .read() + .get(&InternalString::from("custom")), + Some(StyleConfig { + expand: ExpandType::None, + }) + ); + + config.set_record_timestamp(false); + config.set_detached_editing(false); + config.set_merge_interval(99); + config.set_hide_empty_root_containers(false); + config.deleted_root_containers.lock().clear(); + config + .text_style_config + .write() + .insert(InternalString::from("fork-only"), StyleConfig::default()); + + assert!(forked.record_timestamp()); + assert!(forked.detached_editing()); + assert_eq!(forked.merge_interval(), 25); + assert!(forked.hide_empty_root_containers.load(Ordering::Relaxed)); + assert_eq!(forked.deleted_root_containers.lock().len(), 1); + assert!(forked + .text_style_config + .read() + .get(&InternalString::from("fork-only")) + .is_none()); + } + + #[test] + fn default_random_test_mode_uses_the_incrementing_counter_for_integer_helpers() { + let _guard = random_lock().lock().unwrap(); + let random = DefaultRandom; + + let a = random.next_u64(); + let b = random.next_u32(); + let c = random.next_i64(); + let d = random.next_i32(); + + assert_eq!(b as u64, (a + 1) as u32 as u64); + assert_eq!(c as u64, a + 2); + assert_eq!(d as u64, (a + 3) as u32 as u64); + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/container.rs b/3rdparty/loro/crates/loro-internal/src/container.rs new file mode 100644 index 0000000000..3e8ea7df8b --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/container.rs @@ -0,0 +1,314 @@ +//! CRDT [Container]. Each container may have different CRDT type [ContainerType]. +//! Each [Op] has an associated container. It's the [Container]'s responsibility to +//! calculate the state from the [Op]s. +//! +//! Every [Container] can take a [Snapshot], which contains [crate::LoroValue] that describes the state. +//! +use crate::{arena::SharedArena, InternalString, ID}; + +pub mod list; +pub mod map; +pub mod richtext; +pub mod tree; +pub mod idx { + use super::super::ContainerType; + + /// Inner representation for ContainerID. + /// It contains the unique index for the container and the type of the container. + /// It uses top 4 bits to represent the type of the container. + /// + /// It's only used inside this crate and should not be exposed to the user. + /// + /// TODO: make this type private in this crate only + /// + /// + // During a transaction, we may create some containers which are deleted later. And these containers also need a unique ContainerIdx. + // So when we encode snapshot, we need to sort the containers by ContainerIdx and change the `container` of ops to the index of containers. + // An empty store decodes the snapshot, it will create these containers in a sequence of natural numbers so that containers and ops can correspond one-to-one + // + // TODO: PERF: use NonZeroU32 to save memory + #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Hash)] + pub struct ContainerIdx(u32); + + impl std::fmt::Debug for ContainerIdx { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "ContainerIdx({} {})", self.get_type(), self.to_index()) + } + } + + impl std::fmt::Display for ContainerIdx { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "ContainerIdx({} {})", self.get_type(), self.to_index()) + } + } + + impl ContainerIdx { + pub(crate) const TYPE_MASK: u32 = 0b11111 << 27; + pub(crate) const INDEX_MASK: u32 = !Self::TYPE_MASK; + + #[allow(unused)] + pub(crate) fn get_type(self) -> ContainerType { + let a = (self.0 & Self::TYPE_MASK) >> 27; + if self.is_unknown() { + Self::parse_unknown_type(a) + } else { + ContainerType::try_from_u8(a as u8).unwrap() + } + } + + #[allow(unused)] + pub(crate) fn to_index(self) -> u32 { + self.0 & Self::INDEX_MASK + } + + pub(crate) fn from_index_and_type(index: u32, container_type: ContainerType) -> Self { + let prefix: u32 = if matches!(container_type, ContainerType::Unknown(_)) { + Self::unknown_to_prefix(container_type) + } else { + container_type.to_u8() as u32 + } << 27; + + Self(prefix | index) + } + + pub(crate) fn is_unknown(&self) -> bool { + self.0 >> 31 == 1 + } + + // The type_value is >>27 first, so it's 5 bits. + // we want to get the last 4 bits. so we use 0b1111 to get the last 4 bits. + fn parse_unknown_type(type_value: u32) -> ContainerType { + ContainerType::Unknown((type_value & 0b1111) as u8) + } + + // we use the top 5 bits to represent the type of the container. + // the first bit is whether it's an unknown type. + // So when we convert an unknown type to a prefix, we need to set the first bit to 1. + fn unknown_to_prefix(c: ContainerType) -> u32 { + if let ContainerType::Unknown(c) = c { + (0b10000 | c) as u32 + } else { + unreachable!() + } + } + } +} +use idx::ContainerIdx; + +pub use loro_common::ContainerType; + +pub use loro_common::ContainerID; + +#[derive(Debug)] +pub enum ContainerIdRaw { + Root { name: InternalString }, + Normal { id: ID }, +} + +pub trait IntoContainerId { + fn into_container_id(self, arena: &SharedArena, kind: ContainerType) -> ContainerID; +} + +impl IntoContainerId for String { + fn into_container_id(self, _arena: &SharedArena, kind: ContainerType) -> ContainerID { + ContainerID::Root { + name: InternalString::from(self.as_str()), + container_type: kind, + } + } +} + +impl IntoContainerId for &str { + fn into_container_id(self, _arena: &SharedArena, kind: ContainerType) -> ContainerID { + ContainerID::Root { + name: InternalString::from(self), + container_type: kind, + } + } +} + +impl IntoContainerId for ContainerID { + fn into_container_id(self, _arena: &SharedArena, _kind: ContainerType) -> ContainerID { + self + } +} + +impl IntoContainerId for &ContainerID { + fn into_container_id(self, _arena: &SharedArena, _kind: ContainerType) -> ContainerID { + self.clone() + } +} + +impl IntoContainerId for ContainerIdx { + fn into_container_id(self, arena: &SharedArena, kind: ContainerType) -> ContainerID { + assert_eq!(self.get_type(), kind); + arena.get_container_id(self).unwrap() + } +} + +impl IntoContainerId for &ContainerIdx { + fn into_container_id(self, arena: &SharedArena, kind: ContainerType) -> ContainerID { + assert_eq!(self.get_type(), kind); + arena.get_container_id(*self).unwrap() + } +} + +impl From for ContainerIdRaw { + fn from(value: String) -> Self { + ContainerIdRaw::Root { name: value.into() } + } +} + +impl<'a> From<&'a str> for ContainerIdRaw { + fn from(value: &'a str) -> Self { + ContainerIdRaw::Root { name: value.into() } + } +} + +impl From<&ContainerID> for ContainerIdRaw { + fn from(id: &ContainerID) -> Self { + match id { + ContainerID::Root { name, .. } => ContainerIdRaw::Root { name: name.clone() }, + ContainerID::Normal { peer, counter, .. } => ContainerIdRaw::Normal { + id: ID::new(*peer, *counter), + }, + } + } +} + +impl From for ContainerIdRaw { + fn from(id: ContainerID) -> Self { + match id { + ContainerID::Root { name, .. } => ContainerIdRaw::Root { name }, + ContainerID::Normal { peer, counter, .. } => ContainerIdRaw::Normal { + id: ID::new(peer, counter), + }, + } + } +} + +impl ContainerIdRaw { + pub fn with_type(self, container_type: ContainerType) -> ContainerID { + match self { + ContainerIdRaw::Root { name } => ContainerID::Root { + name, + container_type, + }, + ContainerIdRaw::Normal { id } => ContainerID::Normal { + peer: id.peer, + counter: id.counter, + container_type, + }, + } + } +} + +#[cfg(test)] +mod test { + use super::idx::ContainerIdx; + use super::*; + + #[test] + fn container_id_convert() { + let container_id = ContainerID::new_normal(ID::new(12, 12), ContainerType::List); + let s = container_id.to_string(); + assert_eq!(s, "cid:12@12:List"); + let actual = ContainerID::try_from(s.as_str()).unwrap(); + assert_eq!(actual, container_id); + + let container_id = ContainerID::new_root("123", ContainerType::Map); + let s = container_id.to_string(); + assert_eq!(s, "cid:root-123:Map"); + let actual = ContainerID::try_from(s.as_str()).unwrap(); + assert_eq!(actual, container_id); + + let container_id = ContainerID::new_root("kkk", ContainerType::Text); + let s = container_id.to_string(); + assert_eq!(s, "cid:root-kkk:Text"); + let actual = ContainerID::try_from(s.as_str()).unwrap(); + assert_eq!(actual, container_id); + } + + #[test] + fn container_idx_preserves_type_bits_and_index_bits() { + let idx = ContainerIdx::from_index_and_type(42, ContainerType::Text); + assert_eq!(idx.get_type(), ContainerType::Text); + assert_eq!(idx.to_index(), 42); + assert!(!idx.is_unknown()); + assert_eq!(idx.to_string(), "ContainerIdx(Text 42)"); + assert_eq!(format!("{idx:?}"), "ContainerIdx(Text 42)"); + + let max_index = + ContainerIdx::from_index_and_type(ContainerIdx::INDEX_MASK, ContainerType::MovableList); + assert_eq!(max_index.get_type(), ContainerType::MovableList); + assert_eq!(max_index.to_index(), ContainerIdx::INDEX_MASK); + + let unknown = ContainerIdx::from_index_and_type(7, ContainerType::Unknown(13)); + assert!(unknown.is_unknown()); + assert_eq!(unknown.get_type(), ContainerType::Unknown(13)); + assert_eq!(unknown.to_index(), 7); + assert_eq!(unknown.to_string(), "ContainerIdx(Unknown(13) 7)"); + } + + #[test] + fn container_id_raw_reapplies_type_for_root_and_normal_ids() { + let root = ContainerIdRaw::from("notes").with_type(ContainerType::Map); + assert_eq!(root, ContainerID::new_root("notes", ContainerType::Map)); + + let normal_id = ID::new(7, 11); + let normal = ContainerIdRaw::from(ContainerID::new_normal(normal_id, ContainerType::List)) + .with_type(ContainerType::Text); + assert_eq!( + normal, + ContainerID::new_normal(normal_id, ContainerType::Text) + ); + + let root_again = ContainerIdRaw::from(&root).with_type(ContainerType::Tree); + assert_eq!( + root_again, + ContainerID::new_root("notes", ContainerType::Tree) + ); + } + + #[test] + fn into_container_id_uses_root_names_ids_and_arena_indexes() { + let arena = SharedArena::new(); + assert_eq!( + "workspace".into_container_id(&arena, ContainerType::Map), + ContainerID::new_root("workspace", ContainerType::Map) + ); + assert_eq!( + String::from("text").into_container_id(&arena, ContainerType::Text), + ContainerID::new_root("text", ContainerType::Text) + ); + + let normal = ContainerID::new_normal(ID::new(1, 2), ContainerType::List); + assert_eq!( + (&normal).into_container_id(&arena, ContainerType::Map), + normal + ); + assert_eq!( + normal + .clone() + .into_container_id(&arena, ContainerType::Text), + normal + ); + + let idx = arena.register_container(&normal); + assert_eq!(idx.into_container_id(&arena, ContainerType::List), normal); + assert_eq!( + (&idx).into_container_id(&arena, ContainerType::List), + normal + ); + } + + #[test] + #[should_panic(expected = "assertion `left == right` failed")] + fn into_container_id_rejects_mismatched_index_type() { + let arena = SharedArena::new(); + let id = ContainerID::new_root("items", ContainerType::List); + let idx = arena.register_container(&id); + + let _ = idx.into_container_id(&arena, ContainerType::Map); + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/container/list/list_op.rs b/3rdparty/loro/crates/loro-internal/src/container/list/list_op.rs new file mode 100644 index 0000000000..4bee2bf126 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/container/list/list_op.rs @@ -0,0 +1,803 @@ +use std::ops::Range; + +use append_only_bytes::BytesSlice; +use enum_as_inner::EnumAsInner; +use loro_common::{ContainerType, HasId, HasIdSpan, IdLp, LoroValue, ID}; +use rle::{HasLength, Mergable, Sliceable}; +use serde::{Deserialize, Serialize}; + +use crate::{ + container::richtext::richtext_state::unicode_to_utf8_index, + container::richtext::TextStyleInfoFlag, + op::{ListSlice, SliceRange}, + InternalString, +}; + +/// `len` and `pos` is measured in unicode char for text. +// Note: It will be encoded into binary format, so the order of its fields should not be changed. +#[derive(EnumAsInner, Debug, Clone, PartialEq, Serialize, Deserialize)] +pub enum ListOp<'a> { + Insert { + slice: ListSlice<'a>, + pos: usize, + }, + Delete(DeleteSpanWithId), + Move { + from: u32, + to: u32, + elem_id: IdLp, + }, + Set { + elem_id: IdLp, + value: LoroValue, + }, + /// StyleStart and StyleEnd must be paired because the end of a style must take an OpID position. + StyleStart { + start: u32, + end: u32, + key: InternalString, + info: TextStyleInfoFlag, + value: LoroValue, + }, + StyleEnd, +} + +#[derive(EnumAsInner, Debug, Clone)] +pub enum InnerListOp { + // TODO: this is only used for list now? We should rename it to InsertList + Insert { + slice: SliceRange, + pos: usize, + }, + // Note: len may not equal to slice.len() because for text len is unicode len while the slice + // is utf8 bytes. + InsertText { + slice: BytesSlice, + unicode_start: u32, + unicode_len: u32, + pos: u32, + }, + Delete(DeleteSpanWithId), + Move { + from: u32, + /// Element id + elem_id: IdLp, + to: u32, + }, + Set { + elem_id: IdLp, + value: LoroValue, + }, + /// StyleStart and StyleEnd must be paired. + /// The next op of StyleStart must be StyleEnd. + StyleStart { + start: u32, + end: u32, + key: InternalString, + value: LoroValue, + info: TextStyleInfoFlag, + }, + StyleEnd, +} + +impl ListOp<'_> { + pub fn new_del(id_start: ID, pos: usize, len: usize) -> Self { + assert!(len != 0); + Self::Delete(DeleteSpanWithId::new(id_start, pos as isize, len as isize)) + } +} + +impl InnerListOp { + pub fn new_del(id: ID, pos: usize, len: isize) -> Self { + assert!(len != 0); + Self::Delete(DeleteSpanWithId { + id_start: id, + span: DeleteSpan { + pos: pos as isize, + signed_len: len, + }, + }) + } + + pub fn new_insert(slice: Range, pos: usize) -> Self { + Self::Insert { + slice: SliceRange(slice), + pos, + } + } + + pub(crate) fn estimate_storage_size(&self, container_type: ContainerType) -> usize { + match self { + InnerListOp::Insert { slice, .. } => match container_type { + ContainerType::MovableList | ContainerType::List => 4 * slice.atom_len(), + ContainerType::Text => slice.atom_len(), + _ => unreachable!(), + }, + InnerListOp::InsertText { slice, .. } => slice.len(), + InnerListOp::Delete(..) => 8, + InnerListOp::Move { .. } => 8, + InnerListOp::Set { .. } => 7, + InnerListOp::StyleStart { .. } => 10, + InnerListOp::StyleEnd => 1, + } + } +} + +impl HasLength for DeleteSpan { + fn content_len(&self) -> usize { + self.signed_len.unsigned_abs() + } +} + +/// Delete span that the initial id is `id_start`. +/// +/// This span may be reversed. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct DeleteSpanWithId { + /// This is the target id with the smallest counter no matter whether the span is reversed. + /// So it's always the id of the leftmost element of the target span. + pub id_start: ID, + /// The deleted position span + pub span: DeleteSpan, +} + +impl DeleteSpanWithId { + pub fn new(id_start: ID, pos: isize, len: isize) -> Self { + debug_assert!(len != 0); + Self { + id_start, + span: DeleteSpan { + pos, + signed_len: len, + }, + } + } + + #[inline] + pub fn start(&self) -> isize { + self.span.start() + } + + #[inline] + pub fn end(&self) -> isize { + self.span.end() + } + + #[inline] + pub fn last(&self) -> isize { + self.span.last() + } + + #[inline] + pub fn is_reversed(&self) -> bool { + self.span.is_reversed() + } +} + +impl HasLength for DeleteSpanWithId { + fn content_len(&self) -> usize { + self.span.content_len() + } +} + +impl HasId for DeleteSpanWithId { + fn id_start(&self) -> ID { + self.id_start + } +} + +impl Mergable for DeleteSpanWithId { + /// If two spans are mergeable, their ids should be continuous. + /// LHS's end id should be equal to RHS's start id. + /// But their spans may be in a reversed order. + fn is_mergable(&self, rhs: &Self, _conf: &()) -> bool + where + Self: Sized, + { + let this = self.span; + let other = rhs.span; + // merge continuous deletions: + // note that the previous deletions will affect the position of the later deletions + match (self.span.bidirectional(), rhs.span.bidirectional()) { + (true, true) => { + (this.pos == other.pos && self.id_start.inc(1) == rhs.id_start) + || (this.pos == other.pos + 1 && self.id_start == rhs.id_start.inc(1)) + } + (true, false) => { + if this.pos == other.prev_pos() { + if other.signed_len > 0 { + self.id_start.inc(1) == rhs.id_start + } else { + self.id_start == rhs.id_end() + } + } else { + false + } + } + (false, true) => { + if this.next_pos() == other.pos { + if this.signed_len > 0 { + self.id_end() == rhs.id_start + } else { + self.id_start == rhs.id_start.inc(1) + } + } else { + false + } + } + (false, false) => { + if this.next_pos() == other.pos && this.direction() == other.direction() { + if self.span.signed_len > 0 { + self.id_end() == rhs.id_start + } else { + self.id_start == rhs.id_end() + } + } else { + false + } + } + } + } + + fn merge(&mut self, rhs: &Self, _conf: &()) + where + Self: Sized, + { + self.id_start.counter = rhs.id_start.counter.min(self.id_start.counter); + self.span.merge(&rhs.span, &()) + } +} + +impl Sliceable for DeleteSpanWithId { + fn slice(&self, from: usize, to: usize) -> Self { + Self { + id_start: if self.span.signed_len > 0 { + self.id_start.inc(from as i32) + } else { + // If the span is reversed, the id_start should be affected by `to` + // + // Example: + // + // a b c + // - - - <-- deletions happen backward + // 0 1 2 <-- counter of the IDs + // ↑ + // id_start + // + // If from=1, to=2 + // a b c + // - - - <-- deletions happen backward + // 0 1 2 <-- counter of the IDs + // ↑ + // id_start + self.id_start.inc((self.atom_len() - to) as i32) + }, + span: self.span.slice(from, to), + } + } +} + +/// `len` can be negative so that we can merge text deletions efficiently. +/// It looks like [crate::span::CounterSpan], but how should they merge ([Mergable] impl) and slice ([Sliceable] impl) are very different +/// +/// len cannot be zero; +/// +/// pos: 5, len: -3 eq a range of (2, 5] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +// Note: It will be encoded into binary format, so the order of its fields should not be changed. +pub struct DeleteSpan { + pub pos: isize, + pub signed_len: isize, +} + +impl DeleteSpan { + pub fn new(pos: isize, len: isize) -> Self { + debug_assert!(len != 0); + Self { + pos, + signed_len: len, + } + } + + #[inline(always)] + pub fn start(&self) -> isize { + if self.signed_len > 0 { + self.pos + } else { + self.pos + 1 + self.signed_len + } + } + + #[inline(always)] + pub fn last(&self) -> isize { + if self.signed_len > 0 { + self.pos + self.signed_len - 1 + } else { + self.pos + } + } + + #[inline(always)] + pub fn end(&self) -> isize { + if self.signed_len > 0 { + self.pos + self.signed_len + } else { + self.pos + 1 + } + } + + #[inline(always)] + pub fn to_range(self) -> Range { + self.start()..self.end() + } + + #[inline(always)] + pub fn to_urange(self) -> Range { + self.start() as usize..self.end() as usize + } + + #[inline(always)] + pub fn bidirectional(&self) -> bool { + self.signed_len.abs() == 1 + } + + #[inline(always)] + pub fn is_reversed(&self) -> bool { + self.signed_len < 0 + } + + #[inline(always)] + pub fn direction(&self) -> isize { + if self.signed_len > 0 { + 1 + } else { + -1 + } + } + + #[inline(always)] + pub fn next_pos(&self) -> isize { + if self.signed_len > 0 { + self.start() + } else { + self.start() - 1 + } + } + + #[inline(always)] + pub fn prev_pos(&self) -> isize { + if self.signed_len > 0 { + self.pos + } else { + self.end() + } + } + + pub fn len(&self) -> usize { + self.signed_len.unsigned_abs() + } +} + +impl Mergable for DeleteSpan { + fn is_mergable(&self, other: &Self, _conf: &()) -> bool + where + Self: Sized, + { + // merge continuous deletions: + // note that the previous deletions will affect the position of the later deletions + match (self.bidirectional(), other.bidirectional()) { + (true, true) => self.pos == other.pos || self.pos == other.pos + 1, + (true, false) => self.pos == other.prev_pos(), + (false, true) => self.next_pos() == other.pos, + (false, false) => self.next_pos() == other.pos && self.direction() == other.direction(), + } + } + + fn merge(&mut self, other: &Self, _conf: &()) + where + Self: Sized, + { + match (self.bidirectional(), other.bidirectional()) { + (true, true) => { + if self.pos == other.pos { + self.signed_len = 2; + } else if self.pos == other.pos + 1 { + self.signed_len = -2; + } else { + unreachable!() + } + } + (true, false) => { + assert!(self.pos == other.prev_pos()); + self.signed_len = other.signed_len + other.direction(); + } + (false, true) => { + assert!(self.next_pos() == other.pos); + self.signed_len += self.direction(); + } + (false, false) => { + assert!(self.next_pos() == other.pos && self.direction() == other.direction()); + self.signed_len += other.signed_len; + } + } + } +} + +impl Sliceable for DeleteSpan { + fn slice(&self, from: usize, to: usize) -> Self { + if self.signed_len > 0 { + Self::new(self.pos, to as isize - from as isize) + } else { + Self::new(self.pos - from as isize, from as isize - to as isize) + } + } +} + +impl Mergable for ListOp<'_> { + fn is_mergable(&self, _other: &Self, _conf: &()) -> bool + where + Self: Sized, + { + match self { + ListOp::Insert { pos, slice } => match _other { + ListOp::Insert { + pos: other_pos, + slice: other_slice, + } => pos + slice.content_len() == *other_pos && slice.is_mergable(other_slice, &()), + _ => false, + }, + &ListOp::Delete(span) => match _other { + ListOp::Delete(other_span) => span.is_mergable(other_span, &()), + _ => false, + }, + ListOp::StyleStart { .. } + | ListOp::StyleEnd { .. } + | ListOp::Move { .. } + | ListOp::Set { .. } => false, + } + } + + fn merge(&mut self, _other: &Self, _conf: &()) + where + Self: Sized, + { + match self { + ListOp::Insert { slice, .. } => match _other { + ListOp::Insert { + slice: other_slice, .. + } => { + slice.merge(other_slice, &()); + } + _ => unreachable!(), + }, + ListOp::Delete(span) => match _other { + ListOp::Delete(other_span) => span.merge(other_span, &()), + _ => unreachable!(), + }, + ListOp::StyleStart { .. } + | ListOp::StyleEnd { .. } + | ListOp::Move { .. } + | ListOp::Set { .. } => { + unreachable!() + } + } + } +} + +impl HasLength for ListOp<'_> { + fn content_len(&self) -> usize { + match self { + ListOp::Insert { slice, .. } => slice.content_len(), + ListOp::Delete(span) => span.atom_len(), + ListOp::StyleStart { .. } + | ListOp::StyleEnd { .. } + | ListOp::Move { .. } + | ListOp::Set { .. } => 1, + } + } +} + +impl Sliceable for ListOp<'_> { + fn slice(&self, from: usize, to: usize) -> Self { + match self { + ListOp::Insert { slice, pos } => ListOp::Insert { + slice: slice.slice(from, to), + pos: *pos + from, + }, + ListOp::Delete(span) => ListOp::Delete(span.slice(from, to)), + a @ (ListOp::StyleStart { .. } + | ListOp::StyleEnd { .. } + | ListOp::Move { .. } + | ListOp::Set { .. }) => a.clone(), + } + } +} + +impl Mergable for InnerListOp { + fn is_mergable(&self, other: &Self, _conf: &()) -> bool + where + Self: Sized, + { + match (self, other) { + ( + InnerListOp::Insert { pos, slice, .. }, + InnerListOp::Insert { + pos: other_pos, + slice: other_slice, + .. + }, + ) => pos + slice.content_len() == *other_pos && slice.is_mergable(other_slice, &()), + (InnerListOp::Delete(span), InnerListOp::Delete(other_span)) => { + span.is_mergable(other_span, &()) + } + ( + InnerListOp::InsertText { + unicode_start, + slice, + pos, + unicode_len: len, + }, + InnerListOp::InsertText { + slice: other_slice, + pos: other_pos, + unicode_start: other_unicode_start, + unicode_len: _, + }, + ) => { + pos + len == *other_pos + && slice.can_merge(other_slice) + && unicode_start + len == *other_unicode_start + } + _ => false, + } + } + + fn merge(&mut self, other: &Self, _conf: &()) + where + Self: Sized, + { + match (self, other) { + ( + InnerListOp::Insert { slice, .. }, + InnerListOp::Insert { + slice: other_slice, .. + }, + ) => { + slice.merge(other_slice, &()); + } + (InnerListOp::Delete(span), InnerListOp::Delete(other_span)) => { + span.merge(other_span, &()) + } + ( + InnerListOp::InsertText { + slice, + unicode_len: len, + .. + }, + InnerListOp::InsertText { + slice: other_slice, + unicode_len: other_len, + .. + }, + ) => { + slice.merge(other_slice, &()); + *len += *other_len; + } + _ => unreachable!(), + } + } +} + +impl HasLength for InnerListOp { + fn content_len(&self) -> usize { + match self { + InnerListOp::Insert { slice, .. } => slice.content_len(), + InnerListOp::InsertText { + unicode_len: len, .. + } => *len as usize, + InnerListOp::Delete(span) => span.atom_len(), + InnerListOp::StyleStart { .. } + | InnerListOp::StyleEnd + | InnerListOp::Move { .. } + | InnerListOp::Set { .. } => 1, + } + } +} + +impl Sliceable for InnerListOp { + fn slice(&self, from: usize, to: usize) -> Self { + match self { + InnerListOp::Insert { slice, pos } => InnerListOp::Insert { + slice: slice.slice(from, to), + pos: *pos + from, + }, + InnerListOp::InsertText { + slice, + unicode_start, + unicode_len, + pos, + } => InnerListOp::InsertText { + slice: { + // SAFETY: we know it's a valid utf8 string + let s = unsafe { std::str::from_utf8_unchecked(slice) }; + let total_unicode_len = *unicode_len as usize; + let total_bytes_len = slice.len(); + let (from_byte, to_byte) = if total_bytes_len == total_unicode_len { + // ASCII fast path: unicode index equals byte offset. + (from, to) + } else { + let from_byte = if from == 0 { + 0 + } else { + unicode_to_utf8_index(s, from).expect("unicode index should be valid") + }; + let to_byte = if to == total_unicode_len { + total_bytes_len + } else { + unicode_to_utf8_index(s, to).expect("unicode index should be valid") + }; + (from_byte, to_byte) + }; + + slice.slice(from_byte, to_byte) + }, + unicode_start: *unicode_start + from as u32, + unicode_len: (to - from) as u32, + pos: *pos + from as u32, + }, + InnerListOp::Delete(span) => InnerListOp::Delete(span.slice(from, to)), + InnerListOp::StyleStart { .. } + | InnerListOp::StyleEnd + | InnerListOp::Move { .. } + | InnerListOp::Set { .. } => { + assert!(from == 0 && to == 1); + self.clone() + } + } + } +} + +#[cfg(test)] +mod test { + use append_only_bytes::BytesSlice; + use loro_common::ID; + use rle::{HasLength as _, Mergable, Sliceable}; + use std::time::Instant; + + use crate::{container::list::list_op::DeleteSpanWithId, op::ListSlice}; + + use super::{DeleteSpan, InnerListOp, ListOp}; + + #[test] + fn fix_fields_order() { + let list_op = vec![ + ListOp::Insert { + pos: 0, + slice: ListSlice::from_borrowed_str(""), + }, + ListOp::Delete(DeleteSpanWithId::new(ID::new(0, 0), 0, 3)), + ]; + let actual = postcard::to_allocvec(&list_op).unwrap(); + let list_op_buf = vec![2, 0, 1, 0, 0, 0, 1, 0, 0, 0, 6]; + assert_eq!(&actual, &list_op_buf); + assert_eq!( + postcard::from_bytes::>(&list_op_buf).unwrap(), + list_op + ); + + let delete_span = DeleteSpan { + pos: 0, + signed_len: 3, + }; + let delete_span_buf = vec![0, 6]; + assert_eq!( + postcard::from_bytes::(&delete_span_buf).unwrap(), + delete_span + ); + } + + #[test] + fn test_del_span_merge_slice() { + let a = DeleteSpan::new(0, 100); + let mut b = a.slice(0, 1); + let c = a.slice(1, 100); + b.merge(&c, &()); + assert_eq!(b, a); + + // reverse + let a = DeleteSpan::new(99, -100); + let mut b = a.slice(0, 1); + let c = a.slice(1, 100); + b.merge(&c, &()); + assert_eq!(b, a); + + // merge bidirectional + let mut a = DeleteSpan::new(1, -1); + let b = DeleteSpan::new(1, -1); + assert!(a.is_mergable(&b, &())); + a.merge(&b, &()); + assert_eq!(a, DeleteSpan::new(1, 2)); + + let mut a = DeleteSpan::new(1, -1); + let b = DeleteSpan::new(0, -1); + assert_eq!(b.to_range(), 0..1); + assert!(a.is_mergable(&b, &())); + a.merge(&b, &()); + assert_eq!(a, DeleteSpan::new(1, -2)); + + // not merging + let a = DeleteSpan::new(4, 1); + let b = DeleteSpan::new(5, 2); + assert!(!a.is_mergable(&b, &())); + + // next/prev span + let a = DeleteSpan::new(6, -2); + assert_eq!(a.next_pos(), 4); + assert_eq!(a.prev_pos(), 7); + let a = DeleteSpan::new(6, 2); + assert_eq!(a.next_pos(), 6); + assert_eq!(a.prev_pos(), 6); + assert!(a.slice(0, 1).is_mergable(&a.slice(1, 2), &())); + + // neg merge + let mut a = DeleteSpan::new(1, 1); + let b = DeleteSpan::new(0, 1); + a.merge(&b, &()); + assert_eq!(a, DeleteSpan::new(1, -2)); + assert_eq!(a.slice(0, 1), DeleteSpan::new(1, -1)); + assert_eq!(a.slice(1, 2), DeleteSpan::new(0, -1)); + assert_eq!(a.slice(0, 1).to_range(), 1..2); + assert_eq!(a.slice(1, 2).to_range(), 0..1); + } + + #[test] + fn mergeable() { + let a = DeleteSpan::new(14852, 1); + let mut a_with_id = DeleteSpanWithId { + id_start: ID::new(0, 9), + span: a, + }; + let b = DeleteSpan::new(14851, 1); + let b_with_id = DeleteSpanWithId { + id_start: ID::new(0, 8), + span: b, + }; + assert!(a_with_id.is_mergable(&b_with_id, &())); + a_with_id.merge(&b_with_id, &()); + assert!(a_with_id.span.signed_len == -2); + } + + #[test] + #[ignore] + fn perf_insert_text_slice_split_suffix() { + // Run with: + // cargo test -p loro-internal perf_insert_text_slice_split_suffix -- --ignored --nocapture + const TEXT_LEN: usize = 2 * 1024 * 1024; + const CHUNK_UNICODE_LEN: usize = 4096; + + let text = "a".repeat(TEXT_LEN); + let slice = BytesSlice::from_bytes(text.as_bytes()); + let mut op = InnerListOp::InsertText { + slice, + unicode_start: 0, + unicode_len: TEXT_LEN as u32, + pos: 0, + }; + + let start = Instant::now(); + let mut total = 0usize; + while op.content_len() > CHUNK_UNICODE_LEN { + let end = CHUNK_UNICODE_LEN.min(op.content_len()); + let _prefix = op.slice(0, end); + op = op.slice(end, op.content_len()); + total += end; + } + total += op.content_len(); + let elapsed = start.elapsed(); + assert_eq!(total, TEXT_LEN); + println!( + "perf_insert_text_slice_split_suffix: text_len={}, chunk_unicode_len={}, elapsed={:?}", + TEXT_LEN, CHUNK_UNICODE_LEN, elapsed + ); + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/container/list/mod.rs b/3rdparty/loro/crates/loro-internal/src/container/list/mod.rs new file mode 100644 index 0000000000..1da3d6262e --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/container/list/mod.rs @@ -0,0 +1 @@ +pub(crate) mod list_op; diff --git a/3rdparty/loro/crates/loro-internal/src/container/map/map_content.rs b/3rdparty/loro/crates/loro-internal/src/container/map/map_content.rs new file mode 100644 index 0000000000..993718ad57 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/container/map/map_content.rs @@ -0,0 +1,42 @@ +use rle::{HasLength, Mergable, Sliceable}; +use serde::{Deserialize, Serialize}; + +use crate::{InternalString, LoroValue}; + +// Note: It will be encoded into binary format, so the order of its fields should not be changed. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct MapSet { + pub(crate) key: InternalString, + // the key is deleted if value is None + pub(crate) value: Option, +} + +impl Mergable for MapSet {} +impl Sliceable for MapSet { + fn slice(&self, from: usize, to: usize) -> Self { + assert!(from == 0 && to == 1); + self.clone() + } +} +impl HasLength for MapSet { + fn content_len(&self) -> usize { + 1 + } +} + +#[cfg(test)] +mod test { + use super::MapSet; + #[test] + fn fix_fields_order() { + let map_set = vec![MapSet { + key: "key".to_string().into(), + value: Some("value".to_string().into()), + }]; + let map_set_buf = vec![1, 3, 107, 101, 121, 1, 4, 5, 118, 97, 108, 117, 101]; + assert_eq!( + postcard::from_bytes::>(&map_set_buf).unwrap(), + map_set + ); + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/container/map/mod.rs b/3rdparty/loro/crates/loro-internal/src/container/map/mod.rs new file mode 100644 index 0000000000..1a66786541 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/container/map/mod.rs @@ -0,0 +1,3 @@ +mod map_content; + +pub(crate) use map_content::MapSet; diff --git a/3rdparty/loro/crates/loro-internal/src/container/richtext.rs b/3rdparty/loro/crates/loro-internal/src/container/richtext.rs new file mode 100644 index 0000000000..ffe4aa3390 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/container/richtext.rs @@ -0,0 +1,300 @@ +//! # Index +//! +//! There are several types of indexes: +//! +//! - Unicode index: the index of a unicode code point in the text. +//! - Entity index: unicode index + style anchor index. Each unicode code point or style anchor is an entity. +//! - Utf16 index +//! +//! In [crate::op::Op], we always use entity index to persist richtext ops. +//! +//! The users of this type can only operate on unicode index or utf16 index, but calculated entity index will be provided. + +pub(crate) mod config; +mod fugue_span; +pub(crate) mod richtext_state; +pub(crate) mod str_slice; +mod style_range_map; +#[cfg(all(test, miri))] +mod tinyvec; +mod tracker; + +use crate::{change::Lamport, delta::StyleMeta, utils::string_slice::StringSlice, InternalString}; +use fugue_span::*; +use loro_common::{Counter, IdFull, IdLp, LoroValue, PeerID, ID}; +use serde::{Deserialize, Serialize}; +use std::fmt::Debug; + +pub(crate) use fugue_span::{RichtextChunk, RichtextChunkValue}; +pub(crate) use richtext_state::RichtextState; +pub(crate) use style_range_map::Styles; +pub(crate) use tracker::{CrdtRopeDelta, Tracker as RichtextTracker}; + +/// This is the data structure that represents a span of rich text. +/// It's used to communicate with the frontend. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct RichtextSpan { + pub text: StringSlice, + pub attributes: StyleMeta, +} + +/// This is used to communicate with the frontend. +#[derive(Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +pub struct Style { + pub key: InternalString, + pub data: LoroValue, +} + +// TODO: change visibility back to crate after #116 is done +#[derive(Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +pub struct StyleOp { + pub(crate) lamport: Lamport, + pub(crate) peer: PeerID, + pub(crate) cnt: Counter, + pub(crate) key: InternalString, + pub(crate) value: LoroValue, + pub(crate) info: TextStyleInfoFlag, +} + +#[derive(Debug, Hash, Eq, PartialEq, Clone, Serialize, Deserialize)] +pub(crate) enum StyleKey { + Key(InternalString), +} + +impl StyleKey { + pub fn key(&self) -> &InternalString { + match self { + Self::Key(key) => key, + } + } +} + +impl StyleOp { + pub fn to_style(&self) -> Style { + Style { + key: self.key.clone(), + data: self.value.clone(), + } + } + + pub fn to_value(&self) -> LoroValue { + self.value.clone() + } + + pub(crate) fn get_style_key(&self) -> StyleKey { + StyleKey::Key(self.key.clone()) + } + + #[cfg(test)] + pub fn new_for_test(n: isize, key: &str, value: LoroValue, info: TextStyleInfoFlag) -> Self { + Self { + lamport: n as Lamport, + peer: n as PeerID, + cnt: n as Counter, + key: key.to_string().into(), + value, + info, + } + } + + #[inline(always)] + pub fn id(&self) -> ID { + ID::new(self.peer, self.cnt) + } + + pub fn idlp(&self) -> IdLp { + IdLp::new(self.peer, self.lamport) + } + + pub fn id_full(&self) -> IdFull { + IdFull::new(self.peer, self.cnt, self.lamport) + } +} + +impl PartialOrd for StyleOp { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for StyleOp { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.lamport + .cmp(&other.lamport) + .then(self.peer.cmp(&other.peer)) + } +} + +/// TODO: We can remove this type already +/// +/// A compact representation of a rich text style config. +/// +/// Note: we assume style with the same key has the same `Mergeable` and `isContainer` value. +/// +/// - 0 (1st bit) +/// - Expand Before (2nd bit): when inserting new text before this style, whether the new text should inherit this style. +/// - Expand After (3rd bit): when inserting new text after this style, whether the new text should inherit this style. +/// - 0 (4th bit): +/// - 0 (5th bit): +/// - 0 (6th bit) +/// - 0 (7th bit) +/// - 0 (8th bit): +#[derive(Default, Clone, Copy, Eq, PartialEq, Hash, serde::Serialize, serde::Deserialize)] +pub struct TextStyleInfoFlag { + data: u8, +} + +impl Debug for TextStyleInfoFlag { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TextStyleInfo") + // write data in binary format + .field("data", &format!("{:#010b}", self.data)) + .field("expand_before", &self.expand_before()) + .field("expand_after", &self.expand_after()) + .finish() + } +} + +const EXPAND_BEFORE_MASK: u8 = 0b0000_0010; +const EXPAND_AFTER_MASK: u8 = 0b0000_0100; +const ALIVE_MASK: u8 = 0b1000_0000; + +/// Whether to expand the style when inserting new text around it. +/// +/// - Before: when inserting new text before this style, the new text should inherit this style. +/// - After: when inserting new text after this style, the new text should inherit this style. +/// - Both: when inserting new text before or after this style, the new text should inherit this style. +/// - None: when inserting new text before or after this style, the new text should **not** inherit this style. +#[derive(Clone, Copy, Eq, PartialEq, Debug, Hash)] +pub enum ExpandType { + Before, + After, + Both, + None, +} + +#[derive( + Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Hash, serde::Serialize, serde::Deserialize, +)] +pub enum AnchorType { + Start, + End, +} + +impl ExpandType { + #[inline(always)] + pub const fn expand_before(&self) -> bool { + matches!(self, ExpandType::Before | ExpandType::Both) + } + + #[inline(always)] + pub const fn expand_after(&self) -> bool { + matches!(self, ExpandType::After | ExpandType::Both) + } + + /// 'before'|'after'|'both'|'none' + pub fn try_from_str(s: &str) -> Option { + match s { + "before" => Some(ExpandType::Before), + "after" => Some(ExpandType::After), + "both" => Some(ExpandType::Both), + "none" => Some(ExpandType::None), + _ => None, + } + } + + /// Toggle expand type between for deletion and for creation + /// + /// For a style that expand after, when we delete the style, we need to have another style that expands after to nullify it, + /// so that the expand behavior is not changed. + /// + /// Before -> Before + /// After -> After + /// Both -> None + /// None -> Both + /// + /// Because the creation of text styles and the deletion of the text styles have reversed expand type. + /// This method is useful to convert between the two + pub const fn reverse(self) -> Self { + match self { + ExpandType::Before => ExpandType::Before, + ExpandType::After => ExpandType::After, + ExpandType::Both => ExpandType::None, + ExpandType::None => ExpandType::Both, + } + } +} + +impl TextStyleInfoFlag { + /// When inserting new text around this style, prefer inserting after it. + #[inline(always)] + pub const fn expand_before(self) -> bool { + self.data & EXPAND_BEFORE_MASK != 0 + } + + /// When inserting new text around this style, prefer inserting before it. + #[inline(always)] + pub const fn expand_after(self) -> bool { + self.data & EXPAND_AFTER_MASK != 0 + } + + pub const fn expand_type(self) -> ExpandType { + match (self.expand_before(), self.expand_after()) { + (true, true) => ExpandType::Both, + (true, false) => ExpandType::Before, + (false, true) => ExpandType::After, + (false, false) => ExpandType::None, + } + } + + /// This method tells that when we can insert text before/after this style anchor, whether we insert the new text before the anchor. + #[inline] + pub fn prefer_insert_before(self, anchor_type: AnchorType) -> bool { + match anchor_type { + AnchorType::Start => { + // If we need to expand the style, the new text should be inserted **after** the start anchor + !self.expand_before() + } + AnchorType::End => { + // If we need to expand the style, the new text should be inserted **before** the end anchor + self.expand_after() + } + } + } + + pub const fn new(expand_type: ExpandType) -> Self { + let mut data = ALIVE_MASK; + if expand_type.expand_before() { + data |= EXPAND_BEFORE_MASK; + } + if expand_type.expand_after() { + data |= EXPAND_AFTER_MASK; + } + + TextStyleInfoFlag { data } + } + + #[inline(always)] + pub const fn to_delete(self) -> Self { + TextStyleInfoFlag::new(self.expand_type().reverse()) + } + + pub const BOLD: TextStyleInfoFlag = TextStyleInfoFlag::new(ExpandType::After); + pub const LINK: TextStyleInfoFlag = TextStyleInfoFlag::new(ExpandType::None); + pub const COMMENT: TextStyleInfoFlag = TextStyleInfoFlag::new(ExpandType::None); + + pub const fn to_byte(&self) -> u8 { + self.data + } + + pub const fn from_byte(data: u8) -> Self { + Self { data } + } +} + +#[cfg(test)] +mod test { + + #[test] + fn test() {} +} diff --git a/3rdparty/loro/crates/loro-internal/src/container/richtext/config.rs b/3rdparty/loro/crates/loro-internal/src/container/richtext/config.rs new file mode 100644 index 0000000000..8c89264f54 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/container/richtext/config.rs @@ -0,0 +1,133 @@ +use loro_common::InternalString; +use rustc_hash::FxHashMap; + +use super::{ExpandType, TextStyleInfoFlag}; + +#[derive(Debug, Default, Clone)] +pub struct StyleConfigMap { + pub(crate) map: FxHashMap, + pub(crate) default_style: Option, +} + +impl StyleConfigMap { + pub fn new() -> Self { + Self { + map: FxHashMap::default(), + default_style: None, + } + } + + pub fn insert(&mut self, key: InternalString, value: StyleConfig) { + if key.contains(':') { + panic!("style key should not contain ':'"); + } + + self.map.insert(key, value); + } + + pub fn get(&self, key: &InternalString) -> Option { + self.map.get(key).copied().or(self.default_style) + } + + pub fn get_style_flag(&self, key: &InternalString) -> Option { + self._get_style_flag(key, false) + } + + pub fn get_style_flag_for_unmark(&self, key: &InternalString) -> Option { + self._get_style_flag(key, true) + } + + fn _get_style_flag(&self, key: &InternalString, is_del: bool) -> Option { + let f = |x: StyleConfig| { + TextStyleInfoFlag::new(if is_del { x.expand.reverse() } else { x.expand }) + }; + if let Some(index) = key.find(':') { + let key = key[..index].into(); + self.map.get(&key).copied().or(self.default_style).map(f) + } else { + self.map.get(key).copied().or(self.default_style).map(f) + } + } + + pub fn default_rich_text_config() -> Self { + let mut map = Self { + map: FxHashMap::default(), + default_style: None, + }; + + map.map.insert( + "bold".into(), + StyleConfig { + expand: ExpandType::After, + }, + ); + + map.map.insert( + "italic".into(), + StyleConfig { + expand: ExpandType::After, + }, + ); + + map.map.insert( + "underline".into(), + StyleConfig { + expand: ExpandType::After, + }, + ); + + map.map.insert( + "link".into(), + StyleConfig { + expand: ExpandType::None, + }, + ); + + map.map.insert( + "highlight".into(), + StyleConfig { + expand: ExpandType::None, + }, + ); + + map.map.insert( + "comment".into(), + StyleConfig { + expand: ExpandType::None, + }, + ); + + map.map.insert( + "code".into(), + StyleConfig { + expand: ExpandType::None, + }, + ); + + map + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct StyleConfig { + pub expand: ExpandType, +} + +impl StyleConfig { + pub fn new() -> Self { + Self { + expand: ExpandType::None, + } + } + + pub fn expand(mut self, expand: ExpandType) -> Self { + self.expand = expand; + self + } +} + +impl Default for StyleConfig { + fn default() -> Self { + Self::new() + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/container/richtext/fugue_span.rs b/3rdparty/loro/crates/loro-internal/src/container/richtext/fugue_span.rs new file mode 100644 index 0000000000..696807888e --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/container/richtext/fugue_span.rs @@ -0,0 +1,386 @@ +use std::ops::Range; + +use generic_btree::rle::{CanRemove, HasLength, Mergeable, Sliceable, TryInsert}; +use loro_common::{CompactId, Counter, HasId, IdFull, IdSpan, Lamport, ID}; +use serde::{Deserialize, Serialize}; + +use super::{tracker::UNKNOWN_PEER_ID, AnchorType}; + +#[derive(Clone, PartialEq, Eq, Copy, Serialize, Deserialize)] +pub(crate) struct RichtextChunk { + start: u32, + end: u32, +} + +impl std::fmt::Debug for RichtextChunk { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RichtextChunk") + .field("value", &self.value()) + .finish() + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Copy)] +pub(crate) enum RichtextChunkKind { + Text, + StyleAnchor, + Unknown, + /// Move anchor is a special anchor that is used to mark the FugueSpan created + /// by a move operation. + MoveAnchor, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum RichtextChunkValue { + Text(Range), + StyleAnchor { + id: u32, + anchor_type: AnchorType, + }, + Unknown(u32), + /// Move anchor is a special anchor that is used to mark the FugueSpan created + /// by a move operation. + MoveAnchor, +} + +impl RichtextChunk { + pub(crate) const UNKNOWN: u32 = u32::MAX; + pub(crate) const START_STYLE_ANCHOR: u32 = u32::MAX - 1; + pub(crate) const END_STYLE_ANCHOR: u32 = u32::MAX - 2; + pub(crate) const MOVE_ANCHOR: u32 = u32::MAX - 3; + + #[inline] + pub fn new_text(range: Range) -> Self { + Self { + start: range.start, + end: range.end, + } + } + + #[inline] + pub fn new_style_anchor(idx: u32, anchor_type: AnchorType) -> Self { + match anchor_type { + AnchorType::Start => Self { + start: Self::START_STYLE_ANCHOR, + end: idx, + }, + AnchorType::End => Self { + start: Self::END_STYLE_ANCHOR, + end: idx, + }, + } + } + + #[inline] + pub fn new_unknown(len: u32) -> Self { + Self { + start: Self::UNKNOWN, + end: len, + } + } + + #[inline] + pub fn new_move() -> Self { + Self { + start: Self::MOVE_ANCHOR, + end: Self::MOVE_ANCHOR, + } + } + + #[inline] + pub(crate) fn kind(&self) -> RichtextChunkKind { + match self.start { + Self::START_STYLE_ANCHOR => RichtextChunkKind::StyleAnchor, + Self::END_STYLE_ANCHOR => RichtextChunkKind::StyleAnchor, + Self::UNKNOWN => RichtextChunkKind::Unknown, + Self::MOVE_ANCHOR => RichtextChunkKind::MoveAnchor, + _ => RichtextChunkKind::Text, + } + } + + #[inline(always)] + pub fn len(&self) -> usize { + match self.start { + Self::UNKNOWN => self.end as usize, + Self::START_STYLE_ANCHOR | Self::END_STYLE_ANCHOR | Self::MOVE_ANCHOR => 1, + _ => (self.end - self.start) as usize, + } + } + + #[inline] + pub(crate) fn value(&self) -> RichtextChunkValue { + match self.start { + Self::UNKNOWN => RichtextChunkValue::Unknown(self.end), + Self::START_STYLE_ANCHOR => RichtextChunkValue::StyleAnchor { + id: self.end, + anchor_type: AnchorType::Start, + }, + Self::END_STYLE_ANCHOR => RichtextChunkValue::StyleAnchor { + id: self.end, + anchor_type: AnchorType::End, + }, + Self::MOVE_ANCHOR => RichtextChunkValue::MoveAnchor, + _ => RichtextChunkValue::Text(self.start..self.end), + } + } +} + +impl Mergeable for RichtextChunk { + fn can_merge(&self, rhs: &Self) -> bool { + match (self.kind(), rhs.kind()) { + (RichtextChunkKind::Text, RichtextChunkKind::Text) => self.end == rhs.start, + (RichtextChunkKind::Unknown, RichtextChunkKind::Unknown) => true, + _ => false, + } + } + + fn merge_right(&mut self, rhs: &Self) { + match (self.kind(), rhs.kind()) { + (RichtextChunkKind::Text, RichtextChunkKind::Text) => self.end = rhs.end, + (RichtextChunkKind::Unknown, RichtextChunkKind::Unknown) => self.end += rhs.end, + _ => unreachable!(), + } + } + + fn merge_left(&mut self, left: &Self) { + match (self.kind(), left.kind()) { + (RichtextChunkKind::Text, RichtextChunkKind::Text) => self.start = left.start, + (RichtextChunkKind::Unknown, RichtextChunkKind::Unknown) => self.end += left.end, + _ => unreachable!(), + } + } +} + +impl HasLength for RichtextChunk { + #[inline(always)] + fn rle_len(&self) -> usize { + self.len() + } +} + +impl Sliceable for RichtextChunk { + fn _slice(&self, range: Range) -> Self { + match self.kind() { + RichtextChunkKind::Text => { + assert!( + range.len() <= self.len(), + "range: {:?}, self: {:?}", + range, + self + ); + Self { + start: self.start + range.start as u32, + end: self.start + range.end as u32, + } + } + RichtextChunkKind::StyleAnchor | RichtextChunkKind::MoveAnchor => { + assert_eq!(range.len(), 1); + *self + } + RichtextChunkKind::Unknown => { + assert!(range.len() <= self.len()); + Self { + start: Self::UNKNOWN, + end: range.len() as u32, + } + } + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Copy)] +pub(super) struct FugueSpan { + pub id: IdFull, + /// Sometimes, the id of the span is just a placeholder. + /// This field is used to track the real id of the span. + /// We know the real id when deletion happens because we + /// have the start_id info for each deletion. + pub real_id: Option, + /// The status at the current version + pub status: Status, + /// The status at the `new` version. + /// It's used when calculating diff. + pub diff_status: Option, + pub origin_left: Option, + pub origin_right: Option, + pub content: RichtextChunk, +} + +pub(super) enum DiffStatus { + NotChanged, + Created, + Deleted, +} + +impl FugueSpan { + #[inline(always)] + pub fn id_span(&self) -> IdSpan { + IdSpan::new( + self.id.peer, + self.id.counter, + self.id.counter + self.content.len() as Counter, + ) + } + + #[inline] + pub fn diff(&self) -> DiffStatus { + if self.diff_status.is_none() { + return DiffStatus::NotChanged; + } + + match ( + self.status.is_activated(), + self.diff_status.unwrap().is_activated(), + ) { + (true, false) => DiffStatus::Deleted, + (false, true) => DiffStatus::Created, + _ => DiffStatus::NotChanged, + } + } + + pub fn is_activated_in_diff(&self) -> bool { + match self.diff_status { + Some(d) => d.is_activated(), + None => self.status.is_activated(), + } + } + + pub(crate) fn real_id(&self) -> ID { + if let Some(id) = self.real_id { + id.to_id() + } else { + self.id.id() + } + } +} + +impl Sliceable for FugueSpan { + fn _slice(&self, range: Range) -> Self { + Self { + id: self.id.inc(range.start as Counter), + real_id: self.real_id.map(|x| x.inc(range.start as Counter)), + status: self.status, + diff_status: self.diff_status, + origin_left: if range.start == 0 { + self.origin_left + } else { + Some( + self.id + .inc((range.start - 1) as Counter) + .id() + .try_into() + .unwrap(), + ) + }, + origin_right: self.origin_right, + content: self.content._slice(range), + } + } +} + +impl HasLength for FugueSpan { + #[inline(always)] + fn rle_len(&self) -> usize { + self.content.len() + } +} + +impl Mergeable for FugueSpan { + fn can_merge(&self, rhs: &Self) -> bool { + self.id.peer == rhs.id.peer + && self.status == rhs.status + && self.diff_status == rhs.diff_status + && self.id.counter + self.content.len() as Counter == rhs.id.counter + && self.id.lamport + self.content.len() as Lamport == rhs.id.lamport + && rhs.origin_left.is_some() + && rhs.origin_left.unwrap().peer == self.id.peer + && rhs.origin_left.unwrap().counter.get() + == self.id.counter + self.content.len() as Counter - 1 + && self.origin_right == rhs.origin_right + && self.content.can_merge(&rhs.content) + && ((self.real_id.is_none() && rhs.real_id.is_none()) + || (self.real_id.is_some() + && rhs.real_id.is_some() + && self.real_id.unwrap().inc(self.content.len() as i32) + == rhs.real_id.unwrap())) + } + + fn merge_right(&mut self, rhs: &Self) { + self.content.merge_right(&rhs.content); + } + + fn merge_left(&mut self, left: &Self) { + self.id = left.id; + self.real_id = left.real_id; + self.origin_left = left.origin_left; + self.content.merge_left(&left.content); + } +} + +impl TryInsert for FugueSpan { + fn try_insert(&mut self, _pos: usize, elem: Self) -> Result<(), Self> + where + Self: Sized, + { + Err(elem) + } +} + +impl CanRemove for FugueSpan { + fn can_remove(&self) -> bool { + self.content.len() == 0 + } +} + +impl FugueSpan { + #[allow(unused)] + pub fn new(id: IdFull, content: RichtextChunk) -> Self { + Self { + id, + real_id: if id.peer == UNKNOWN_PEER_ID { + None + } else { + Some(id.id().try_into().unwrap()) + }, + status: Status::default(), + diff_status: None, + origin_left: None, + origin_right: None, + content, + } + } + + #[inline(always)] + pub fn is_activated(&self) -> bool { + self.status.is_activated() + } + + #[inline] + pub fn activated_len(&self) -> usize { + if self.is_activated() { + self.content.len() + } else { + 0 + } + } +} + +impl HasId for FugueSpan { + fn id_start(&self) -> ID { + self.id.id() + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Default, Hash, Copy)] +pub(super) struct Status { + /// is this span from a future operation + pub future: bool, + pub delete_times: i16, +} + +impl Status { + #[inline(always)] + pub fn is_activated(&self) -> bool { + self.delete_times == 0 && !self.future + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/container/richtext/richtext_state.rs b/3rdparty/loro/crates/loro-internal/src/container/richtext/richtext_state.rs new file mode 100644 index 0000000000..fb8dad91b1 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/container/richtext/richtext_state.rs @@ -0,0 +1,3522 @@ +use append_only_bytes::BytesSlice; +use generic_btree::{ + rle::{CanRemove, HasLength, Mergeable, Sliceable, TryInsert}, + BTree, BTreeTrait, Cursor, LeafIndex, +}; +use loro_common::{Counter, IdFull, IdSpan, LoroError, LoroResult, LoroValue, ID}; +use query::{ByteQuery, ByteQueryT}; +use rustc_hash::{FxHashMap, FxHashSet}; +use serde::{ser::SerializeStruct, Serialize}; +use std::{ + fmt::{Display, Formatter}, + ops::{Bound, RangeBounds}, +}; +use std::{ + ops::{Add, AddAssign, Range, Sub}, + str::Utf8Error, + sync::Arc, +}; +use smallvec::SmallVec; +use tracing::instrument; + +use crate::{ + container::richtext::style_range_map::EMPTY_STYLES, + delta::{DeltaValue, StyleMeta}, + utils::query_by_len::{EntityIndexQueryWithEventIndex, IndexQueryWithEntityIndex, QueryByLen}, +}; + +use self::query::{ + EntityQuery, EntityQueryT, EventIndexQuery, EventIndexQueryT, UnicodeQuery, UnicodeQueryT, + Utf16Query, Utf16QueryT, +}; + +use super::{ + style_range_map::{IterAnchorItem, StyleRangeMap, Styles}, + AnchorType, RichtextSpan, StyleKey, StyleOp, +}; + +pub(crate) use crate::cursor::PosType; + +#[derive(Clone, Debug, Default)] +pub(crate) struct RichtextState { + tree: BTree, + style_ranges: Option>, + cached_cursor: Option, +} + +impl Display for RichtextState { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + for span in self.tree.iter() { + match span { + RichtextStateChunk::Style { .. } => {} + RichtextStateChunk::Text(s) => { + f.write_str(s.as_str())?; + } + } + } + + Ok(()) + } +} + +use cache::CachedCursor; + +mod cache { + use super::*; + + #[derive(Clone, Debug)] + pub(super) struct CachedCursor { + leaf: LeafIndex, + index: FxHashMap, + } + + impl RichtextState { + pub(super) fn clear_cache(&mut self) { + self.cached_cursor = None; + } + + pub(super) fn record_cache( + &mut self, + leaf: LeafIndex, + pos: usize, + pos_type: PosType, + entity_offset: usize, + entity_index: Option, + ) { + let offset = if entity_offset != 0 { + let elem = self.tree.get_elem(leaf).unwrap(); + entity_offset_to_pos_type_offset(pos_type, elem, entity_offset) + } else { + 0 + }; + + let pos = pos - offset; + match &mut self.cached_cursor { + Some(c) => { + if c.leaf == leaf { + c.index.insert(pos_type, pos); + } else { + c.leaf = leaf; + c.index.clear(); + c.index.insert(pos_type, pos); + } + } + None => { + self.cached_cursor = Some(CachedCursor { + leaf, + index: FxHashMap::default(), + }); + self.cached_cursor + .as_mut() + .unwrap() + .index + .insert(pos_type, pos); + } + } + if let Some(entity_index) = entity_index { + self.cached_cursor + .as_mut() + .unwrap() + .index + .insert(PosType::Entity, entity_index - entity_offset); + } + } + + pub(super) fn try_get_cache_or_clean( + &mut self, + index: usize, + pos_type: PosType, + ) -> Option { + let mut cursor = self.cached_cursor.take(); + let ans = 'block: { + match &mut cursor { + Some(c) => { + let cache_index = c.index.get(&pos_type); + if let Some(cache_index) = cache_index { + if index < *cache_index { + return None; + } + } + + let cached_index = match c.index.entry(pos_type) { + std::collections::hash_map::Entry::Vacant(vacant_entry) => { + let index = self.get_index_from_cursor( + Cursor { + leaf: c.leaf, + offset: 0, + }, + pos_type, + )?; + vacant_entry.insert(index); + index + } + std::collections::hash_map::Entry::Occupied(occupied_entry) => { + *occupied_entry.get() + } + }; + + if index < cached_index { + return None; + } + + if cached_index == index { + break 'block Some(Cursor { + leaf: c.leaf, + offset: 0, + }); + } + + let elem = self.tree.get_elem(c.leaf)?; + let elem_len = elem.len_with(pos_type); + if cached_index + elem_len == index { + let offset = elem.len_with(PosType::Entity); + break 'block Some(Cursor { + leaf: c.leaf, + offset, + }); + } + + if index > cached_index + elem_len { + return None; + } + + let offset = + pos_type_offset_to_entity_offset(pos_type, elem, index - cached_index)?; + Some(Cursor { + leaf: c.leaf, + offset, + }) + } + None => None, + } + }; + + self.cached_cursor = cursor; + #[cfg(debug_assertions)] + { + if let Some(c) = ans.as_ref() { + let actual = self.get_index_from_cursor(*c, pos_type); + assert_eq!(actual.unwrap(), index); + } + } + ans + } + + pub(super) fn get_cache_entity_index(&mut self) -> Option { + self.get_cache_leaf_start_index(PosType::Entity) + } + + /// Index (in `pos_type` units) of the start of the currently cached leaf, + /// computing it once via `get_index_from_cursor` and memoizing it on the + /// cached cursor for reuse by later same-leaf queries. + pub(super) fn get_cache_leaf_start_index(&mut self, pos_type: PosType) -> Option { + let mut cursor = self.cached_cursor.take()?; + let ans = { + let leaf = cursor.leaf; + match cursor.index.entry(pos_type) { + std::collections::hash_map::Entry::Vacant(vacant_entry) => { + let index = + self.get_index_from_cursor(Cursor { leaf, offset: 0 }, pos_type)?; + vacant_entry.insert(index); + index + } + std::collections::hash_map::Entry::Occupied(occupied_entry) => { + *occupied_entry.get() + } + } + }; + + self.cached_cursor = Some(cursor); + Some(ans) + } + + /// Resolve the `pos_type` index of an arbitrary cursor using the cached + /// leaf, avoiding a full `visit_previous_caches` walk when the cursor + /// lands in the currently cached leaf. Returns `None` (miss) otherwise. + pub(super) fn cursor_index_via_cache( + &mut self, + cursor: Cursor, + pos_type: PosType, + ) -> Option { + match &self.cached_cursor { + Some(c) if c.leaf == cursor.leaf => {} + _ => return None, + } + let leaf_start = self.get_cache_leaf_start_index(pos_type)?; + if cursor.offset == 0 { + return Some(leaf_start); + } + let elem = self.tree.get_elem(cursor.leaf)?; + Some(leaf_start + entity_offset_to_pos_type_offset(pos_type, elem, cursor.offset)) + } + + pub(crate) fn check_cache(&self) { + #[cfg(debug_assertions)] + { + if let Some(c) = &self.cached_cursor { + for (pos_type, index) in &c.index { + let actual = self.get_index_from_cursor( + Cursor { + leaf: c.leaf, + offset: 0, + }, + *pos_type, + ); + + assert_eq!(actual.unwrap(), *index); + } + } + } + } + } +} + +pub(crate) use text_chunk::TextChunk; +mod text_chunk { + use std::ops::Range; + + use append_only_bytes::BytesSlice; + use loro_common::{IdFull, ID}; + + #[derive(Clone, PartialEq)] + pub(crate) struct TextChunk { + bytes: BytesSlice, + unicode_len: i32, + utf16_len: i32, + id: IdFull, + } + + impl std::fmt::Debug for TextChunk { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TextChunk") + .field("text", &self.as_str()) + .field("unicode_len", &self.unicode_len) + .field("utf16_len", &self.utf16_len) + .field("id", &self.id) + .finish() + } + } + + impl TextChunk { + pub fn new(bytes: BytesSlice, id: IdFull) -> Self { + let mut utf16_len = 0; + let mut unicode_len = 0; + for c in std::str::from_utf8(&bytes).unwrap().chars() { + utf16_len += c.len_utf16(); + unicode_len += 1; + } + + Self { + unicode_len, + bytes, + utf16_len: utf16_len as i32, + id, + } + } + + #[inline] + pub fn id(&self) -> ID { + self.id.id() + } + + #[inline] + #[allow(unused)] + pub fn id_full(&self) -> IdFull { + self.id + } + + #[inline] + pub fn bytes(&self) -> &BytesSlice { + &self.bytes + } + + #[inline] + pub fn as_str(&self) -> &str { + // SAFETY: We know that the text is valid UTF-8 + unsafe { std::str::from_utf8_unchecked(&self.bytes) } + } + + #[inline] + pub fn len(&self) -> i32 { + self.unicode_len + } + + #[inline] + pub fn utf8_len(&self) -> i32 { + self.bytes.len() as i32 + } + + #[inline] + pub fn unicode_len(&self) -> i32 { + self.unicode_len + } + + #[inline] + pub fn utf16_len(&self) -> i32 { + self.utf16_len + } + + #[inline] + pub fn event_len(&self) -> i32 { + if cfg!(feature = "wasm") { + self.utf16_len + } else { + self.unicode_len + } + } + + /// Convert a unicode index on this text to an event index + pub fn convert_unicode_offset_to_event_offset(&self, offset: usize) -> usize { + if cfg!(feature = "wasm") { + let mut event_offset = 0; + for (i, c) in self.as_str().chars().enumerate() { + if i == offset { + return event_offset; + } + event_offset += c.len_utf16(); + } + event_offset + } else { + offset + } + } + + pub fn new_empty() -> Self { + Self { + unicode_len: 0, + bytes: BytesSlice::empty(), + utf16_len: 0, + // This is a dummy value. + // It's fine because the length is 0. We never actually use this value. + id: IdFull::NONE_ID, + } + } + + /// Whether every char in this chunk is in the Basic Multilingual Plane, + /// i.e. each char is a single UTF-16 code unit. When true, UTF-16 offsets + /// equal unicode offsets, so conversions are O(1) instead of O(offset). + #[inline] + fn is_all_bmp(&self) -> bool { + self.utf16_len == self.unicode_len + } + + /// Whether this chunk is pure ASCII, i.e. each char is a single byte. + /// When true, byte offsets equal unicode offsets. + #[inline] + fn is_all_ascii(&self) -> bool { + self.bytes.len() as i32 == self.unicode_len + } + + /// Convert a UTF-16 offset within this chunk to a unicode offset. + /// + /// O(1) when the chunk has no astral-plane characters; otherwise it + /// scans up to `utf16_offset` chars. Mirrors [`utf16_to_unicode_index`]. + pub fn utf16_offset_to_unicode(&self, utf16_offset: usize) -> Result { + if self.is_all_bmp() { + return Ok(utf16_offset); + } + super::utf16_to_unicode_index(self.as_str(), utf16_offset) + } + + /// Convert a UTF-8 (byte) offset within this chunk to a unicode offset. + /// O(1) when the chunk is pure ASCII. + pub fn utf8_offset_to_unicode(&self, utf8_offset: usize) -> Result { + if self.is_all_ascii() { + return Ok(utf8_offset); + } + super::utf8_to_unicode_index(self.as_str(), utf8_offset) + } + + /// Convert a unicode offset within this chunk to a UTF-16 offset. + /// O(1) when the chunk has no astral-plane characters. + pub fn unicode_offset_to_utf16(&self, unicode_offset: usize) -> Option { + if self.is_all_bmp() { + return Some(unicode_offset); + } + super::unicode_to_utf16_index(self.as_str(), unicode_offset) + } + + /// Convert a unicode offset within this chunk to a UTF-8 (byte) offset. + /// O(1) when the chunk is pure ASCII. + pub fn unicode_offset_to_utf8(&self, unicode_offset: usize) -> Option { + if self.is_all_ascii() { + return Some(unicode_offset); + } + super::unicode_to_utf8_index(self.as_str(), unicode_offset) + } + + /// Slice this chunk by unicode offsets, returning the substring. + /// O(1) byte lookup when the chunk is pure ASCII. + pub fn unicode_slice(&self, start: usize, end: usize) -> Result<&str, ()> { + if self.is_all_ascii() { + return self.as_str().get(start..end).ok_or(()); + } + super::unicode_slice(self.as_str(), start, end) + } + + pub(crate) fn delete_by_entity_index( + &mut self, + unicode_offset: usize, + unicode_len: usize, + ) -> (Option, usize) { + let s = self.as_str(); + let mut start_byte = 0; + let mut end_byte = self.bytes().len(); + let start_unicode_index = unicode_offset; + let end_unicode_index = unicode_offset + unicode_len; + let mut start_utf16_index = 0; + let mut current_utf16_index = 0; + let mut current_utf8_index = 0; + for (current_unicode_index, c) in s.chars().enumerate() { + if current_unicode_index == start_unicode_index { + start_utf16_index = current_utf16_index; + start_byte = current_utf8_index; + } + + if current_unicode_index == end_unicode_index { + end_byte = current_utf8_index; + break; + } + + current_utf16_index += c.len_utf16(); + current_utf8_index += c.len_utf8(); + } + + self.utf16_len -= (current_utf16_index - start_utf16_index) as i32; + + let event_len = if cfg!(feature = "wasm") { + current_utf16_index - start_utf16_index + } else { + unicode_len + }; + + self.unicode_len -= unicode_len as i32; + let next = match (start_byte == 0, end_byte == self.bytes.len()) { + (true, true) => { + self.bytes = BytesSlice::empty(); + None + } + (true, false) => { + self.bytes.slice_(end_byte..); + self.id = self.id.inc(end_unicode_index as i32); + None + } + (false, true) => { + self.bytes.slice_(..start_byte); + None + } + (false, false) => { + let next = self.bytes.slice_clone(end_byte..); + let next = Self::new(next, self.id.inc(end_unicode_index as i32)); + self.unicode_len -= next.unicode_len; + self.utf16_len -= next.utf16_len; + self.bytes.slice_(..start_byte); + Some(next) + } + }; + + self.check(); + if let Some(next) = next.as_ref() { + next.check(); + } + (next, event_len) + } + + fn check(&self) { + if !cfg!(any(debug_assertions, test)) { + return; + } + + let bytes_len = self.bytes.len() as i32; + if bytes_len == self.unicode_len { + assert_eq!(self.utf16_len, self.unicode_len); + return; + } + + assert_eq!(self.unicode_len, self.as_str().chars().count() as i32); + assert_eq!( + self.utf16_len, + self.as_str().chars().map(|c| c.len_utf16()).sum::() as i32 + ); + } + + pub(crate) fn entity_range_to_event_range(&self, range: Range) -> Range { + if cfg!(feature = "wasm") { + assert!(range.start <= range.end); + if range.start == 0 && range.end == self.unicode_len as usize { + return 0..self.utf16_len as usize; + } + + let mut start = 0; + let mut end = 0; + let mut utf16_index = 0; + for (unicode_index, c) in self.as_str().chars().enumerate() { + if unicode_index == range.start { + start = utf16_index; + } + if unicode_index == range.end { + end = utf16_index; + break; + } + + utf16_index += c.len_utf16(); + } + + if end == 0 { + end = utf16_index; + } + + start..end + } else { + range + } + } + } + + impl generic_btree::rle::HasLength for TextChunk { + fn rle_len(&self) -> usize { + self.unicode_len as usize + } + } + + impl generic_btree::rle::Sliceable for TextChunk { + fn _slice(&self, range: Range) -> Self { + assert!(range.start < range.end); + let total_unicode_len = self.unicode_len as usize; + if range.start == 0 && range.end == total_unicode_len { + return self.clone(); + } + + // Fast path for ASCII text: unicode index == byte index, and utf16 index == unicode index. + if self.is_all_ascii() { + let ans = Self { + unicode_len: range.len() as i32, + bytes: self.bytes.slice_clone(range.start..range.end), + utf16_len: range.len() as i32, + id: self.id.inc(range.start as i32), + }; + ans.check(); + return ans; + } + + // Fast path for slicing a suffix/prefix to avoid scanning the whole chunk. + if range.end == total_unicode_len { + let mut utf16_offset = 0; + let mut start_byte = self.bytes.len(); + for (unicode_index, (byte_index, c)) in self.as_str().char_indices().enumerate() { + if unicode_index == range.start { + start_byte = byte_index; + break; + } + utf16_offset += c.len_utf16(); + } + + let ans = Self { + unicode_len: (total_unicode_len - range.start) as i32, + bytes: self.bytes.slice_clone(start_byte..), + utf16_len: self.utf16_len - utf16_offset as i32, + id: self.id.inc(range.start as i32), + }; + ans.check(); + return ans; + } + + if range.start == 0 { + let mut utf16_len = 0; + let mut end_byte = self.bytes.len(); + for (unicode_index, (byte_index, c)) in self.as_str().char_indices().enumerate() { + if unicode_index == range.end { + end_byte = byte_index; + break; + } + utf16_len += c.len_utf16(); + } + + let ans = Self { + unicode_len: range.end as i32, + bytes: self.bytes.slice_clone(..end_byte), + utf16_len: utf16_len as i32, + id: self.id, + }; + ans.check(); + return ans; + } + + // General middle slice. + let mut utf16_len = 0; + let mut start = 0; + let mut end = 0; + let mut started = false; + let mut last_unicode_index = 0; + for (unicode_index, (i, c)) in self.as_str().char_indices().enumerate() { + if unicode_index == range.start { + start = i; + started = true; + } + + if unicode_index == range.end { + end = i; + break; + } + if started { + utf16_len += c.len_utf16(); + } + + last_unicode_index = unicode_index; + } + + assert!(started); + if end == 0 { + assert_eq!(last_unicode_index + 1, range.end); + end = self.bytes.len(); + } + + let ans = Self { + unicode_len: range.len() as i32, + bytes: self.bytes.slice_clone(start..end), + utf16_len: utf16_len as i32, + id: self.id.inc(range.start as i32), + }; + ans.check(); + ans + } + + fn split(&mut self, pos: usize) -> Self { + let mut utf16_len = 0; + let mut byte_offset = 0; + for (unicode_index, (i, c)) in self.as_str().char_indices().enumerate() { + if unicode_index == pos { + byte_offset = i; + break; + } + + utf16_len += c.len_utf16(); + } + let right = Self { + unicode_len: self.unicode_len - pos as i32, + bytes: self.bytes.slice_clone(byte_offset..), + utf16_len: self.utf16_len - utf16_len as i32, + id: self.id.inc(pos as i32), + }; + + self.unicode_len = pos as i32; + self.utf16_len = utf16_len as i32; + self.bytes.slice_(..byte_offset); + right.check(); + self.check(); + right + } + } + + impl generic_btree::rle::Mergeable for TextChunk { + fn can_merge(&self, rhs: &Self) -> bool { + self.bytes.can_merge(&rhs.bytes) && self.id.inc(self.unicode_len) == rhs.id + } + + fn merge_right(&mut self, rhs: &Self) { + self.bytes.try_merge(&rhs.bytes).unwrap(); + self.utf16_len += rhs.utf16_len; + self.unicode_len += rhs.unicode_len; + self.check(); + } + + fn merge_left(&mut self, left: &Self) { + let mut new = left.bytes.clone(); + new.try_merge(&self.bytes).unwrap(); + self.bytes = new; + self.utf16_len += left.utf16_len; + self.unicode_len += left.unicode_len; + self.id = left.id; + self.check(); + } + } +} + +// TODO: change visibility back to crate after #116 is done +#[derive(Clone, Debug, PartialEq)] +pub(crate) enum RichtextStateChunk { + Text(TextChunk), + Style { + style: Arc, + anchor_type: AnchorType, + }, +} + +impl Default for RichtextStateChunk { + fn default() -> Self { + Self::new_empty() + } +} + +impl RichtextStateChunk { + pub fn new_text(s: BytesSlice, id: IdFull) -> Self { + Self::Text(TextChunk::new(s, id)) + } + + pub fn new_empty() -> Self { + Self::Text(TextChunk::new_empty()) + } + + pub fn new_style(style: Arc, anchor_type: AnchorType) -> Self { + Self::Style { style, anchor_type } + } + + pub(crate) fn get_id_span(&self) -> IdSpan { + match self { + RichtextStateChunk::Text(t) => { + let id = t.id(); + IdSpan::new(id.peer, id.counter, id.counter + t.unicode_len() as Counter) + } + RichtextStateChunk::Style { style, anchor_type } => match anchor_type { + AnchorType::Start => style.id().into(), + AnchorType::End => { + let id = style.id(); + id.to_span(1) + } + }, + } + } + + pub(crate) fn counter(&self) -> Counter { + match self { + RichtextStateChunk::Text(t) => t.id().counter, + RichtextStateChunk::Style { style, anchor_type } => match anchor_type { + AnchorType::Start => style.id().counter, + AnchorType::End => { + let id = style.id(); + id.counter + 1 + } + }, + } + } + + pub fn entity_range_to_event_range(&self, range: Range) -> Range { + match self { + RichtextStateChunk::Text(t) => t.entity_range_to_event_range(range), + RichtextStateChunk::Style { .. } => { + assert_eq!(range.start, 0); + assert_eq!(range.end, 1); + 0..1 + } + } + } + + pub fn len_with(&self, pos_type: PosType) -> usize { + match self { + RichtextStateChunk::Text(t) => match pos_type { + PosType::Bytes => t.utf8_len() as usize, + PosType::Utf16 => t.utf16_len() as usize, + PosType::Event if cfg!(feature = "wasm") => t.utf16_len() as usize, + PosType::Event => t.unicode_len() as usize, + PosType::Entity => t.unicode_len() as usize, + PosType::Unicode => t.unicode_len() as usize, + }, + RichtextStateChunk::Style { .. } => { + if let PosType::Entity = pos_type { + 1 + } else { + 0 + } + } + } + } +} + +impl loro_delta::delta_trait::DeltaValue for RichtextStateChunk {} + +impl DeltaValue for RichtextStateChunk { + fn value_extend(&mut self, other: Self) -> Result<(), Self> { + Err(other) + } + + fn take(&mut self, length: usize) -> Self { + let mut right = self.split(length); + std::mem::swap(self, &mut right); + right + } + + fn length(&self) -> usize { + self.rle_len() + } +} + +impl Serialize for RichtextStateChunk { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + match self { + RichtextStateChunk::Text(text) => { + let mut state = serializer.serialize_struct("RichtextStateChunk", 3)?; + state.serialize_field("type", "Text")?; + state.serialize_field("unicode_len", &text.unicode_len())?; + state.serialize_field("text", text.as_str())?; + state.end() + } + RichtextStateChunk::Style { style, anchor_type } => { + let mut state = serializer.serialize_struct("RichtextStateChunk", 3)?; + state.serialize_field("type", "Style")?; + state.serialize_field("style", &style.key)?; + state.serialize_field("anchor_type", anchor_type)?; + state.end() + } + } + } +} + +impl RichtextStateChunk { + pub fn try_new(s: BytesSlice, id: IdFull) -> Result { + std::str::from_utf8(&s)?; + Ok(RichtextStateChunk::Text(TextChunk::new(s, id))) + } + + pub fn from_style(style: Arc, anchor_type: AnchorType) -> Self { + Self::Style { style, anchor_type } + } + + pub fn as_str(&self) -> Option<&str> { + match self { + RichtextStateChunk::Text(text) => Some(text.as_str()), + _ => None, + } + } +} + +impl HasLength for RichtextStateChunk { + fn rle_len(&self) -> usize { + match self { + RichtextStateChunk::Text(s) => s.rle_len(), + RichtextStateChunk::Style { .. } => 1, + } + } +} + +impl Mergeable for RichtextStateChunk { + fn can_merge(&self, rhs: &Self) -> bool { + match (self, rhs) { + (RichtextStateChunk::Text(l), RichtextStateChunk::Text(r)) => l.can_merge(r), + _ => false, + } + } + + fn merge_right(&mut self, rhs: &Self) { + match (self, rhs) { + (RichtextStateChunk::Text(l), RichtextStateChunk::Text(r)) => l.merge_right(r), + _ => unreachable!(), + } + } + + fn merge_left(&mut self, left: &Self) { + match (self, left) { + (RichtextStateChunk::Text(this), RichtextStateChunk::Text(left)) => { + this.merge_left(left) + } + _ => unreachable!(), + } + } +} + +impl Sliceable for RichtextStateChunk { + fn _slice(&self, range: Range) -> Self { + match self { + RichtextStateChunk::Text(s) => RichtextStateChunk::Text(s._slice(range)), + RichtextStateChunk::Style { style, anchor_type } => { + assert_eq!(range.start, 0); + assert_eq!(range.end, 1); + RichtextStateChunk::Style { + style: style.clone(), + anchor_type: *anchor_type, + } + } + } + } + + fn split(&mut self, pos: usize) -> Self { + match self { + RichtextStateChunk::Text(s) => RichtextStateChunk::Text(s.split(pos)), + RichtextStateChunk::Style { .. } => { + unreachable!() + } + } + } +} + +impl TryInsert for RichtextStateChunk { + fn try_insert(&mut self, _pos: usize, elem: Self) -> Result<(), Self> + where + Self: Sized, + { + Err(elem) + } +} + +impl CanRemove for RichtextStateChunk { + fn can_remove(&self) -> bool { + self.rle_len() == 0 + } +} + +//TODO: start/end can be scanned in one loop, but now it takes twice the time +fn unicode_slice(s: &str, start_index: usize, end_index: usize) -> Result<&str, ()> { + let (Some(start), Some(end)) = ( + unicode_to_utf8_index(s, start_index), + unicode_to_utf8_index(s, end_index), + ) else { + return Err(()); + }; + Ok(&s[start..end]) +} + +pub(crate) fn unicode_to_utf8_index(s: &str, unicode_index: usize) -> Option { + let mut current_unicode_index = 0; + for (byte_index, _) in s.char_indices() { + if current_unicode_index == unicode_index { + return Some(byte_index); + } + current_unicode_index += 1; + } + + if current_unicode_index == unicode_index { + return Some(s.len()); + } + + None +} + +pub(crate) fn unicode_to_utf16_index(s: &str, unicode_index: usize) -> Option { + if unicode_index == 0 { + return Some(0); + } + + let mut current_unicode_index = 0; + let mut current_utf16_index = 0; + for c in s.chars() { + let len = c.len_utf16(); + current_unicode_index += 1; + current_utf16_index += len; + if current_unicode_index == unicode_index { + return Some(current_utf16_index); + } + } + + None +} + +pub(crate) fn utf16_to_utf8_index(s: &str, utf16_index: usize) -> Option { + if utf16_index == 0 { + return Some(0); + } + + let mut current_utf16_index = 0; + for (byte_index, c) in s.char_indices() { + let len = c.len_utf16(); + current_utf16_index += len; + if current_utf16_index == utf16_index { + return Some(byte_index + c.len_utf8()); + } + } + + if current_utf16_index == utf16_index { + return Some(s.len()); + } + + None +} + +/// Returns the unicode index of the character at the given utf16 index. +/// +/// If the given utf16 index is not at the correct boundary, returns the unicode index of the +/// character before the given utf16 index. +pub(crate) fn utf16_to_unicode_index(s: &str, utf16_index: usize) -> Result { + if utf16_index == 0 { + return Ok(0); + } + + let mut current_utf16_index = 0; + let mut current_unicode_index = 0; + for (i, c) in s.chars().enumerate() { + current_utf16_index += c.len_utf16(); + if current_utf16_index == utf16_index { + return Ok(i + 1); + } + if current_utf16_index > utf16_index { + loro_common::info!("WARNING: UTF16 MISMATCHED!"); + return Err(i); + } + current_unicode_index = i + 1; + } + + loro_common::info!("WARNING: UTF16 MISMATCHED!"); + Err(current_unicode_index) +} + +pub(crate) fn utf8_to_unicode_index(s: &str, utf8_index: usize) -> Result { + if utf8_index == 0 { + return Ok(0); + } + + let mut current_utf8_index = 0; + let mut current_unicode_index = 0; + for (i, c) in s.chars().enumerate() { + let char_start = current_utf8_index; + current_utf8_index += c.len_utf8(); + + if utf8_index == char_start { + return Ok(i); + } + + if utf8_index < current_utf8_index { + loro_common::info!("WARNING: UTF-8 index is in the middle of a codepoint!"); + return Err(i); + } + current_unicode_index = i + 1; + } + + if current_utf8_index == utf8_index { + Ok(current_unicode_index) + } else { + Err(current_unicode_index) + } +} + +#[derive(Clone, Debug, Copy, PartialEq, Eq, Default)] +pub(crate) struct PosCache { + pub(super) unicode_len: i32, + pub(super) bytes: i32, + pub(super) utf16_len: i32, + pub(crate) entity_len: i32, +} + +impl PosCache { + pub(crate) fn event_len(&self) -> i32 { + if cfg!(feature = "wasm") { + self.utf16_len + } else { + self.unicode_len + } + } + + #[allow(unused)] + fn get_len(&self, pos_type: PosType) -> i32 { + match pos_type { + PosType::Bytes => self.bytes, + PosType::Unicode => self.unicode_len, + PosType::Utf16 => self.utf16_len, + PosType::Entity => self.entity_len, + PosType::Event => self.event_len(), + } + } +} + +impl AddAssign for PosCache { + fn add_assign(&mut self, rhs: Self) { + self.unicode_len += rhs.unicode_len; + self.bytes += rhs.bytes; + self.utf16_len += rhs.utf16_len; + self.entity_len += rhs.entity_len; + } +} + +impl Add for PosCache { + type Output = Self; + + fn add(self, rhs: Self) -> Self::Output { + Self { + bytes: self.bytes + rhs.bytes, + unicode_len: self.unicode_len + rhs.unicode_len, + utf16_len: self.utf16_len + rhs.utf16_len, + entity_len: self.entity_len + rhs.entity_len, + } + } +} + +impl Sub for PosCache { + type Output = Self; + + fn sub(self, rhs: Self) -> Self::Output { + Self { + bytes: self.bytes - rhs.bytes, + unicode_len: self.unicode_len - rhs.unicode_len, + utf16_len: self.utf16_len - rhs.utf16_len, + entity_len: self.entity_len - rhs.entity_len, + } + } +} + +impl CanRemove for PosCache { + fn can_remove(&self) -> bool { + self.bytes == 0 + } +} + +pub(crate) struct RichtextTreeTrait; + +#[derive(Debug)] +pub(crate) struct EntityRangeInfo { + pub id_start: ID, + pub entity_start: usize, + pub entity_end: usize, + pub event_len: usize, +} + +/// Entity ranges produced by a single text query. Inline storage avoids a heap +/// allocation on the per-op delete hot path, where a contiguous delete is +/// usually a single merged range. +pub(crate) type EntityRanges = SmallVec<[EntityRangeInfo; 2]>; + +impl EntityRangeInfo { + pub fn entity_len(&self) -> usize { + self.entity_end - self.entity_start + } +} + +impl BTreeTrait for RichtextTreeTrait { + type Elem = RichtextStateChunk; + + type Cache = PosCache; + + type CacheDiff = PosCache; + + fn calc_cache_internal( + cache: &mut Self::Cache, + caches: &[generic_btree::Child], + ) -> Self::CacheDiff { + let mut new_cache = PosCache::default(); + for child in caches { + new_cache += child.cache; + } + + let diff = new_cache - *cache; + *cache = new_cache; + diff + } + + #[inline(always)] + fn merge_cache_diff(diff1: &mut Self::CacheDiff, diff2: &Self::CacheDiff) { + *diff1 += *diff2; + } + + #[inline(always)] + fn apply_cache_diff(cache: &mut Self::Cache, diff: &Self::CacheDiff) { + *cache += *diff; + } + + #[inline] + fn get_elem_cache(elem: &Self::Elem) -> Self::Cache { + match elem { + RichtextStateChunk::Text(s) => PosCache { + bytes: s.bytes().len() as i32, + unicode_len: s.unicode_len(), + utf16_len: s.utf16_len(), + entity_len: s.unicode_len(), + }, + RichtextStateChunk::Style { .. } => PosCache { + bytes: 0, + unicode_len: 0, + utf16_len: 0, + entity_len: 1, + }, + } + } + + #[inline(always)] + fn new_cache_to_diff(cache: &Self::Cache) -> Self::CacheDiff { + *cache + } + + #[inline(always)] + fn sub_cache(cache_lhs: &Self::Cache, cache_rhs: &Self::Cache) -> Self::CacheDiff { + PosCache { + bytes: cache_lhs.bytes - cache_rhs.bytes, + unicode_len: cache_lhs.unicode_len - cache_rhs.unicode_len, + utf16_len: cache_lhs.utf16_len - cache_rhs.utf16_len, + entity_len: cache_lhs.entity_len - cache_rhs.entity_len, + } + } +} + +// This query implementation will prefer right element when both left element and right element are valid. +mod query { + use crate::utils::query_by_len::{IndexQuery, QueryByLen}; + + use super::*; + + #[cfg(not(feature = "wasm"))] + pub(super) type EventIndexQuery = UnicodeQuery; + #[cfg(feature = "wasm")] + pub(super) type EventIndexQuery = Utf16Query; + + #[cfg(not(feature = "wasm"))] + pub(super) type EventIndexQueryT = UnicodeQueryT; + #[cfg(feature = "wasm")] + pub(super) type EventIndexQueryT = Utf16QueryT; + + pub(super) struct UnicodeQueryT; + pub(super) type UnicodeQuery = IndexQuery; + + impl QueryByLen for UnicodeQueryT { + fn get_cache_len(cache: &::Cache) -> usize { + cache.unicode_len as usize + } + + fn get_elem_len(elem: &::Elem) -> usize { + match elem { + RichtextStateChunk::Text(s) => s.rle_len(), + RichtextStateChunk::Style { .. } => 0, + } + } + + fn get_offset_and_found( + left: usize, + elem: &::Elem, + ) -> (usize, bool) { + match elem { + RichtextStateChunk::Text(s) => { + if s.rle_len() >= left { + return (left, true); + } + + (left, false) + } + RichtextStateChunk::Style { .. } => (1, false), + } + } + + fn get_cache_entity_len(cache: &::Cache) -> usize { + cache.entity_len as usize + } + } + + pub(crate) struct Utf16QueryT; + pub(crate) type Utf16Query = IndexQuery; + + impl QueryByLen for Utf16QueryT { + fn get_cache_len(cache: &::Cache) -> usize { + cache.utf16_len as usize + } + + fn get_elem_len(elem: &::Elem) -> usize { + match elem { + RichtextStateChunk::Text(s) => s.utf16_len() as usize, + RichtextStateChunk::Style { .. } => 0, + } + } + + fn get_offset_and_found( + left: usize, + elem: &::Elem, + ) -> (usize, bool) { + match elem { + RichtextStateChunk::Text(s) => { + if left == 0 { + return (0, true); + } + + // Allow left to not at the correct utf16 boundary. If so fallback to the last position. + // TODO: if we remove the use of query(pos-1), we won't need this fallback behavior + // WARNING: Unable to report error!!! + let offset = s.utf16_offset_to_unicode(left).unwrap_or_else(|e| e); + (offset, true) + } + RichtextStateChunk::Style { .. } => (1, false), + } + } + + fn get_cache_entity_len(cache: &::Cache) -> usize { + cache.entity_len as usize + } + } + + pub(super) struct EntityQueryT; + pub(super) type EntityQuery = IndexQuery; + + impl QueryByLen for EntityQueryT { + fn get_cache_len(cache: &::Cache) -> usize { + cache.entity_len as usize + } + + fn get_elem_len(elem: &::Elem) -> usize { + match elem { + RichtextStateChunk::Text(s) => s.rle_len(), + RichtextStateChunk::Style { .. } => 1, + } + } + + fn get_offset_and_found( + left: usize, + elem: &::Elem, + ) -> (usize, bool) { + match elem { + RichtextStateChunk::Text(s) => { + if s.rle_len() >= left { + return (left, true); + } + + (left, false) + } + RichtextStateChunk::Style { .. } => { + if left == 0 { + return (0, true); + } + + (left, false) + } + } + } + + fn get_cache_entity_len(cache: &::Cache) -> usize { + cache.entity_len as usize + } + } + + pub(super) struct ByteQueryT; + pub(super) type ByteQuery = IndexQuery; + impl QueryByLen for ByteQueryT { + fn get_cache_len(cache: &::Cache) -> usize { + cache.bytes as usize + } + fn get_elem_len(elem: &::Elem) -> usize { + match elem { + RichtextStateChunk::Text(s) => s.utf8_len() as usize, + RichtextStateChunk::Style { .. } => 0, + } + } + + fn get_offset_and_found( + left: usize, + elem: &::Elem, + ) -> (usize, bool) { + match elem { + RichtextStateChunk::Text(s) => { + if left == 0 { + return (0, true); + } + + // Allow left to not at the correct utf16 boundary. If so fallback to the last position. + // TODO: if we remove the use of query(pos-1), we won't need this fallback behavior + // WARNING: Unable to report error!!! + let offset = s.utf8_offset_to_unicode(left).unwrap_or_else(|e| e); + (offset, true) + } + RichtextStateChunk::Style { .. } => (1, false), + } + } + + fn get_cache_entity_len(cache: &::Cache) -> usize { + cache.entity_len as usize + } + } +} + +impl RichtextState { + pub(crate) fn from_chunks>>(i: I) -> Self { + Self { + tree: i.collect(), + style_ranges: Default::default(), + cached_cursor: None, + } + } + + pub(crate) fn get_entity_index_for_text_insert( + &mut self, + pos: usize, + pos_type: PosType, + ) -> Result<(usize, Option), LoroError> { + self.check_cache(); + let result = { + if self.tree.is_empty() { + return Ok((0, None)); + } + + if let Some(c) = self.try_get_cache_or_clean(pos, pos_type) { + let entity_index = self.get_cache_entity_index().unwrap(); + Ok((entity_index + c.offset, Some(c))) + } else { + let (c, entity_index) = match pos_type { + PosType::Bytes => self.find_best_insert_pos::(pos), + PosType::Unicode => self.find_best_insert_pos::(pos), + PosType::Utf16 => self.find_best_insert_pos::(pos), + PosType::Entity => self.find_best_insert_pos::(pos), + PosType::Event => self.find_best_insert_pos::(pos), + }; + + if let Some(c) = c { + debug_assert_eq!( + self.get_index_from_cursor(c, PosType::Entity).unwrap(), + entity_index + ); + self.record_cache( + c.leaf, + entity_index, + PosType::Entity, + c.offset, + Some(entity_index), + ); + } + Ok((entity_index, c)) + } + }; + self.check_cache(); + result + } + + pub(crate) fn has_styles(&self) -> bool { + self.style_ranges + .as_ref() + .map(|x| x.has_style()) + .unwrap_or(false) + } + + pub(crate) fn range_has_style_key(&mut self, range: Range, key: &StyleKey) -> bool { + self.check_cache(); + let result = match self.style_ranges.as_ref() { + Some(s) => s.range_contains_key(range, key), + None => false, + }; + self.check_cache(); + result + } + + /// Return the entity range and text styles at the given range. + /// If in the target range the leaves are not in the same span, the returned styles would be None + pub(crate) fn get_entity_range_and_text_styles_at_range( + &mut self, + range: Range, + pos_type: PosType, + ) -> (Range, Option<&Styles>) { + self.check_cache(); + let result = { + if self.tree.is_empty() { + return (0..0, None); + } + + let (start, _) = self + .get_entity_index_for_text_insert(range.start, pos_type) + .unwrap(); + let (end, _) = self + .get_entity_index_for_text_insert(range.end, pos_type) + .unwrap(); + if self.has_styles() { + ( + start..end, + self.style_ranges + .as_ref() + .unwrap() + .get_styles_of_range(start..end), + ) + } else { + (start..end, None) + } + }; + self.check_cache(); + result + } + + /// Get the insert text styles at the given entity index if we insert text at that position + /// + // TODO: PERF we can avoid this calculation by getting it when inserting new text + // but that requires a lot of changes + pub(crate) fn get_styles_at_entity_index_for_insert( + &mut self, + entity_index: usize, + ) -> StyleMeta { + self.check_cache(); + let result = { + if !self.has_styles() { + return Default::default(); + } + + self.style_ranges + .as_mut() + .unwrap() + .get_styles_for_insert(entity_index) + }; + self.check_cache(); + result + } + + /// This is used to accept changes from DiffCalculator + pub(crate) fn insert_at_entity_index( + &mut self, + entity_index: usize, + text: BytesSlice, + id: IdFull, + ) -> Cursor { + self.check_cache(); + let result = { + let elem = RichtextStateChunk::try_new(text, id).unwrap(); + self.style_ranges + .as_mut() + .map(|x| x.insert(entity_index, elem.rle_len())); + let q = &entity_index; + if let Some(c) = self.try_get_cache_or_clean(entity_index, PosType::Entity) { + let p = self.tree.prefer_left(c).unwrap_or(c); + self.tree.insert_by_path(p, elem).0 + } else { + match self.tree.query::(q) { + Some(result) => { + let p = self + .tree + .prefer_left(result.cursor) + .unwrap_or(result.cursor); + self.tree.insert_by_path(p, elem).0 + } + None => self.tree.push(elem), + } + } + }; + self.record_cache( + result.leaf, + entity_index, + PosType::Entity, + result.offset, + None, + ); + self.check_cache(); + result + } + + /// This is used to accept changes from DiffCalculator. + /// + /// Return (event_index, styles) + pub(crate) fn insert_elem_at_entity_index( + &mut self, + entity_index: usize, + elem: RichtextStateChunk, + ) -> (usize, &Styles) { + self.clear_cache(); + let result = { + debug_assert!( + entity_index <= self.len_entity(), + "entity_index={} len={} self={:#?}", + entity_index, + self.len_entity(), + &self + ); + + let (c, f) = self + .tree + .query_with_finder_return::(&entity_index); + let cursor = c.map(|x| x.cursor); + let event_index = f.event_index; + self.clear_cache(); + + match cursor { + Some(cursor) => { + let styles = self + .style_ranges + .as_mut() + .map(|x| x.insert(entity_index, elem.rle_len())) + .unwrap_or(&EMPTY_STYLES); + self.tree.insert_by_path(cursor, elem); + (event_index, styles) + } + None => { + let styles = self + .style_ranges + .as_mut() + .map(|x| x.insert(entity_index, elem.rle_len())) + .unwrap_or(&EMPTY_STYLES); + self.tree.push(elem); + (0, styles) + } + } + }; + result + } + + /// Convert cursor position to event index: + /// + /// - If feature="wasm", index is utf16 index, + /// - If feature!="wasm", index is unicode index, + pub(crate) fn cursor_to_event_index(&self, cursor: Cursor) -> usize { + self.check_cache(); + let result = self.get_index_from_cursor(cursor, PosType::Event).unwrap(); + self.check_cache(); + result + } + + /// Cache-aware (`&mut`) variant of [`Self::cursor_to_event_index`] for the + /// per-op insert hot path. When the cursor lands in the currently cached + /// leaf, this reuses the memoized event-index start instead of doing a fresh + /// `visit_previous_caches` walk on every insert. + pub(crate) fn cursor_to_event_index_cached(&mut self, cursor: Cursor) -> usize { + self.check_cache(); + let ans = match self.cursor_index_via_cache(cursor, PosType::Event) { + Some(i) => { + debug_assert_eq!( + i, + self.get_index_from_cursor(cursor, PosType::Event).unwrap() + ); + i + } + None => self.get_index_from_cursor(cursor, PosType::Event).unwrap(), + }; + self.check_cache(); + ans + } + + pub(crate) fn cursor_to_unicode_index(&self, cursor: Cursor) -> usize { + self.check_cache(); + let result = self + .get_index_from_cursor(cursor, PosType::Unicode) + .unwrap(); + self.check_cache(); + result + } + + /// This method only updates `style_ranges`. + /// When this method is called, the style start anchor and the style end anchor should already have been inserted. + pub(crate) fn annotate_style_range(&mut self, range: Range, style: Arc) { + self.check_cache(); + self.clear_cache(); + self.ensure_style_ranges_mut().annotate(range, style, None); + self.check_cache(); + } + + /// This method only updates `style_ranges`. + /// When this method is called, the style start anchor and the style end anchor should already have been inserted. + /// + /// This method will return the event of this annotation in event length + pub(crate) fn annotate_style_range_with_event( + &mut self, + range: Range, + style: Arc, + ) -> impl Iterator + '_ { + self.check_cache(); + self.clear_cache(); + let mut ranges_in_entity_index: Vec<(StyleMeta, Range)> = Vec::new(); + let mut start = range.start; + let end = range.end; + self.ensure_style_ranges_mut().annotate( + range, + style, + Some(&mut |s, len| { + let range = start..start + len; + start += len; + ranges_in_entity_index.push((s.into(), range)); + }), + ); + + assert_eq!(ranges_in_entity_index.last().unwrap().1.end, end); + let mut converter = ContinuousIndexConverter::new(self); + let result = ranges_in_entity_index + .into_iter() + .filter_map(move |(meta, range)| { + let start = converter.convert_entity_index_to_event_index(range.start); + let end = converter.convert_entity_index_to_event_index(range.end); + if end == start { + return None; + } + + Some((meta, end - start)) + }); + self.check_cache(); + result + } + + /// init style ranges if not initialized + fn ensure_style_ranges_mut(&mut self) -> &mut StyleRangeMap { + self.clear_cache(); + let result = { + if self.style_ranges.is_none() { + self.style_ranges = Some(Box::default()); + } + + self.style_ranges.as_mut().unwrap() + }; + result + } + + pub(crate) fn get_char_by_event_index(&self, pos: usize) -> Result { + self.check_cache(); + let result = { + let cursor = self.tree.query::(&pos).unwrap().cursor; + let Some(str) = &self.tree.get_elem(cursor.leaf) else { + return Err(()); + }; + if cfg!(not(feature = "wasm")) { + let mut char_iter = str.as_str().unwrap().chars(); + match &mut char_iter.nth(cursor.offset) { + Some(c) => Ok(*c), + None => Err(()), + } + } else { + let s = str.as_str().unwrap(); + let utf16offset = unicode_to_utf16_index(s, cursor.offset).unwrap(); + // Convert utf16 offset to actual character by finding the character at that position + let char_iter = s.chars(); + let mut current_utf16_offset = 0; + + for c in char_iter { + if current_utf16_offset == utf16offset { + return Ok(c); + } + current_utf16_offset += c.len_utf16(); + if current_utf16_offset > utf16offset { + return Err(()); + } + } + Err(()) + } + }; + self.check_cache(); + result + } + + /// Find the best insert position based on the rich-text CRDT algorithm. + /// + /// The result is only different from `query` when there are style anchors around the insert pos. + /// Returns the right neighbor of the insert pos and the entity index. + /// + /// 1. Insertions occur before style anchors that contain the beginning of new marks. + /// 2. Insertions occur before style anchors that contain the end of bold-like marks + /// 3. Insertions occur after style anchors that contain the end of link-like marks + /// + /// Rule 1 should be satisfied before rules 2 and 3 to avoid creating a new style out of nowhere + /// + /// The current method will scan forward to find the last position that satisfies 1 and 2. + /// Then it scans backward to find the first position that satisfies 3. + fn find_best_insert_pos>( + &self, + pos: usize, + ) -> (Option, usize) { + type Query = IndexQueryWithEntityIndex; + if self.tree.is_empty() { + return (None, 0); + } + + // There are a range of elements may share the same unicode index + // because style anchors' lengths are zero in unicode index. + + // Find the start and the end of the range, and entity index of left cursor + let (left, right, mut entity_index) = if pos == 0 { + let left = self.tree.start_cursor().unwrap(); + let mut right = left; + let mut elem = self.tree.get_elem(right.leaf).unwrap(); + let entity_index = 0; + if matches!(elem, RichtextStateChunk::Text { .. }) { + return (Some(right), 0); + } else { + while Q::get_elem_len(elem) == 0 { + match self.tree.next_elem(right) { + Some(r) => { + right = r; + elem = self.tree.get_elem(right.leaf).unwrap(); + } + None => { + right.offset = elem.rle_len(); + break; + } + } + } + + (left, right, entity_index) + } + } else { + // The query prefers right when there are empty elements (style anchors) + // So the nodes between (pos-1) and pos are all style anchors. + let (q, f) = self.tree.query_with_finder_return::>(&(pos - 1)); + let q = q.unwrap(); + let mut entity_index = f.entity_index(); + + let elem = self.tree.get_elem(q.leaf()).unwrap(); + let mut right = q.cursor; + right.offset += 1; + entity_index += 1; + if elem.rle_len() > right.offset { + // The cursor is in the middle of a style anchor + return (Some(right), entity_index); + } + + match self.tree.next_elem(q.cursor) { + // If next is None, we know the range is empty, return directly + None => return (self.tree.end_cursor(), entity_index), + Some(x) => { + assert_eq!(right.offset, elem.rle_len()); + right = x; + let mut elem = self.tree.get_elem(right.leaf).unwrap(); + if matches!(elem, RichtextStateChunk::Text { .. }) { + return (Some(right), entity_index); + } + + let left = x; + while matches!(elem, RichtextStateChunk::Style { .. }) { + match self.tree.next_elem(right) { + Some(r) => { + right = r; + elem = self.tree.get_elem(right.leaf).unwrap(); + } + None => { + // this is last element + right.offset = 1; + break; + } + } + } + + (left, right, entity_index) + } + } + }; + + let mut iter = left; + + // need to scan from left to right + let mut visited = Vec::new(); + while iter != right { + let Some(elem) = self.tree.get_elem(iter.leaf) else { + break; + }; + + let (style, anchor_type) = match elem { + RichtextStateChunk::Text { .. } => unreachable!(), + RichtextStateChunk::Style { style, anchor_type } => (style, *anchor_type), + }; + + visited.push((style, anchor_type, iter, entity_index)); + if anchor_type == AnchorType::Start && (style.value.is_null() || style.value.is_false()) + { + // case 1. should be before this anchor + break; + } + + if style.info.prefer_insert_before(anchor_type) { + // case 2. + break; + } + + iter = match self.tree.next_elem(iter) { + Some(x) => x, + None => self.tree.end_cursor().unwrap(), + }; + entity_index += 1; + } + + while let Some((style, anchor_type, top_elem, top_entity_index)) = visited.pop() { + if !style.info.prefer_insert_before(anchor_type) { + // case 3. + break; + } + + iter = top_elem; + entity_index = top_entity_index; + } + + (Some(iter), entity_index) + } + + fn get_entity_index_from_path(&self, right: generic_btree::Cursor) -> usize { + self.get_index_from_cursor(right, PosType::Entity).unwrap() + } + + pub fn get_index_from_cursor( + &self, + cursor: generic_btree::Cursor, + pos_type: PosType, + ) -> Option { + let mut index = 0; + self.tree + .visit_previous_caches(cursor, |cache| match cache { + generic_btree::PreviousCache::NodeCache(c) => { + index += c.get_len(pos_type) as usize; + } + generic_btree::PreviousCache::PrevSiblingElem(c) => { + index += c.len_with(pos_type); + } + generic_btree::PreviousCache::ThisElemAndOffset { elem, offset } => { + if offset != 0 { + index += entity_offset_to_pos_type_offset(pos_type, elem, offset) + } + } + }); + + Some(index) + } + + pub(crate) fn get_text_entity_ranges( + &self, + pos: usize, + len: usize, + pos_type: PosType, + ) -> LoroResult { + self.check_cache(); + let result = { + if self.tree.is_empty() { + return Ok(EntityRanges::new()); + } + + if len == 0 { + return Ok(EntityRanges::new()); + } + + if pos + len > self.len(pos_type) { + return Ok(EntityRanges::new()); + } + + let mut ans: EntityRanges = SmallVec::new(); + let (start, end) = match pos_type { + PosType::Bytes => ( + self.tree.query::(&pos).unwrap().cursor, + self.tree.query::(&(pos + len)).unwrap().cursor, + ), + PosType::Unicode => ( + self.tree.query::(&pos).unwrap().cursor, + self.tree + .query::(&(pos + len)) + .unwrap() + .cursor, + ), + PosType::Utf16 => ( + self.tree.query::(&pos).unwrap().cursor, + self.tree.query::(&(pos + len)).unwrap().cursor, + ), + PosType::Entity => ( + self.tree.query::(&pos).unwrap().cursor, + self.tree.query::(&(pos + len)).unwrap().cursor, + ), + PosType::Event => ( + self.tree.query::(&pos).unwrap().cursor, + self.tree + .query::(&(pos + len)) + .unwrap() + .cursor, + ), + }; + + // TODO: assert end cursor is valid + let mut entity_index = self.get_entity_index_from_path(start); + for span in self.tree.iter_range(start..end) { + let start = span.start.unwrap_or(0); + let end = span.end.unwrap_or(span.elem.rle_len()); + if end == 0 { + break; + } + + let len = end - start; + match span.elem { + RichtextStateChunk::Text(s) => { + let event_len = s.entity_range_to_event_range(start..end).len(); + let id = s.id().inc(start as i32); + match ans.last_mut() { + Some(last) + if last.entity_end == entity_index + && last.id_start.inc(last.event_len as i32) == id => + { + last.entity_end += len; + last.event_len += event_len; + } + _ => { + ans.push(EntityRangeInfo { + id_start: id, + entity_start: entity_index, + entity_end: entity_index + len, + event_len, + }); + } + } + entity_index += len; + } + RichtextStateChunk::Style { .. } => { + entity_index += 1; + } + } + } + + Ok(ans) + }; + self.check_cache(); + result + } + + pub(crate) fn get_text_slice_by_event_index( + &self, + pos: usize, + len: usize, + ) -> LoroResult { + self.check_cache(); + let result = { + if self.tree.is_empty() { + return Ok(String::new()); + } + + if len == 0 { + return Ok(String::new()); + } + + if pos + len > self.len_event() { + return Err(LoroError::OutOfBound { + pos: pos + len, + len: self.len_event(), + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + }); + } + + let mut ans = String::new(); + let (start, end) = ( + self.tree.query::(&pos).unwrap().cursor, + self.tree + .query::(&(pos + len)) + .unwrap() + .cursor, + ); + + for span in self.tree.iter_range(start..end) { + let start = span.start.unwrap_or(0); + let end = span.end.unwrap_or(span.elem.rle_len()); + if end == 0 { + break; + } + + if let RichtextStateChunk::Text(s) = span.elem { + match s.unicode_slice(start, end) { + Ok(x) => ans.push_str(x), + Err(()) => { + return Err(LoroError::UTF16InUnicodeCodePoint { pos: pos + len }) + } + } + } + } + + Ok(ans) + }; + self.check_cache(); + result + } + + pub(crate) fn slice_delta( + &self, + start_index: usize, + end_index: usize, + pos_type: PosType, + ) -> LoroResult> { + if end_index < start_index { + return Err(LoroError::EndIndexLessThanStartIndex { + start: start_index, + end: end_index, + }); + } + + if end_index == start_index { + return Ok(Vec::new()); + } + + self.check_cache(); + let result = { + // 1. Convert start/end pos to cursors + let (start_query, end_query) = match pos_type { + PosType::Event => ( + self.tree.query::(&start_index), + self.tree.query::(&end_index), + ), + PosType::Bytes => ( + self.tree.query::(&start_index), + self.tree.query::(&end_index), + ), + PosType::Unicode => ( + self.tree.query::(&start_index), + self.tree.query::(&end_index), + ), + PosType::Utf16 => ( + self.tree.query::(&start_index), + self.tree.query::(&end_index), + ), + PosType::Entity => ( + self.tree.query::(&start_index), + self.tree.query::(&end_index), + ), + }; + + let start_cursor = start_query + .ok_or(LoroError::OutOfBound { + pos: start_index, + len: self.len(pos_type), + info: "".into(), + })? + .cursor; + let end_cursor = end_query + .ok_or(LoroError::OutOfBound { + pos: end_index, + len: self.len(pos_type), + info: "".into(), + })? + .cursor; + + // 2. Get start entity index + let mut current_entity_index = self + .get_index_from_cursor(start_cursor, PosType::Entity) + .unwrap(); + // We need end entity index for style iterator + let end_entity_index = self + .get_index_from_cursor(end_cursor, PosType::Entity) + .unwrap(); + + // 3. Setup style iterator + let mut style_range_iter: Box, &Styles)>> = + match &self.style_ranges { + Some(s) => { + let mut idx = current_entity_index; + Box::new(s.iter_range(current_entity_index..end_entity_index).map( + move |elem_slice| { + let len = elem_slice.end.unwrap_or(elem_slice.elem.len) + - elem_slice.start.unwrap_or(0); + let range = idx..idx + len; + idx += len; + (range, &elem_slice.elem.styles) + }, + )) + } + None => Box::new(Some((0..usize::MAX / 2, &*EMPTY_STYLES)).into_iter()), + }; + + let mut cur_style_range = style_range_iter.next(); + let mut cur_styles: Option = + cur_style_range.as_ref().map(|x| x.1.clone().into()); + + let mut ans: Vec<(String, StyleMeta)> = Vec::new(); + + // 4. Iterate tree range + for span in self.tree.iter_range(start_cursor..end_cursor) { + match &span.elem { + RichtextStateChunk::Text(t) => { + let chunk_len = + span.end.unwrap_or(span.elem.rle_len()) - span.start.unwrap_or(0); // length in rle_len (unicode_len) + let mut processed_len = 0; + + while processed_len < chunk_len { + while let Some((inner_cur_range, _)) = cur_style_range.as_ref() { + if current_entity_index >= inner_cur_range.end { + cur_style_range = style_range_iter.next(); + cur_styles = + cur_style_range.as_ref().map(|x| x.1.clone().into()); + } else { + break; + } + } + + if cur_style_range.is_none() { + cur_styles = Some(StyleMeta::default()); + cur_style_range = + Some((current_entity_index..usize::MAX, &*EMPTY_STYLES)); + } + + let (inner_cur_range, _) = cur_style_range.as_ref().unwrap(); + + let remaining_text = chunk_len - processed_len; + let remaining_style = inner_cur_range.end - current_entity_index; + let take_len = remaining_text.min(remaining_style); + + let slice_start = span.start.unwrap_or(0) + processed_len; + let slice_end = slice_start + take_len; + + let text_content = + t.unicode_slice(slice_start, slice_end).map_err(|_| { + LoroError::OutOfBound { + pos: slice_end, + len: t.unicode_len() as usize, + info: "Slice delta out of bound".into(), + } + })?; + + let styles = cur_styles.as_ref().unwrap(); + if let Some(last) = ans.last_mut() { + if &last.1 == styles { + last.0.push_str(text_content); + processed_len += take_len; + current_entity_index += take_len; + continue; + } + } + + ans.push((text_content.to_string(), styles.clone())); + + processed_len += take_len; + current_entity_index += take_len; + } + } + RichtextStateChunk::Style { .. } => { + current_entity_index += 1; + } + } + } + + Ok(ans) + }; + self.check_cache(); + result + } + + // PERF: can be splitted into two methods. One is without cursor_to_event_index + // PERF: can be speed up a lot by detecting whether the range is in a single leaf first + /// This is used to accept changes from DiffCalculator + #[instrument(skip(self, f))] + pub(crate) fn drain_by_entity_index( + &mut self, + pos: usize, + len: usize, + mut f: Option<&mut dyn FnMut(RichtextStateChunk)>, + ) -> DrainInfo { + let result = { + assert!( + pos + len <= self.len_entity(), + "pos: {}, len: {}, self.len(): {}", + pos, + len, + &self.len_entity(), + ); + + self.clear_cache(); + // PERF: may use cache to speed up + let range = pos..pos + len; + let (start, start_f) = self + .tree + .query_with_finder_return::(&range.start); + let start_cursor = start.unwrap().cursor(); + let elem = self.tree.get_elem(start_cursor.leaf).unwrap(); + + /// This struct remove the corresponding style ranges if the start style anchor is removed + struct StyleRangeUpdater<'a> { + style_ranges: Option<&'a mut StyleRangeMap>, + current_index: usize, + start: usize, + end: usize, + } + + impl<'a> StyleRangeUpdater<'a> { + fn update(&mut self, elem: &RichtextStateChunk) { + match &elem { + RichtextStateChunk::Text(t) => { + self.current_index += t.unicode_len() as usize; + } + RichtextStateChunk::Style { style, anchor_type } => { + if matches!(anchor_type, AnchorType::End) { + self.end = self.end.max(self.current_index); + if let Some(s) = self.style_ranges.as_mut() { + let start = + s.remove_style_scanning_backward(style, self.current_index); + self.start = self.start.min(start); + } + } + + self.current_index += 1; + } + } + } + + fn new( + style_ranges: Option<&'a mut Box>, + start_index: usize, + ) -> Self { + Self { + style_ranges: style_ranges.map(|x| &mut **x), + current_index: start_index, + end: 0, + start: usize::MAX, + } + } + + fn get_affected_range(&self, pos: usize) -> Option> { + if self.start == usize::MAX { + None + } else { + let start = self.start.min(pos); + let end = self.end.min(pos); + if start == end { + None + } else { + Some(start..end) + } + } + } + } + + if elem.rle_len() >= start_cursor.offset + len { + // drop in place + let mut event_len = 0; + let mut updater = StyleRangeUpdater::new(self.style_ranges.as_mut(), pos); + self.tree.update_leaf(start_cursor.leaf, |elem| { + updater.update(&*elem); + match elem { + RichtextStateChunk::Text(text) => { + if let Some(f) = f { + let span = + text.slice(start_cursor.offset..start_cursor.offset + len); + f(RichtextStateChunk::Text(span)); + } + let (next, event_len_) = + text.delete_by_entity_index(start_cursor.offset, len); + event_len = event_len_; + (true, next.map(RichtextStateChunk::Text), None) + } + RichtextStateChunk::Style { .. } => { + if let Some(f) = f { + let v = std::mem::replace( + elem, + RichtextStateChunk::Text(TextChunk::new_empty()), + ); + f(v); + } else { + *elem = RichtextStateChunk::Text(TextChunk::new_empty()); + } + (true, None, None) + } + } + }); + + let affected_range = updater.get_affected_range(pos); + if let Some(s) = self.style_ranges.as_mut() { + s.delete(pos..pos + len); + } + + DrainInfo { + start_event_index: start_f.event_index, + end_event_index: (start_f.event_index + event_len), + affected_style_range: affected_range.map(|entity_range| { + ( + entity_range.clone(), + self.entity_index_to_event_index(entity_range.start) + ..self.entity_index_to_event_index(entity_range.end), + ) + }), + } + } else { + let (end, end_f) = self + .tree + .query_with_finder_return::(&range.end); + let mut updater = StyleRangeUpdater::new(self.style_ranges.as_mut(), pos); + for iter in generic_btree::iter::Drain::new(&mut self.tree, start, end) { + updater.update(&iter); + if let Some(f) = f.as_mut() { + f(iter) + } + } + + let affected_range = updater.get_affected_range(pos); + if let Some(s) = self.style_ranges.as_mut() { + s.delete(pos..pos + len); + } + + DrainInfo { + start_event_index: start_f.event_index, + end_event_index: end_f.event_index, + affected_style_range: affected_range.map(|entity_range| { + ( + entity_range.clone(), + self.entity_index_to_event_index(entity_range.start) + ..self.entity_index_to_event_index(entity_range.end), + ) + }), + } + } + }; + result + } + + pub fn entity_index_to_event_index(&self, index: usize) -> usize { + if index == 0 { + // the tree maybe empty + return 0; + } + let cursor = self.tree.query::(&index).unwrap(); + self.cursor_to_event_index(cursor.cursor) + } + + pub fn index_to_event_index(&self, index: usize, pos_type: PosType) -> usize { + if self.tree.is_empty() { + return 0; + } + + let cursor = match pos_type { + PosType::Entity => self.tree.query::(&index).unwrap(), + PosType::Utf16 => self.tree.query::(&index).unwrap(), + PosType::Bytes => self.tree.query::(&index).unwrap(), + PosType::Event => return index, + PosType::Unicode => self.tree.query::(&index).unwrap(), + }; + + self.cursor_to_event_index(cursor.cursor) + } + + pub fn event_index_to_unicode_index(&self, index: usize) -> usize { + if !cfg!(feature = "wasm") { + return index; + } + + let Some(cursor) = self.tree.query::(&index) else { + return 0; + }; + + self.cursor_to_unicode_index(cursor.cursor) + } + + /// Convert an event-index position into the index of `pos_type`. + /// + /// This runs in O(log n) by querying the tree once and reading the prefix + /// caches from the resulting cursor, instead of materializing the whole + /// prefix string. + pub fn event_index_to_index(&self, event_index: usize, pos_type: PosType) -> usize { + if self.tree.is_empty() { + return 0; + } + + let cursor = self.tree.query::(&event_index).unwrap(); + self.get_index_from_cursor(cursor.cursor, pos_type).unwrap() + } + + #[allow(unused)] + pub(crate) fn check(&self) { + if !cfg!(any(debug_assertions, test)) { + return; + } + + self.tree.check(); + self.check_consistency_between_content_and_style_ranges(); + self.check_style_anchors_appear_in_pairs(); + } + + pub(crate) fn mark_with_entity_index(&mut self, range: Range, style: Arc) { + self.check_cache(); + self.clear_cache(); + if self.tree.is_empty() { + panic!("Cannot mark an empty tree"); + } + + self.insert_elem_at_entity_index( + range.end, + RichtextStateChunk::from_style(style.clone(), AnchorType::End), + ); + self.insert_elem_at_entity_index( + range.start, + RichtextStateChunk::from_style(style.clone(), AnchorType::Start), + ); + // end_entity_index + 2, because + // 1. We inserted a start anchor before end_entity_index, so we need to +1 + // 2. We need to include the end anchor in the range, so we need to +1 + self.ensure_style_ranges_mut() + .annotate(range.start..range.end + 2, style, None); + self.check_cache(); + } + + pub fn iter(&self) -> impl Iterator + '_ { + let mut entity_index = 0; + let mut style_range_iter: Box, &Styles)>> = + match &self.style_ranges { + Some(s) => Box::new(s.iter()), + None => Box::new(Some((0..usize::MAX / 2, &*EMPTY_STYLES)).into_iter()), + }; + let mut cur_style_range = style_range_iter.next(); + let mut cur_styles: Option = + cur_style_range.as_ref().map(|x| x.1.clone().into()); + + self.tree.iter().filter_map(move |x| match x { + RichtextStateChunk::Text(s) => { + let mut styles = Default::default(); + while let Some((inner_cur_range, _)) = cur_style_range.as_ref() { + if entity_index < inner_cur_range.start { + break; + } + + if entity_index < inner_cur_range.end { + styles = cur_styles.as_ref().unwrap().clone(); + break; + } else { + cur_style_range = style_range_iter.next(); + cur_styles = cur_style_range.as_ref().map(|x| x.1.clone().into()); + } + } + + entity_index += s.rle_len(); + Some(RichtextSpan { + text: s.bytes().clone().into(), + attributes: styles, + }) + } + RichtextStateChunk::Style { .. } => { + entity_index += 1; + None + } + }) + } + + #[inline] + pub fn iter_chunk(&self) -> impl Iterator { + self.tree.iter() + } + + pub fn get_richtext_value(&self) -> LoroValue { + self.check_cache(); + let result = { + let mut ans: Vec = Vec::new(); + let mut last_attributes: Option = None; + for span in self.iter() { + let attributes: LoroValue = span.attributes.to_value(); + if let Some(last) = last_attributes.as_ref() { + if &attributes == last { + let hash_map = ans.last_mut().unwrap().as_map_mut().unwrap(); + let s = hash_map + .make_mut() + .get_mut("insert") + .unwrap() + .as_string_mut() + .unwrap(); + s.make_mut().push_str(span.text.as_str()); + continue; + } + } + + let mut value = FxHashMap::default(); + value.insert( + "insert".into(), + LoroValue::String(span.text.as_str().into()), + ); + + if !attributes.as_map().unwrap().is_empty() { + value.insert("attributes".into(), attributes.clone()); + } + + ans.push(LoroValue::Map(value.into())); + last_attributes = Some(attributes); + } + + LoroValue::List(ans.into()) + }; + self.check_cache(); + result + } + + #[allow(unused)] + pub fn to_vec(&self) -> Vec { + self.iter().collect() + } + + #[cfg(test)] + #[allow(unused)] + pub(crate) fn debug(&self) { + dbg!(&self.tree); + dbg!(&self.style_ranges); + } + + #[inline(always)] + pub fn len_unicode(&self) -> usize { + self.tree.root_cache().unicode_len as usize + } + + #[inline(always)] + pub fn len_utf16(&self) -> usize { + self.tree.root_cache().utf16_len as usize + } + + #[inline(always)] + pub fn len_utf8(&self) -> usize { + self.tree.root_cache().bytes as usize + } + + #[inline(always)] + pub fn is_empty(&self) -> bool { + self.tree.root_cache().entity_len == 0 + } + + #[inline(always)] + pub fn len_entity(&self) -> usize { + self.tree.root_cache().entity_len as usize + } + + #[inline(always)] + pub(crate) fn content_node_len(&self) -> usize { + self.tree.node_len() + } + + pub fn diagnose(&self) { + println!( + "rope_nodes: {}, style_nodes: {}, text_len: {}", + self.tree.node_len(), + self.style_ranges + .as_ref() + .map(|x| x.tree.node_len()) + .unwrap_or(0), + self.tree.root_cache().bytes + ); + } + + /// Check if the content and style ranges are consistent. + /// + /// Panic if inconsistent. + pub(crate) fn check_consistency_between_content_and_style_ranges(&self) { + if !cfg!(debug_assertions) { + return; + } + + let mut entity_index_to_style_anchor: FxHashMap = + FxHashMap::default(); + let mut index = 0; + for c in self.iter_chunk() { + if matches!(c, RichtextStateChunk::Style { .. }) { + entity_index_to_style_anchor.insert(index, c); + } + + index += c.length() + } + + if let Some(s) = &self.style_ranges { + for IterAnchorItem { + index, + op, + anchor_type: iter_anchor_type, + } in s.iter_anchors() + { + let c = entity_index_to_style_anchor + .remove(&index) + .unwrap_or_else(|| { + panic!( + "Inconsistency found {} {:?} {:?}", + index, iter_anchor_type, &op + ); + }); + + match c { + RichtextStateChunk::Text(_) => { + unreachable!() + } + RichtextStateChunk::Style { style, anchor_type } => { + assert_eq!(style, &op); + assert_eq!(&iter_anchor_type, anchor_type); + } + } + } + } + + assert!( + entity_index_to_style_anchor.is_empty(), + "Inconsistency found. Some anchors are not reflected in style ranges {:#?}", + &entity_index_to_style_anchor + ); + } + + /// Allow StyleAnchors to appear in pairs, so that there won't be unmatched single StyleAnchors. + pub(crate) fn check_style_anchors_appear_in_pairs(&self) { + if !cfg!(any(debug_assertions, test)) { + return; + } + + let mut start_ops: FxHashSet<&Arc> = Default::default(); + for item in self.iter_chunk() { + match item { + RichtextStateChunk::Text(_) => {} + RichtextStateChunk::Style { style, anchor_type } => match anchor_type { + AnchorType::Start => { + start_ops.insert(style); + } + AnchorType::End => { + assert!(start_ops.remove(style), "End anchor without start anchor"); + } + }, + } + } + assert!( + start_ops.is_empty(), + "Only has start anchors {:#?}", + &start_ops + ); + } + + /// Iter style ranges in the given range in entity index + pub(crate) fn iter_range( + &self, + range: impl RangeBounds, + ) -> impl Iterator> + '_ { + self.check_cache(); + let result = { + let start = match range.start_bound() { + Bound::Included(x) => *x, + Bound::Excluded(x) => x + 1, + Bound::Unbounded => 0, + }; + let end = match range.end_bound() { + Bound::Included(x) => x + 1, + Bound::Excluded(x) => *x, + Bound::Unbounded => self.len_entity(), + }; + assert!(end > start); + assert!(end <= self.len_entity()); + + let mut style_iter = self + .style_ranges + .as_ref() + .map(|x| x.iter_range(range)) + .into_iter() + .flatten(); + + let start = self.tree.query::(&start).unwrap(); + let end = self.tree.query::(&end).unwrap(); + let mut content_iter = self.tree.iter_range(start.cursor..end.cursor); + let mut style_left_len = usize::MAX; + let mut cur_style = style_iter + .next() + .map(|x| { + style_left_len = x.elem.len - x.start.unwrap_or(0); + &x.elem.styles + }) + .unwrap_or(&*EMPTY_STYLES); + let mut chunk = content_iter.next(); + let mut offset = 0; + let mut chunk_left_len = chunk + .as_ref() + .map(|x| { + let len = x.elem.rle_len(); + offset = x.start.unwrap_or(0); + x.end.map(|v| v.min(len)).unwrap_or(len) - offset + }) + .unwrap_or(0); + std::iter::from_fn(move || { + if chunk_left_len == 0 { + chunk = content_iter.next(); + chunk_left_len = chunk + .as_ref() + .map(|x| { + let len = x.elem.rle_len(); + x.end.map(|v| v.min(len)).unwrap_or(len) + }) + .unwrap_or(0); + offset = 0; + } + + let iter_chunk = chunk.as_ref()?; + + let styles = cur_style; + let iter_len; + let event_range; + if chunk_left_len >= style_left_len { + iter_len = style_left_len; + event_range = iter_chunk + .elem + .entity_range_to_event_range(offset..offset + iter_len); + chunk_left_len -= style_left_len; + offset += style_left_len; + style_left_len = 0; + } else { + iter_len = chunk_left_len; + event_range = iter_chunk + .elem + .entity_range_to_event_range(offset..offset + iter_len); + style_left_len -= chunk_left_len; + chunk_left_len = 0; + } + + if style_left_len == 0 { + cur_style = style_iter + .next() + .map(|x| { + style_left_len = x.elem.len; + &x.elem.styles + }) + .unwrap_or(&*EMPTY_STYLES); + } + + Some(IterRangeItem { + chunk: iter_chunk.elem, + styles, + entity_len: iter_len, + event_len: event_range.len(), + }) + }) + }; + self.check_cache(); + result + } + + pub(crate) fn get_stable_position_at_event_index( + &self, + pos: usize, + kind: PosType, + ) -> Option { + self.check_cache(); + let result = { + let v = &self.get_text_entity_ranges(pos, 1, kind).unwrap(); + let a = v.first()?; + Some(a.id_start) + }; + self.check_cache(); + result + } + + pub(crate) fn len_event(&self) -> usize { + self.check_cache(); + let result = { + if cfg!(feature = "wasm") { + self.len_utf16() + } else { + self.len_unicode() + } + }; + self.check_cache(); + result + } + + pub(crate) fn len(&self, pos_type: PosType) -> usize { + self.check_cache(); + let result = { + match pos_type { + PosType::Unicode => self.len_unicode(), + PosType::Utf16 => self.len_utf16(), + PosType::Entity => self.len_entity(), + PosType::Event => self.len_event(), + PosType::Bytes => self.len_utf8(), + } + }; + self.check_cache(); + result + } +} + +fn entity_offset_to_pos_type_offset( + pos_type: PosType, + elem: &RichtextStateChunk, + offset: usize, +) -> usize { + match pos_type { + PosType::Bytes => match elem { + RichtextStateChunk::Text(t) => t.unicode_offset_to_utf8(offset).unwrap(), + RichtextStateChunk::Style { .. } => 0, + }, + PosType::Unicode => match elem { + RichtextStateChunk::Text(_) => offset, + RichtextStateChunk::Style { .. } => 0, + }, + PosType::Utf16 => match elem { + RichtextStateChunk::Text(t) => t.unicode_offset_to_utf16(offset).unwrap(), + RichtextStateChunk::Style { .. } => 0, + }, + PosType::Entity => offset, + PosType::Event => match elem { + RichtextStateChunk::Text(t) => { + if cfg!(feature = "wasm") { + t.unicode_offset_to_utf16(offset).unwrap() + } else { + offset + } + } + RichtextStateChunk::Style { .. } => 0, + }, + } +} + +fn pos_type_offset_to_entity_offset( + pos_type: PosType, + elem: &RichtextStateChunk, + offset: usize, +) -> Option { + match pos_type { + PosType::Bytes => match elem { + RichtextStateChunk::Text(t) => t.utf8_offset_to_unicode(offset).ok(), + RichtextStateChunk::Style { .. } => { + if offset > 0 { + None + } else { + Some(0) + } + } + }, + PosType::Unicode => Some(offset), + PosType::Utf16 => match elem { + RichtextStateChunk::Text(t) => t.utf16_offset_to_unicode(offset).ok(), + RichtextStateChunk::Style { .. } => { + if offset > 0 { + None + } else { + Some(0) + } + } + }, + PosType::Entity => { + if offset > elem.rle_len() { + None + } else { + Some(offset) + } + } + PosType::Event => match elem { + RichtextStateChunk::Text(t) => { + if cfg!(feature = "wasm") { + t.utf16_offset_to_unicode(offset).ok() + } else if offset < t.unicode_len() as usize { + Some(offset) + } else { + None + } + } + RichtextStateChunk::Style { .. } => { + if offset > 0 { + None + } else { + Some(0) + } + } + }, + } +} + +pub(crate) struct DrainInfo { + pub start_event_index: usize, + pub end_event_index: usize, + // entity range, event range + pub affected_style_range: Option<(Range, Range)>, +} + +pub(crate) struct IterRangeItem<'a> { + pub(crate) chunk: &'a RichtextStateChunk, + pub(crate) styles: &'a Styles, + pub(crate) entity_len: usize, + pub(crate) event_len: usize, +} + +use converter::ContinuousIndexConverter; +mod converter { + use generic_btree::{rle::HasLength, Cursor}; + + use super::{query::EntityQuery, RichtextState}; + + /// Convert entity index into event index. + /// It assumes the entity_index are in ascending order + pub(super) struct ContinuousIndexConverter<'a> { + state: &'a RichtextState, + last_entity_index_cache: Option, + } + + struct ConverterCache { + entity_index: usize, + cursor: Cursor, + event_index: usize, + cursor_elem_len: usize, + } + + impl<'a> ContinuousIndexConverter<'a> { + pub fn new(state: &'a RichtextState) -> Self { + Self { + state, + last_entity_index_cache: None, + } + } + + pub fn convert_entity_index_to_event_index(&mut self, entity_index: usize) -> usize { + self.state.check_cache(); + let result = { + if let Some(last) = self.last_entity_index_cache.as_ref() { + if last.entity_index == entity_index { + return last.event_index; + } + + assert!(entity_index > last.entity_index); + if last.cursor.offset + entity_index - last.entity_index < last.cursor_elem_len + { + // in the same cursor + return self.state.cursor_to_event_index(Cursor { + leaf: last.cursor.leaf, + offset: last.cursor.offset + entity_index - last.entity_index, + }); + } + } + + let cursor = self + .state + .tree + .query::(&entity_index) + .unwrap() + .cursor; + let ans = self.state.cursor_to_event_index(cursor); + let len = self.state.tree.get_elem(cursor.leaf).unwrap().rle_len(); + self.last_entity_index_cache = Some(ConverterCache { + entity_index, + cursor, + event_index: ans, + cursor_elem_len: len, + }); + ans + }; + self.state.check_cache(); + result + } + } +} + +#[cfg(test)] +mod test { + use append_only_bytes::AppendOnlyBytes; + use serde_json::json; + + use crate::{container::richtext::TextStyleInfoFlag, ToJson}; + + use super::*; + + #[derive(Debug, Default, Clone)] + struct SimpleWrapper { + state: RichtextState, + bytes: AppendOnlyBytes, + } + + impl SimpleWrapper { + fn insert(&mut self, pos: usize, text: &str) { + let start = self.bytes.len(); + self.bytes.push_str(text); + { + let state = &mut self.state; + let text = self.bytes.slice(start..); + let (entity_index, _) = state + .get_entity_index_for_text_insert(pos, PosType::Unicode) + .unwrap(); + state.insert_at_entity_index(entity_index, text, IdFull::new(0, 0, 0)); + }; + } + + fn delete(&mut self, pos: usize, len: usize) { + let ranges = self + .state + .get_text_entity_ranges(pos, len, PosType::Unicode) + .unwrap(); + for range in ranges.into_iter().rev() { + self.state.drain_by_entity_index( + range.entity_start, + range.entity_end - range.entity_start, + None, + ); + } + } + + fn mark(&mut self, range: Range, style: Arc) { + let (start, _) = self + .state + .get_entity_index_for_text_insert(range.start, PosType::Unicode) + .unwrap(); + let (end, _) = self + .state + .get_entity_index_for_text_insert(range.end, PosType::Unicode) + .unwrap(); + self.state.mark_with_entity_index(start..end, style); + } + } + + fn bold(n: isize) -> Arc { + Arc::new(StyleOp::new_for_test( + n, + "bold", + true.into(), + TextStyleInfoFlag::BOLD, + )) + } + + fn comment(n: isize) -> Arc { + Arc::new(StyleOp::new_for_test( + n, + &format!("comment:{}", n), + "comment".into(), + TextStyleInfoFlag::COMMENT, + )) + } + + fn unbold(n: isize) -> Arc { + Arc::new(StyleOp::new_for_test( + n, + "bold", + LoroValue::Null, + TextStyleInfoFlag::BOLD.to_delete(), + )) + } + + fn link(n: isize) -> Arc { + Arc::new(StyleOp::new_for_test( + n, + "link", + true.into(), + TextStyleInfoFlag::LINK, + )) + } + + #[test] + fn test() { + let mut wrapper = SimpleWrapper::default(); + wrapper.insert(0, "Hello World!"); + wrapper.mark(0..5, bold(0)); + assert_eq!( + wrapper.state.get_richtext_value().to_json_value(), + json!([ + { + "insert": "Hello", + "attributes": { + "bold": true + } + }, + { + "insert": " World!" + } + ]) + ); + wrapper.mark(2..7, link(1)); + assert_eq!( + wrapper.state.get_richtext_value().to_json_value(), + json!([ + { + "insert": "He", + "attributes": { + "bold": true + } + }, + { + "insert": "llo", + "attributes": { + "bold": true, + "link": true + } + }, + { + "insert": " W", + "attributes": { + "link": true + } + }, + { + "insert": "orld!" + } + + ]) + ); + } + + #[test] + fn delete_text() { + let mut wrapper = SimpleWrapper::default(); + wrapper.insert(0, "Hello World!"); + assert_eq!(wrapper.state.len_unicode(), 12); + assert_eq!(wrapper.state.len_entity(), 12); + wrapper.delete(0, 5); + assert_eq!( + wrapper.state.get_richtext_value().to_json_value(), + json!([ + { + "insert": " World!" + } + ]) + ); + + assert_eq!(wrapper.state.len_unicode(), 7); + assert_eq!(wrapper.state.len_entity(), 7); + wrapper.delete(1, 1); + + assert_eq!(wrapper.state.len_unicode(), 6); + assert_eq!(wrapper.state.len_entity(), 6); + assert_eq!( + wrapper.state.get_richtext_value().to_json_value(), + json!([ + { + "insert": " orld!" + } + ]) + ); + + wrapper.delete(5, 1); + assert_eq!(wrapper.state.len_unicode(), 5); + assert_eq!(wrapper.state.len_entity(), 5); + assert_eq!( + wrapper.state.get_richtext_value().to_json_value(), + json!([ + { + "insert": " orld" + } + ]) + ); + + wrapper.delete(0, 5); + assert_eq!(wrapper.state.len_unicode(), 0); + assert_eq!( + wrapper.state.get_richtext_value().to_json_value(), + json!([]) + ); + } + + #[test] + #[ignore] + fn insert_cache_hit() { + let mut wrapper = SimpleWrapper::default(); + wrapper.insert(0, "H"); + wrapper.insert(1, "H"); + dbg!(&wrapper); + wrapper.insert(2, "e"); + dbg!(&wrapper); + wrapper.insert(3, "k"); + wrapper.state.diagnose(); + } + + #[test] + fn bold_should_expand() { + let mut wrapper = SimpleWrapper::default(); + wrapper.insert(0, "Hello World!"); + wrapper.mark(0..5, bold(0)); + wrapper.insert(5, " Test"); + assert_eq!( + wrapper.state.get_richtext_value().to_json_value(), + json!([ + { + "insert": "Hello Test", + "attributes": { + "bold": true + } + }, + { + "insert": " World!" + } + ]) + ); + } + + #[test] + fn link_should_not_expand() { + let mut wrapper = SimpleWrapper::default(); + wrapper.insert(0, "Hello World!"); + wrapper.mark(0..5, link(0)); + wrapper.insert(5, " Test"); + assert_eq!( + wrapper.state.get_richtext_value().to_json_value(), + json!([ + { + "insert": "Hello", + "attributes": { + "link": true + } + }, + { + "insert": " Test World!" + }, + ]) + ); + } + + #[test] + fn continuous_text_insert_should_be_merged() { + let mut wrapper = SimpleWrapper::default(); + wrapper.insert(0, "Hello"); + wrapper.insert(5, " World!"); + assert_eq!( + wrapper.state.get_richtext_value().to_json_value(), + json!([ + { + "insert": "Hello World!" + }, + ]) + ); + } + + #[test] + fn continuous_text_insert_should_be_merged_and_have_bold() { + let mut wrapper = SimpleWrapper::default(); + wrapper.insert(0, "Hello"); + wrapper.mark(0..5, bold(0)); + wrapper.insert(5, " World!"); + dbg!(&wrapper.state); + assert_eq!( + wrapper.state.get_richtext_value().to_json_value(), + json!([ + { + "insert": "Hello World!", + "attributes": { + "bold": true + } + }, + ]) + ); + } + + #[test] + fn continuous_text_insert_should_not_be_merged_when_prev_is_link() { + let mut wrapper = SimpleWrapper::default(); + wrapper.insert(0, "Hello"); + wrapper.mark(0..5, link(0)); + wrapper.insert(5, " World!"); + assert_eq!( + wrapper.state.get_richtext_value().to_json_value(), + json!([ + { + "insert": "Hello", + "attributes": { + "link": true + } + }, + { + + "insert": " World!", + } + ]) + ); + } + + #[test] + fn delete_bold() { + let mut wrapper = SimpleWrapper::default(); + wrapper.insert(0, "Hello World!"); + wrapper.mark(0..12, bold(0)); + wrapper.mark(5..12, unbold(1)); + assert_eq!( + wrapper.state.get_richtext_value().to_json_value(), + json!([ + { + "insert": "Hello", + "attributes": { + "bold": true + } + }, + { + "insert": " World!" + } + ]) + ); + wrapper.insert(5, "A"); + assert_eq!( + wrapper.state.get_richtext_value().to_json_value(), + json!([ + { + "insert": "HelloA", + "attributes": { + "bold": true + } + }, + { + "insert": " World!", + } + ]) + ); + + wrapper.insert(0, "A"); + assert_eq!( + wrapper.state.get_richtext_value().to_json_value(), + json!([ + { + "insert": "A", + }, + { + "insert": "HelloA", + "attributes": { + "bold": true + } + }, + { + "insert": " World!", + } + ]) + ); + } + + #[test] + fn bold_and_link_at_the_same_place() { + let mut wrapper = SimpleWrapper::default(); + wrapper.insert(0, "Hello"); + wrapper.mark(0..5, link(0)); + wrapper.mark(0..5, bold(1)); + wrapper.insert(5, "A"); + assert_eq!( + wrapper.state.get_richtext_value().to_json_value(), + json!([ + { + "insert": "Hello", + "attributes": { + "bold": true, + "link": true + } + }, + { + "insert": "A", + "attributes": { + "bold": true, + } + }, + ]) + ); + } + + #[test] + fn test_comments() { + let mut wrapper = SimpleWrapper::default(); + wrapper.insert(0, "Hello World!"); + wrapper.mark(0..5, comment(0)); + wrapper.mark(1..6, comment(1)); + assert_eq!( + wrapper.state.get_richtext_value().to_json_value(), + json!([ + { + "insert": "H", + "attributes": { + "comment:0": "comment", + }, + }, + { + "insert": "ello", + "attributes": { + "comment:0": "comment", + "comment:1": "comment", + }, + }, + + { + "insert": " ", + "attributes": { + "comment:1": "comment", + }, + }, + { + "insert": "World!", + } + ]) + ); + } + + #[test] + fn remove_style_anchors_should_also_delete_style() { + let mut wrapper = SimpleWrapper::default(); + wrapper.insert(0, "Hello World!"); + wrapper.mark(0..5, bold(0)); + let mut count = 0; + wrapper.state.drain_by_entity_index( + 0, + 7, + Some(&mut |span| { + if matches!(span, RichtextStateChunk::Style { .. }) { + count += 1; + } + }), + ); + + assert_eq!(count, 2); + assert_eq!( + wrapper.state.get_richtext_value().to_json_value(), + json!([{ + "insert": " World!" + }]) + ); + } + + #[test] + fn remove_start_anchor_should_remove_style() { + let mut wrapper = SimpleWrapper::default(); + wrapper.insert(0, "Hello World!"); + wrapper.mark(0..5, bold(0)); + wrapper.state.drain_by_entity_index(6, 1, None); + wrapper.state.drain_by_entity_index(0, 1, None); + assert_eq!( + wrapper.state.get_richtext_value().to_json_value(), + json!([{ + "insert": "Hello World!" + }]) + ); + } + + #[test] + fn remove_start_anchor_in_the_middle_should_remove_style() { + let mut wrapper = SimpleWrapper::default(); + wrapper.insert(0, "Hello World!"); + wrapper.mark(2..5, bold(0)); + wrapper.state.drain_by_entity_index(6, 1, None); + wrapper.state.drain_by_entity_index(1, 2, None); + assert_eq!( + wrapper.state.get_richtext_value().to_json_value(), + json!([{ + "insert": "Hllo World!" + }]) + ); + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/container/richtext/str_slice.rs b/3rdparty/loro/crates/loro-internal/src/container/richtext/str_slice.rs new file mode 100644 index 0000000000..d118e497c8 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/container/richtext/str_slice.rs @@ -0,0 +1,44 @@ +use append_only_bytes::{AppendOnlyBytes, BytesSlice}; + +/// It's just a wrapper around [BytesSlice] that makes sure +/// the content of the bytes is a valid utf-8 string +pub(crate) struct StrSlice(BytesSlice); + +impl StrSlice { + #[allow(unused)] + pub fn new(bytes: BytesSlice) -> Option { + let _str = std::str::from_utf8(&bytes).ok()?; + Some(Self(bytes)) + } + + pub fn new_from_str(str: &str) -> Self { + let mut a = AppendOnlyBytes::new(); + a.push_str(str); + Self(a.slice(..)) + } + + pub fn bytes(&self) -> &BytesSlice { + &self.0 + } + + pub fn as_str(&self) -> &str { + // SAFETY: We ensure that the content is always valid utf8 + unsafe { std::str::from_utf8_unchecked(&self.0) } + } + + pub fn split_at_unicode_pos(&self, pos: usize) -> (Self, Self) { + let s = self.as_str(); + let mut split_at = self.0.len(); + for (u, (i, _)) in s.char_indices().enumerate() { + if u == pos { + split_at = i; + break; + } + } + + ( + Self(self.0.slice_clone(..split_at)), + Self(self.0.slice_clone(split_at..)), + ) + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/container/richtext/style_range_map.rs b/3rdparty/loro/crates/loro-internal/src/container/richtext/style_range_map.rs new file mode 100644 index 0000000000..f7487d4db1 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/container/richtext/style_range_map.rs @@ -0,0 +1,648 @@ +//! This map a Range to a set of style +//! + +use std::{ + collections::BTreeSet, + ops::{ControlFlow, Deref, DerefMut, Range, RangeBounds}, + sync::Arc, +}; + +use generic_btree::{ + rle::{CanRemove, HasLength, Mergeable, Sliceable, TryInsert}, + BTree, BTreeTrait, ElemSlice, LengthFinder, UseLengthFinder, +}; +use rustc_hash::FxHashMap; + +use once_cell::sync::Lazy; + +use crate::delta::StyleMeta; + +use super::{AnchorType, StyleKey, StyleOp}; + +/// This struct keep the mapping of ranges to numbers +/// +/// It's initialized with usize::MAX/2 length. +#[derive(Debug, Clone)] +pub(super) struct StyleRangeMap { + pub(super) tree: BTree, + has_style: bool, +} + +#[derive(Debug, Clone)] +pub(super) struct RangeNumMapTrait; + +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub(crate) struct Styles { + pub(crate) styles: FxHashMap, +} + +impl Styles { + pub(crate) fn has_key_value(&self, key: &str, value: &loro_common::LoroValue) -> bool { + match self.get(&StyleKey::Key(key.into())) { + Some(v) => match v.get() { + Some(v) => &v.value == value, + _ => false, + }, + _ => false, + } + } + + /// Infer the anchors between the neighbor styles. + /// Returns the last anchor of the left style and the first anchor of the right style. + fn infer_anchors(&self, next: &Self) -> (Option>, Option>) { + let mut left_anchor = None; + let mut right_anchor = None; + let empty_set: BTreeSet<_> = Default::default(); + for (key, set) in self.styles.iter() { + let right_set = next.styles.get(key).map(|x| &x.set).unwrap_or(&empty_set); + for diff in set.set.difference(right_set) { + assert!(left_anchor.is_none(), "left anchor should be unique"); + left_anchor = Some(diff.clone()); + } + } + + for (key, set) in next.styles.iter() { + let left_set = self.styles.get(key).map(|x| &x.set).unwrap_or(&empty_set); + for diff in set.set.difference(left_set) { + assert!(right_anchor.is_none(), "right anchor should be unique"); + right_anchor = Some(diff.clone()); + } + } + + (left_anchor, right_anchor) + } +} + +impl Deref for Styles { + type Target = FxHashMap; + + fn deref(&self) -> &Self::Target { + &self.styles + } +} + +impl DerefMut for Styles { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.styles + } +} + +pub(super) static EMPTY_STYLES: Lazy = Lazy::new(Default::default); + +#[derive(Debug, Clone)] +pub(crate) struct Elem { + pub(crate) styles: Styles, + pub(crate) len: usize, +} + +#[derive(Clone, Default, Debug, PartialEq, Eq)] +pub(crate) struct StyleValue { + // we need a set here because we need to calculate the intersection of styles when + // users insert new text between two style sets + set: BTreeSet>, +} + +impl StyleValue { + pub fn insert(&mut self, value: Arc) { + self.set.insert(value); + } + + pub fn get(&self) -> Option<&Arc> { + self.set.last() + } +} + +impl Default for StyleRangeMap { + fn default() -> Self { + Self::new() + } +} + +type YieldStyle<'a> = Option<&'a mut dyn FnMut(&Styles, usize)>; + +impl StyleRangeMap { + pub fn new() -> Self { + let mut tree = BTree::new(); + tree.push(Elem { + styles: Default::default(), + len: usize::MAX / 4, + }); + + Self { + tree, + has_style: false, + } + } + + pub fn annotate( + &mut self, + range: Range, + style: Arc, + mut yield_style: YieldStyle, + ) { + let range = self.tree.range::(range); + if range.is_none() { + unreachable!(); + } + + self.has_style = true; + let range = range.unwrap(); + self.tree + .update(range.start.cursor..range.end.cursor, &mut |x| { + if let Some(set) = x.styles.get_mut(&style.get_style_key()) { + set.set.insert(style.clone()); + } else { + let key = style.get_style_key(); + let mut value = StyleValue::default(); + value.insert(style.clone()); + x.styles.insert(key, value); + } + + if let Some(y) = yield_style.as_mut() { + y(&x.styles, x.len); + } + None + }); + } + + /// Get the styles of the range. If the range is not in the same leaf, return None. + pub(crate) fn get_styles_of_range(&self, range: Range) -> Option<&Styles> { + if !self.has_style { + return None; + } + + let right = self + .tree + .query::(&(range.end - 1)) + .unwrap() + .cursor; + let left = self + .tree + .query::(&range.start) + .unwrap() + .cursor; + if left.leaf == right.leaf { + Some(&self.tree.get_elem(left.leaf).unwrap().styles) + } else { + None + } + } + + pub(crate) fn range_contains_key(&self, range: Range, key: &StyleKey) -> bool { + if range.is_empty() || !self.has_style { + return false; + } + + let mut query = self.tree.query::(&range.start).unwrap(); + let mut pos = range.start; + loop { + let elem = self.tree.get_elem(query.cursor.leaf).unwrap(); + if elem.styles.contains_key(key) { + return true; + } + + let remaining_in_elem = elem.len - query.cursor.offset; + let next_pos = pos + remaining_in_elem; + if next_pos >= range.end { + break; + } + + match self.tree.next_elem(query.cursor) { + Some(next_cursor) => { + pos = next_pos; + query.cursor = next_cursor; + } + None => break, + } + } + + false + } + + /// Insert entities at `pos` with length of `len` + /// + /// # Internal + /// + /// When inserting new text, we need to calculate the StyleSet of the new text based on the StyleSet before and after the insertion position. + /// (It should be the intersection of the StyleSet before and after). The proof is as follows: + /// + /// Suppose when inserting text at position pos, the style set at positions pos - 1 and pos are called leftStyleSet and rightStyleSet respectively. + /// + /// - If there is a style x that exists in leftStyleSet but not in rightStyleSet, it means that the position pos - 1 is the end anchor of x. + /// The newly inserted text is after the end anchor of x, so the StyleSet of the new text should not include this style. + /// - If there is a style x that exists in rightStyleSet but not in leftStyleSet, it means that the position pos is the start anchor of x. + /// The newly inserted text is before the start anchor of x, so the StyleSet of the new text should not include this style. + /// - If both leftStyleSet and rightStyleSet contain style x, it means that the newly inserted text is within the style range, so the StyleSet should include x. + pub fn insert(&mut self, pos: usize, len: usize) -> &Styles { + if !self.has_style { + return &EMPTY_STYLES; + } + + if pos == 0 { + self.tree.prepend(Elem { + len, + styles: Default::default(), + }); + return &EMPTY_STYLES; + } + + if pos as isize == *self.tree.root_cache() { + self.tree.push(Elem { + len, + styles: Default::default(), + }); + return &EMPTY_STYLES; + } + + let right = self.tree.query::(&pos).unwrap().cursor; + let left = self.tree.query::(&(pos - 1)).unwrap().cursor; + if left.leaf == right.leaf { + // left and right are in the same element, we can increase the length of the element directly + self.tree.update_leaf(left.leaf, |x| { + x.len += len; + (true, None, None) + }); + return &self.tree.get_elem(left.leaf).unwrap().styles; + } + + // insert by the intersection of left styles and right styles + let mut styles = self.tree.get_elem(left.leaf).unwrap().styles.clone(); + let right_styles = &self.tree.get_elem(right.leaf).unwrap().styles; + styles.retain(|key, value| { + if let Some(right_value) = right_styles.get(key) { + value.set.retain(|x| right_value.set.contains(x)); + return !value.set.is_empty(); + } + + false + }); + + let (target, _) = self.tree.insert_by_path(right, Elem { len, styles }); + &self.tree.get_elem(target.leaf).unwrap().styles + } + + /// Return the style sets beside `index` and get the intersection of them. + pub fn get_styles_for_insert(&self, index: usize) -> StyleMeta { + if index == 0 || !self.has_style { + return StyleMeta::default(); + } + + let left = self + .tree + .query::(&(index - 1)) + .unwrap() + .cursor; + let right = self.tree.shift_path_by_one_offset(left).unwrap(); + if left.leaf == right.leaf { + let styles = &self.tree.get_elem(left.leaf).unwrap().styles; + styles.into() + } else { + let mut styles = self.tree.get_elem(left.leaf).unwrap().styles.clone(); + let right_styles = &self.tree.get_elem(right.leaf).unwrap().styles; + styles.retain(|key, value| { + if let Some(right_value) = right_styles.get(key) { + value.set.retain(|x| right_value.set.contains(x)); + return !value.set.is_empty(); + } + + false + }); + + styles.into() + } + } + + pub fn iter(&self) -> impl Iterator, &Styles)> + '_ { + let mut index = 0; + self.tree.iter().filter_map(move |elem| { + let len = elem.len; + let value = &elem.styles; + let range = index..index + len; + index += len; + if elem.styles.is_empty() { + return None; + } + + Some((range, value)) + }) + } + + /// Update the styles from `pos` to the start of the document. + fn update_styles_scanning_backward( + &mut self, + pos: usize, + mut f: impl FnMut(&mut Elem) -> ControlFlow<()>, + ) { + let mut cursor = self.tree.query::(&pos).map(|x| x.cursor); + while let Some(inner_cursor) = cursor { + cursor = self.tree.prev_elem(inner_cursor); + let node = self.tree.get_elem_mut(inner_cursor.leaf).unwrap(); + match f(node) { + ControlFlow::Continue(_) => {} + ControlFlow::Break(_) => { + break; + } + } + } + } + + pub(crate) fn iter_range( + &self, + range: impl RangeBounds, + ) -> impl Iterator> + '_ { + let start = match range.start_bound() { + std::ops::Bound::Included(x) => *x, + std::ops::Bound::Excluded(x) => *x + 1, + std::ops::Bound::Unbounded => 0, + }; + + let end = match range.end_bound() { + std::ops::Bound::Included(x) => *x + 1, + std::ops::Bound::Excluded(x) => *x, + std::ops::Bound::Unbounded => usize::MAX, + }; + + let start = self.tree.query::(&start).unwrap(); + let end = self.tree.query::(&end).unwrap(); + self.tree.iter_range(start.cursor..end.cursor) + } + + /// Return the expected style anchors with their indexes. + pub(super) fn iter_anchors(&self) -> impl Iterator + '_ { + let mut index = 0; + let empty_styles = &EMPTY_STYLES; + let mut last: Option<&Elem> = None; + let mut vec = Vec::new(); + for cur in self.tree.iter() { + let last_styles = last.map(|x| &x.styles).unwrap_or(empty_styles); + let (left_anchor, right_anchor) = last_styles.infer_anchors(&cur.styles); + if let Some(left) = left_anchor { + vec.push(IterAnchorItem { + index: index - 1, + op: left.clone(), + anchor_type: AnchorType::End, + }); + } + if let Some(right) = right_anchor { + vec.push(IterAnchorItem { + index, + op: right.clone(), + anchor_type: AnchorType::Start, + }); + } + + last = Some(cur); + index += cur.len; + } + + let last_styles = last.map(|x| &x.styles).unwrap_or(empty_styles); + let (left_anchor, right_anchor) = last_styles.infer_anchors(empty_styles); + if let Some(left) = left_anchor { + vec.push(IterAnchorItem { + index: index - 1, + op: left.clone(), + anchor_type: AnchorType::End, + }); + } + if let Some(right) = right_anchor { + vec.push(IterAnchorItem { + index, + op: right.clone(), + anchor_type: AnchorType::Start, + }); + } + + vec.into_iter() + } + + /// Remove the style scanning backward, return the start_entity_index + pub fn remove_style_scanning_backward( + &mut self, + to_remove: &Arc, + last_index: usize, + ) -> usize { + let mut removed_len = 0; + self.update_styles_scanning_backward(last_index, |elem| { + removed_len += elem.len; + let styles = &mut elem.styles; + let key = to_remove.get_style_key(); + let mut has_removed = false; + if let Some(value) = styles.get_mut(&key) { + has_removed = value.set.remove(to_remove); + if value.set.is_empty() { + styles.remove(&key); + } + } + + if has_removed { + ControlFlow::Continue(()) + } else { + ControlFlow::Break(()) + } + }); + + last_index + 1 - removed_len + } + + pub fn delete(&mut self, range: Range) { + if !self.has_style { + return; + } + + let start = self.tree.query::(&range.start).unwrap(); + let end = self.tree.query::(&range.end).unwrap(); + if start.cursor.leaf == end.cursor.leaf { + // delete in the same element + self.tree.update_leaf(start.cursor.leaf, |x| { + x.len -= range.len(); + (true, None, None) + }); + return; + } + + self.tree.drain(start..end); + } + + pub(crate) fn has_style(&self) -> bool { + self.has_style + } +} + +pub(super) struct IterAnchorItem { + pub(super) index: usize, + pub(super) op: Arc, + pub(super) anchor_type: AnchorType, +} + +impl UseLengthFinder for RangeNumMapTrait { + fn get_len(cache: &isize) -> usize { + *cache as usize + } +} + +impl HasLength for Elem { + fn rle_len(&self) -> usize { + self.len + } +} + +impl Mergeable for Elem { + fn can_merge(&self, rhs: &Self) -> bool { + self.styles == rhs.styles || rhs.len == 0 + } + + fn merge_right(&mut self, rhs: &Self) { + self.len += rhs.len + } + + fn merge_left(&mut self, left: &Self) { + self.len += left.len; + } +} + +impl Sliceable for Elem { + fn _slice(&self, range: std::ops::Range) -> Self { + let len = range.len(); + Elem { + styles: self.styles.clone(), + len, + } + } +} + +impl TryInsert for Elem { + fn try_insert(&mut self, _pos: usize, elem: Self) -> Result<(), Self> + where + Self: Sized, + { + if self.styles == elem.styles { + self.len += elem.len; + Ok(()) + } else { + Err(elem) + } + } +} + +impl CanRemove for Elem { + fn can_remove(&self) -> bool { + self.len == 0 + } +} + +impl BTreeTrait for RangeNumMapTrait { + type Elem = Elem; + type Cache = isize; + type CacheDiff = isize; + const USE_DIFF: bool = true; + + fn calc_cache_internal( + cache: &mut Self::Cache, + caches: &[generic_btree::Child], + ) -> isize { + let new_cache = caches.iter().map(|c| c.cache).sum(); + let diff = new_cache - *cache; + *cache = new_cache; + diff + } + + fn merge_cache_diff(diff1: &mut Self::CacheDiff, diff2: &Self::CacheDiff) { + *diff1 += diff2; + } + + fn apply_cache_diff(cache: &mut Self::Cache, diff: &Self::CacheDiff) { + *cache += diff; + } + + fn get_elem_cache(elem: &Self::Elem) -> Self::Cache { + elem.len as isize + } + + fn new_cache_to_diff(cache: &Self::Cache) -> Self::CacheDiff { + *cache + } + + fn sub_cache(cache_lhs: &Self::Cache, cache_rhs: &Self::Cache) -> Self::CacheDiff { + *cache_lhs - *cache_rhs + } +} + +#[cfg(test)] +mod test { + use loro_common::PeerID; + + use crate::{change::Lamport, container::richtext::TextStyleInfoFlag}; + + use super::*; + + fn new_style(n: i32) -> Arc { + Arc::new(StyleOp { + lamport: n as Lamport, + peer: n as PeerID, + cnt: n, + key: n.to_string().into(), + info: TextStyleInfoFlag::default(), + value: loro_common::LoroValue::Bool(true), + }) + } + + #[test] + fn test_basic_insert() { + let mut map = StyleRangeMap::default(); + map.annotate(1..10, new_style(1), None); + { + map.insert(0, 1); + assert_eq!(map.iter().count(), 1); + for (range, map) in map.iter() { + assert_eq!(range, 2..11); + assert_eq!(map.len(), 1); + } + } + { + map.insert(11, 1); + assert_eq!(map.iter().count(), 1); + for (range, map) in map.iter() { + assert_eq!(range, 2..11); + assert_eq!(map.len(), 1); + } + } + { + map.insert(10, 1); + assert_eq!(map.iter().count(), 1); + for (range, map) in map.iter() { + assert_eq!(range, 2..12); + assert_eq!(map.len(), 1); + } + } + } + + #[test] + fn delete_style() { + let mut map = StyleRangeMap::default(); + map.annotate(1..10, new_style(1), None); + { + map.delete(0..2); + assert_eq!(map.iter().count(), 1); + for (range, map) in map.iter() { + assert_eq!(range, 0..8); + assert_eq!(map.len(), 1); + } + } + { + map.delete(2..4); + for (range, map) in map.iter() { + assert_eq!(range, 0..6); + assert_eq!(map.len(), 1); + } + assert_eq!(map.iter().count(), 1); + } + { + map.delete(6..8); + assert_eq!(map.iter().count(), 1); + for (range, map) in map.iter() { + assert_eq!(range, 0..6); + assert_eq!(map.len(), 1); + } + } + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/container/richtext/tinyvec.rs b/3rdparty/loro/crates/loro-internal/src/container/richtext/tinyvec.rs new file mode 100644 index 0000000000..6f852fe1c5 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/container/richtext/tinyvec.rs @@ -0,0 +1,289 @@ +use std::{fmt::Debug, mem::MaybeUninit}; + +pub(crate) struct TinyVec { + len: u8, + data: [MaybeUninit; N], +} + +impl Debug for TinyVec { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_list().entries(self.iter()).finish() + } +} + +impl Clone for TinyVec { + fn clone(&self) -> Self { + let mut result = Self::new(); + for item in self.iter() { + result + .push(item.clone()) + .ok() + .expect("TinyVec clone should fit in the same capacity"); + } + result + } +} + +impl std::ops::Index for TinyVec { + type Output = T; + + fn index(&self, index: usize) -> &Self::Output { + assert!(index < self.len as usize); + // SAFETY: This is safe because we know that the data is initialized + unsafe { self.data[index].assume_init_ref() } + } +} + +impl TinyVec { + #[inline] + pub fn new() -> Self { + if N > u8::MAX as usize { + panic!("TinyVec size too large"); + } + + Self { + len: 0, + // SAFETY: This initialization is copied from std + // SAFETY: An uninitialized `[MaybeUninit<_>; LEN]` is valid. + data: unsafe { MaybeUninit::<[MaybeUninit; N]>::uninit().assume_init() }, + } + } + + pub fn push(&mut self, value: T) -> Result<(), T> { + if self.len == N as u8 { + return Err(value); + } + + self.data[self.len as usize] = MaybeUninit::new(value); + self.len += 1; + Ok(()) + } + + #[inline(always)] + pub fn get(&self, index: usize) -> &T { + &self[index] + } + + pub fn pop(&mut self) -> Option { + if self.len == 0 { + None + } else { + self.len -= 1; + // SAFETY: This is safe because we know that the last element is initialized + Some(unsafe { self.data[self.len as usize].assume_init_read() }) + } + } + + #[inline(always)] + pub fn len(&self) -> usize { + self.len as usize + } + + #[inline(always)] + pub fn is_empty(&self) -> bool { + self.len == 0 + } + + pub fn iter(&self) -> impl Iterator { + let mut index = 0; + std::iter::from_fn(move || { + if index < self.len as usize { + let result = &self[index]; + index += 1; + Some(result) + } else { + None + } + }) + } + + pub fn can_merge(&self, other: &Self) -> bool { + if self.len() + other.len() > N { + return false; + } + + self.len() + other.len() <= self.data.len() + } + + pub fn merge(&mut self, other: &Self) + where + T: Clone, + { + if !self.can_merge(other) { + panic!("TinyVec cannot merge"); + } + + for item in other.iter() { + self.push(item.clone()) + .ok() + .expect("TinyVec merge should fit after can_merge"); + } + } + + pub fn merge_left(&mut self, left: &Self) + where + T: Clone, + { + if !self.can_merge(left) { + panic!("TinyVec cannot merge"); + } + + let mut result = Self::new(); + for item in left.iter().chain(self.iter()) { + result + .push(item.clone()) + .ok() + .expect("TinyVec merge_left should fit after can_merge"); + } + *self = result; + } + + pub fn slice(&self, start: usize, end: usize) -> Self + where + T: Clone, + { + assert!(start <= end && end <= self.len as usize); + let mut result = Self::new(); + for item in self.iter().skip(start).take(end - start) { + result + .push(item.clone()) + .ok() + .expect("TinyVec slice should fit in the same capacity"); + } + result + } + + #[inline] + pub fn split(&mut self, pos: usize) -> Self { + assert!(pos <= self.len()); + let old_len = self.len(); + let mut result = Self::new(); + for i in pos..old_len { + // SAFETY: indexes below old_len are initialized. The value is moved + // into result and self.len is truncated below before self is dropped. + let value = unsafe { self.data[i].assume_init_read() }; + result + .push(value) + .ok() + .expect("TinyVec split should fit in the same capacity"); + } + self.len = pos as u8; + result + } +} + +impl TinyVec { + pub fn to_vec(&self) -> Vec { + let mut result = Vec::with_capacity(self.len as usize); + for i in 0..self.len as usize { + result.push(self[i].clone()); + } + result + } +} + +impl PartialEq for TinyVec { + fn eq(&self, other: &Self) -> bool { + if self.len != other.len { + return false; + } + + for i in 0..self.len as usize { + if self[i] != other[i] { + return false; + } + } + + true + } +} + +impl Eq for TinyVec {} + +impl Drop for TinyVec { + fn drop(&mut self) { + for i in 0..self.len as usize { + // SAFETY: This is safe because we know that the element is initialized + unsafe { std::ptr::drop_in_place(self.data[i].as_mut_ptr()) }; + } + } +} + +#[cfg(test)] +mod test { + use std::sync::Arc; + + use super::TinyVec; + + #[test] + fn pop_should_drop() { + let mut arr: TinyVec<_, 2> = TinyVec::new(); + let elem = Arc::new(10); + assert_eq!(Arc::strong_count(&elem), 1); + arr.push(elem.clone()).unwrap(); + assert_eq!(Arc::strong_count(&elem), 2); + arr.pop(); + assert_eq!(Arc::strong_count(&elem), 1); + } + + #[test] + fn dropping_should_drop_all_elem() { + let mut arr: TinyVec<_, 2> = TinyVec::new(); + let elem = Arc::new(10); + assert_eq!(Arc::strong_count(&elem), 1); + arr.push(elem.clone()).unwrap(); + assert_eq!(Arc::strong_count(&elem), 2); + drop(arr); + assert_eq!(Arc::strong_count(&elem), 1); + } + + #[test] + fn split() { + let mut arr: TinyVec = TinyVec::new(); + arr.push(1).unwrap(); + arr.push(2).unwrap(); + arr.push(3).unwrap(); + arr.push(4).unwrap(); + let new = arr.split(2); + assert_eq!(arr.to_vec(), vec![1, 2]); + assert_eq!(new.to_vec(), vec![3, 4]); + } + + #[test] + fn slice() { + let mut arr: TinyVec = TinyVec::new(); + arr.push(1).unwrap(); + arr.push(2).unwrap(); + arr.push(3).unwrap(); + arr.push(4).unwrap(); + arr.push(5).unwrap(); + let arr = arr; + let new = arr.slice(2, 4); + assert_eq!(new.to_vec(), vec![3, 4]); + } + + #[test] + fn clone() { + let mut arr: TinyVec = TinyVec::new(); + arr.push(1).unwrap(); + arr.push(2).unwrap(); + arr.push(3).unwrap(); + arr.push(4).unwrap(); + arr.push(5).unwrap(); + let arr = arr; + let new = arr.clone(); + assert_eq!(arr, new); + assert_eq!(new.to_vec(), [1, 2, 3, 4, 5]); + } + + #[cfg(miri)] + #[test] + fn clone_of_owned_values_does_not_duplicate_ownership_with_raw_copy() { + let mut arr: TinyVec, 2> = TinyVec::new(); + arr.push(Box::new(1)).unwrap(); + + let cloned = arr.clone(); + drop(arr); + drop(cloned); + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/container/richtext/tracker.rs b/3rdparty/loro/crates/loro-internal/src/container/richtext/tracker.rs new file mode 100644 index 0000000000..53e3fe89a6 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/container/richtext/tracker.rs @@ -0,0 +1,859 @@ +use std::ops::ControlFlow; + +use generic_btree::{ + rle::{HasLength as _, Sliceable}, + LeafIndex, +}; +use loro_common::{Counter, HasId, HasIdSpan, IdFull, IdSpan, Lamport, PeerID, ID}; +use rle::HasLength as _; +use tracing::instrument; + +use crate::{cursor::AbsolutePosition, VersionVector}; + +use self::{crdt_rope::CrdtRope, id_to_cursor::IdToCursor}; + +use super::{ + fugue_span::{FugueSpan, Status}, + RichtextChunk, +}; + +mod crdt_rope; +mod id_to_cursor; +pub(crate) use crdt_rope::CrdtRopeDelta; + +#[derive(Debug)] +pub(crate) struct Tracker { + applied_vv: VersionVector, + current_vv: VersionVector, + rope: CrdtRope, + id_to_cursor: IdToCursor, +} + +impl Default for Tracker { + fn default() -> Self { + Self::new_with_unknown() + } +} + +pub(super) const UNKNOWN_PEER_ID: PeerID = u64::MAX; +impl Tracker { + pub fn new_with_unknown() -> Self { + let mut this = Self { + rope: CrdtRope::new(), + id_to_cursor: IdToCursor::default(), + applied_vv: Default::default(), + current_vv: Default::default(), + }; + + let result = this.rope.tree.push(FugueSpan { + content: RichtextChunk::new_unknown(u32::MAX / 4), + id: IdFull::new(UNKNOWN_PEER_ID, 0, 0), + real_id: None, + status: Status::default(), + diff_status: None, + origin_left: None, + origin_right: None, + }); + this.id_to_cursor.insert_without_split( + ID::new(UNKNOWN_PEER_ID, 0), + id_to_cursor::Cursor::new_insert(result.leaf, u32::MAX as usize / 4), + ); + this + } + + #[allow(unused)] + fn new() -> Self { + Self::new_empty() + } + + pub(crate) fn new_empty() -> Self { + Self { + rope: CrdtRope::new(), + id_to_cursor: IdToCursor::default(), + applied_vv: Default::default(), + current_vv: Default::default(), + } + } + + #[inline] + pub fn all_vv(&self) -> &VersionVector { + &self.applied_vv + } + + #[inline] + pub fn current_vv(&self) -> &VersionVector { + &self.current_vv + } + + pub(crate) fn insert(&mut self, mut op_id: IdFull, mut pos: usize, mut content: RichtextChunk) { + // trace!( + // "TrackerInsert op_id = {:#?}, pos = {:#?}, content = {:#?}", + // op_id, + // &pos, + // &content + // ); + // tracing::span!(tracing::Level::INFO, "TrackerInsert"); + if let ControlFlow::Break(_) = + self.skip_applied(op_id.id(), content.len(), |applied_counter_end| { + // the op is partially included, need to slice the content + let start = (applied_counter_end - op_id.counter) as usize; + op_id.lamport += (applied_counter_end - op_id.counter) as Lamport; + op_id.counter = applied_counter_end; + pos += start; + content = content.slice(start..); + }) + { + return; + } + + // { + // tracing::span!(tracing::Level::INFO, "before insert {} pos={}", op_id, pos); + // debug_log::debug_dbg!(&self); + // } + self._insert(pos, content, op_id); + } + + fn _insert(&mut self, pos: usize, content: RichtextChunk, op_id: IdFull) { + self._insert_inner(pos, content, op_id, true); + } + + pub(crate) fn insert_seeded(&mut self, pos: usize, content: RichtextChunk, op_id: IdFull) { + self._insert_inner(pos, content, op_id, false); + } + + fn _insert_inner( + &mut self, + pos: usize, + content: RichtextChunk, + op_id: IdFull, + update_vv: bool, + ) { + let result = self.rope.insert( + pos, + FugueSpan { + content, + id: op_id, + real_id: if op_id.peer == UNKNOWN_PEER_ID { + None + } else { + Some(op_id.id().try_into().unwrap()) + }, + status: Status::default(), + diff_status: None, + origin_left: None, + origin_right: None, + }, + |id| self.id_to_cursor.get_insert(id).unwrap(), + ); + self.id_to_cursor.insert( + op_id.id(), + id_to_cursor::Cursor::new_insert(result.leaf, content.len()), + ); + + self.update_insert_by_split(&result.splitted.arr); + + if update_vv { + let end_id = op_id.inc(content.len() as Counter); + self.current_vv.extend_to_include_end_id(end_id.id()); + self.applied_vv.extend_to_include_end_id(end_id.id()); + } + } + + fn update_insert_by_split(&mut self, split: &[LeafIndex]) { + match split.len() { + 0 => {} + 1 => { + let new_leaf_idx = split[0]; + let leaf = self.rope.tree().get_elem(new_leaf_idx).unwrap(); + self.id_to_cursor + .update_insert(leaf.id_span(), new_leaf_idx); + } + _ => { + let mut updates = Vec::with_capacity(split.len()); + for &new_leaf_idx in split { + let leaf = self.rope.tree().get_elem(new_leaf_idx).unwrap(); + updates.push((leaf.id_span(), new_leaf_idx)); + } + self.id_to_cursor.update_insert_batch(&mut updates); + } + } + } + + /// Delete the element from pos..pos+len + /// + /// If `reverse` is true, the deletion happens from the end of the range to the start. + /// So the first op is the one that deletes element at `pos+len-1`, the last op + /// is the one that deletes element at `pos`. + /// + /// - op_id: the first op that perform the deletion + /// - target_start_id: in the target deleted span, it's the first id of the span + /// - pos: the start pos of the deletion in the target span + /// - len: the length of the deletion span + /// - reverse: if true, the kth op delete the last kth element of the span + pub(crate) fn delete( + &mut self, + mut op_id: ID, + mut target_start_id: ID, + pos: usize, + mut len: usize, + reverse: bool, + ) { + if let ControlFlow::Break(_) = self.skip_applied(op_id, len, |applied_counter_end: i32| { + // the op is partially included, need to slice the op + let start = (applied_counter_end - op_id.counter) as usize; + op_id.counter = applied_counter_end; + if !reverse { + target_start_id = target_start_id.inc(start as i32); + } + // Okay, this looks pretty weird, but it's correct. + // If it's reverse, we don't need to change the target_start_id, because target_start_id always pointing towards the + // leftmost element of the span. After applying the initial part of the deletion, which starts from the right side, + // the target_start_id will be still pointing towards the same leftmost element, thus no need to change. + len -= start; + // If reverse, don't need to change the pos, because it's deleting backwards. + // If not reverse, we don't need to change the pos either, because the `start` chars after it are already deleted + }) { + return; + } + + // tracing::info!("after forwarding pos={} len={}", pos, len); + + self._delete(target_start_id, pos, len, reverse, op_id); + } + + fn _delete(&mut self, target_start_id: ID, pos: usize, len: usize, reverse: bool, op_id: ID) { + let mut ans = Vec::new(); + let split = self + .rope + .delete(target_start_id, pos, len, reverse, &mut |span| { + let mut id_span = span.id_span(); + if reverse { + id_span.reverse(); + } + ans.push(id_span); + }); + + let mut cur_id = op_id; + for id_span in ans { + let len = id_span.atom_len(); + self.id_to_cursor + .insert(cur_id, id_to_cursor::Cursor::Delete(id_span)); + cur_id = cur_id.inc(len as Counter); + } + + debug_assert_eq!(cur_id.counter - op_id.counter, len as Counter); + for s in split { + self.update_insert_by_split(&s.arr); + } + + let end_id = op_id.inc(len as Counter); + self.current_vv.extend_to_include_end_id(end_id); + self.applied_vv.extend_to_include_end_id(end_id); + } + + fn skip_applied( + &mut self, + op_id: ID, + len: usize, + mut f: impl FnMut(Counter), + ) -> ControlFlow<()> { + let last_id = op_id.inc(len as Counter - 1); + let applied_counter_end = self.applied_vv.get(&last_id.peer).copied().unwrap_or(0); + if applied_counter_end > op_id.counter { + if !self.current_vv.includes_id(last_id) { + // PERF: may be slow + let mut updates = Default::default(); + let cnt_start = self.current_vv.get(&op_id.peer).copied().unwrap_or(0); + self.forward( + IdSpan::new(op_id.peer, cnt_start, op_id.counter + len as Counter), + &mut updates, + ); + self.batch_update(updates, false); + } + + if applied_counter_end > last_id.counter { + self.current_vv.extend_to_include_last_id(last_id); + return ControlFlow::Break(()); + } + + f(applied_counter_end); + } + ControlFlow::Continue(()) + } + + /// Internally it's delete at the src and insert at the dst. + /// + /// But it needs special behavior for id_to_cursor data structure + /// + #[instrument(skip(self))] + pub(crate) fn move_item( + &mut self, + op_id: IdFull, + deleted_id: ID, + from_pos: usize, + to_pos: usize, + ) { + if let ControlFlow::Break(_) = self.skip_applied(op_id.id(), 1, |_| unreachable!()) { + return; + } + + // We record the **fake** id of the deleted item, and store it in the `id_to_cursor`. + // This is because when we retreat, we need to know the **fake** id of the deleted item, + // so that we can look up the insert pos in `id_to_cursor` + // + // > `id_to_cursor` only stores the mappings from **fake** insert id to the leaf index. + // > **Fake** means the id may be a temporary placeholder, created with UNKNOWN_PEER_ID. + let mut fake_delete_id = None; + let split = self.rope.delete(deleted_id, from_pos, 1, false, &mut |s| { + debug_assert_eq!(s.rle_len(), 1); + fake_delete_id = Some(s.id.id()); + }); + + for s in split { + self.update_insert_by_split(&s.arr); + } + + let result = self.rope.insert( + to_pos, + FugueSpan { + // we need to use the special move content to avoid + // its merging with other [FugueSpan], which will make + // id_to_cursor need to track its split. + // It would be much harder to implement correctly + content: RichtextChunk::new_move(), + id: op_id, + real_id: if op_id.peer == UNKNOWN_PEER_ID { + None + } else { + Some(op_id.id().try_into().unwrap()) + }, + status: Status::default(), + diff_status: None, + origin_left: None, + origin_right: None, + }, + |id| self.id_to_cursor.get_insert(id).unwrap(), + ); + self.update_insert_by_split(&result.splitted.arr); + + self.id_to_cursor.insert( + op_id.id(), + id_to_cursor::Cursor::new_move(result.leaf, fake_delete_id.unwrap()), + ); + + let end_id = op_id.inc(1); + self.current_vv.extend_to_include_end_id(end_id.id()); + self.applied_vv.extend_to_include_end_id(end_id.id()); + } + + #[inline] + pub(crate) fn checkout(&mut self, vv: &VersionVector) { + self._checkout(vv, false); + } + + fn _checkout(&mut self, vv: &VersionVector, on_diff_status: bool) { + // tracing::info!("Checkout to {:?} from {:?}", vv, self.current_vv); + if on_diff_status { + self.rope.clear_diff_status(); + } + + let current_vv = std::mem::take(&mut self.current_vv); + let (retreat, forward) = current_vv.diff_iter(vv); + let mut updates = Vec::new(); + for span in retreat { + for c in self.id_to_cursor.iter(span) { + match c { + id_to_cursor::IterCursor::Insert { leaf, id_span } => { + updates.push(crdt_rope::LeafUpdate { + leaf, + id_span, + set_future: Some(true), + delete_times_diff: 0, + }) + } + id_to_cursor::IterCursor::Delete(span) => { + for to_del in self.id_to_cursor.iter(span) { + match to_del { + id_to_cursor::IterCursor::Insert { leaf, id_span } => { + updates.push(crdt_rope::LeafUpdate { + leaf, + id_span, + set_future: None, + delete_times_diff: -1, + }) + } + id_to_cursor::IterCursor::Move { + from_id: _, + to_leaf, + new_op_id, + } => updates.push(crdt_rope::LeafUpdate { + leaf: to_leaf, + id_span: new_op_id.to_span(1), + set_future: None, + delete_times_diff: -1, + }), + _ => unreachable!(), + } + } + } + id_to_cursor::IterCursor::Move { + from_id: from, + to_leaf: to, + new_op_id: op_id, + } => { + let mut visited = false; + for to_del in self.id_to_cursor.iter(IdSpan::new( + from.peer, + from.counter, + from.counter + 1, + )) { + visited = true; + + match to_del { + id_to_cursor::IterCursor::Move { + from_id: _, + to_leaf: to, + new_op_id: op_id, + } => updates.push(crdt_rope::LeafUpdate { + leaf: to, + id_span: op_id.to_span(1), + set_future: None, + delete_times_diff: -1, + }), + // Un delete the from + id_to_cursor::IterCursor::Insert { leaf, id_span } => { + debug_assert_eq!(id_span.atom_len(), 1); + debug_assert_eq!(id_span.counter.start, from.counter); + updates.push(crdt_rope::LeafUpdate { + leaf, + id_span, + set_future: None, + delete_times_diff: -1, + }) + } + _ => unreachable!(), + } + } + assert!(visited); + // insert the new + updates.push(crdt_rope::LeafUpdate { + leaf: to, + id_span: IdSpan::new(op_id.peer, op_id.counter, op_id.counter + 1), + set_future: Some(true), + delete_times_diff: 0, + }); + } + } + } + } + + for span in forward { + self.forward(span, &mut updates); + } + + if !on_diff_status { + self.current_vv = vv.clone(); + } else { + self.current_vv = current_vv; + } + + self.batch_update(updates, on_diff_status); + } + + fn batch_update(&mut self, updates: Vec, on_diff_status: bool) { + let leaf_indexes = self.rope.update(updates, on_diff_status); + self.update_insert_by_split(&leaf_indexes); + } + + fn forward(&mut self, span: loro_common::IdSpan, updates: &mut Vec) { + for c in self.id_to_cursor.iter(span) { + match c { + id_to_cursor::IterCursor::Insert { leaf, id_span } => { + updates.push(crdt_rope::LeafUpdate { + leaf, + id_span, + set_future: Some(false), + delete_times_diff: 0, + }) + } + id_to_cursor::IterCursor::Delete(span) => { + for to_del in self.id_to_cursor.iter(span) { + match to_del { + id_to_cursor::IterCursor::Insert { leaf, id_span } => { + updates.push(crdt_rope::LeafUpdate { + leaf, + id_span, + set_future: None, + delete_times_diff: 1, + }) + } + id_to_cursor::IterCursor::Move { + from_id: _, + to_leaf, + new_op_id, + } => updates.push(crdt_rope::LeafUpdate { + leaf: to_leaf, + id_span: new_op_id.to_span(1), + set_future: None, + delete_times_diff: 1, + }), + _ => unreachable!(), + } + } + } + id_to_cursor::IterCursor::Move { + from_id: from, + to_leaf: to, + new_op_id: op_id, + } => { + for to_del in self.id_to_cursor.iter(IdSpan::new( + from.peer, + from.counter, + from.counter + 1, + )) { + match to_del { + id_to_cursor::IterCursor::Move { + from_id: _, + to_leaf: to, + new_op_id: op_id, + } => updates.push(crdt_rope::LeafUpdate { + leaf: to, + id_span: op_id.to_span(1), + set_future: None, + delete_times_diff: 1, + }), + id_to_cursor::IterCursor::Insert { leaf, id_span } => { + updates.push(crdt_rope::LeafUpdate { + leaf, + id_span, + set_future: None, + delete_times_diff: 1, + }) + } + _ => unreachable!(), + } + } + + updates.push(crdt_rope::LeafUpdate { + leaf: to, + id_span: IdSpan::new(op_id.peer, op_id.counter, op_id.counter + 1), + set_future: Some(false), + delete_times_diff: 0, + }); + } + } + } + } + + #[allow(unused)] + pub(crate) fn check(&self) { + if !cfg!(debug_assertions) { + return; + } + + self.check_vv_correctness(); + self.check_id_to_cursor_insertions_correctness(); + } + + fn check_vv_correctness(&self) { + if !cfg!(debug_assertions) { + return; + } + + for span in self.rope.tree().iter() { + if span.id.peer == UNKNOWN_PEER_ID { + continue; + } + + let id_span = span.id_span(); + assert!(self.all_vv().includes_id(id_span.id_last())); + if span.status.future { + assert!(!self.current_vv.includes_id(id_span.id_start())); + } else { + assert!(self.current_vv.includes_id(id_span.id_last())); + } + } + } + + // It can only check the correctness of insertions in id_to_cursor. + // The deletions are not checked. + fn check_id_to_cursor_insertions_correctness(&self) { + if !cfg!(debug_assertions) { + return; + } + + for rope_elem in self.rope.tree().iter() { + let id_span = rope_elem.id_span(); + let leaf_from_start = self.id_to_cursor.get_insert(id_span.id_start()).unwrap(); + let leaf_from_last = self.id_to_cursor.get_insert(id_span.id_last()).unwrap(); + assert_eq!(leaf_from_start, leaf_from_last); + let elem_from_id_to_cursor_map = self.rope.tree().get_elem(leaf_from_last).unwrap(); + assert_eq!(rope_elem, elem_from_id_to_cursor_map); + } + + for content in self.id_to_cursor.iter_all() { + match content { + id_to_cursor::IterCursor::Insert { leaf, id_span } => { + let leaf = self.rope.tree().get_elem(leaf).unwrap(); + let span = leaf.id_span(); + span.contains(id_span.id_start()); + span.contains(id_span.id_last()); + } + id_to_cursor::IterCursor::Delete(_) => {} + id_to_cursor::IterCursor::Move { .. } => {} + } + } + } + + pub(crate) fn get_target_id_latest_index_at_new_version( + &self, + id: ID, + ) -> Option { + // TODO: PERF this can be sped up from O(n) to O(log(n)) but I'm not sure if it's worth it + let mut index = 0; + for span in self.rope.tree.iter() { + let is_activated = span.is_activated_in_diff(); + let span_id = span.real_id(); + let id_span = span_id.to_span(span.rle_len()); + if id_span.contains(id) { + if is_activated { + index += (id.counter - id_span.counter.start) as usize; + } + + return Some(AbsolutePosition { + pos: index, + side: if is_activated { + crate::cursor::Side::Middle + } else { + crate::cursor::Side::Left + }, + }); + } + + if is_activated { + index += span.rle_len(); + } + } + + None + } + + // #[tracing::instrument(skip(self), level = "info")] + pub(crate) fn diff( + &mut self, + from: &VersionVector, + to: &VersionVector, + ) -> impl Iterator + '_ { + // tracing::info!("Init: {:#?}, ", &self); + self._checkout(from, false); + self._checkout(to, true); + // self.id_to_cursor.diagnose(); + // tracing::trace!("Trace::diff {:#?}, ", &self); + + self.rope.get_diff() + } + + pub(crate) fn len(&self) -> usize { + self.rope.len() + } + + pub(crate) fn active_real_span_at(&self, pos: usize) -> Option<(ID, usize)> { + self.rope.active_real_span_at(pos) + } + + pub(crate) fn active_segments_of_real_id_span( + &self, + target_start: ID, + len: usize, + ) -> Vec<(ID, usize, usize)> { + let target_end = target_start.counter + len as Counter; + let mut index = 0; + let mut ans = Vec::new(); + + for span in self.rope.tree().iter() { + if !span.is_activated() { + continue; + } + + let span_len = span.rle_len(); + let real_id = span.real_id(); + if real_id.peer == UNKNOWN_PEER_ID { + index += span_len; + continue; + } + + if real_id.peer == target_start.peer { + let span_end = real_id.counter + span_len as Counter; + let overlap_start = real_id.counter.max(target_start.counter); + let overlap_end = span_end.min(target_end); + if overlap_start < overlap_end { + let offset = (overlap_start - real_id.counter) as usize; + let len = (overlap_end - overlap_start) as usize; + ans.push(( + ID::new(target_start.peer, overlap_start), + index + offset, + len, + )); + } + } + + index += span_len; + } + + ans + } + + pub(crate) fn mark_shallow_root_applied(&mut self, vv: &VersionVector) { + self.applied_vv = vv.clone(); + self.current_vv = vv.clone(); + } +} + +#[cfg(test)] +mod test { + use crate::{container::richtext::RichtextChunk, vv}; + use generic_btree::rle::HasLength; + + use super::*; + use std::time::Instant; + + #[test] + fn test_len() { + let mut t = Tracker::new(); + t.insert(IdFull::new(1, 0, 0), 0, RichtextChunk::new_text(0..2)); + assert_eq!(t.rope.len(), 2); + t.checkout(&Default::default()); + assert_eq!(t.rope.len(), 0); + t.insert(IdFull::new(2, 0, 0), 0, RichtextChunk::new_text(2..4)); + let v = vv!(1 => 2, 2 => 2); + t.checkout(&v); + assert_eq!(&t.applied_vv, &v); + assert_eq!(t.rope.len(), 4); + } + + #[test] + fn test_retreat_and_forward_delete() { + let mut t = Tracker::new(); + t.insert(IdFull::new(1, 0, 0), 0, RichtextChunk::new_text(0..10)); + t.delete(ID::new(2, 0), ID::NONE_ID, 0, 10, true); + t.checkout(&vv!(1 => 10, 2=>5)); + assert_eq!(t.rope.len(), 5); + t.checkout(&vv!(1 => 10, 2=>0)); + assert_eq!(t.rope.len(), 10); + t.checkout(&vv!(1 => 10, 2=>10)); + assert_eq!(t.rope.len(), 0); + t.checkout(&vv!(1 => 10, 2=>0)); + assert_eq!(t.rope.len(), 10); + } + + #[test] + fn repeated_tail_splits_keep_id_to_cursor_consistent() { + let mut t = Tracker::new(); + t.insert(IdFull::new(1, 0, 0), 0, RichtextChunk::new_text(0..300)); + + for (i, pos) in [100, 201, 252, 278].into_iter().enumerate() { + let op_id = IdFull::new(2, i as Counter, i as Lamport); + let start = 1000 + i as u32; + t.insert(op_id, pos, RichtextChunk::new_text(start..start + 1)); + } + + t.check(); + } + + #[test] + fn test_checkout_in_doc_with_del_span() { + let mut t = Tracker::new(); + t.insert(IdFull::new(1, 0, 0), 0, RichtextChunk::new_text(0..10)); + t.delete(ID::new(2, 0), ID::NONE_ID, 0, 10, false); + t.checkout(&vv!(1 => 10, 2=>4)); + let v: Vec = t.rope.tree().iter().copied().collect(); + assert_eq!(v.len(), 2); + assert!(!v[0].is_activated()); + assert_eq!(v[0].rle_len(), 4); + assert!(v[1].is_activated()); + assert_eq!(v[1].rle_len(), 6); + } + + #[test] + #[ignore] + fn perf_update_insert_by_split_quadratic() { + // Run with: + // cargo test -p loro-internal perf_update_insert_by_split_quadratic -- --ignored --nocapture + const CHUNK_LEN: usize = 256; + let fragments: usize = std::env::var("LORO_PERF_FRAGMENTS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(8192); + const PEER_A: PeerID = 1; + const PEER_B: PeerID = 2; + + let doc_len = CHUNK_LEN * fragments; + + let mut t = Tracker::new(); + t.insert( + IdFull::new(PEER_A, 0, 0), + 0, + RichtextChunk::new_text(0..doc_len as u32), + ); + t.id_to_cursor.diagnose(); + + let start = Instant::now(); + let expected_fragment_updates = (fragments as u64) * ((fragments - 1) as u64) / 2; + + for i in 0..(fragments - 1) { + let pos = (i + 1) * CHUNK_LEN + i; + let op_id = IdFull::new(PEER_B, i as Counter, i as Lamport); + let chunk = RichtextChunk::new_text( + (doc_len as u32 + i as u32)..(doc_len as u32 + i as u32 + 1), + ); + t.insert(op_id, pos, chunk); + } + + let elapsed = start.elapsed(); + let before_vv = vv!(PEER_A => doc_len as Counter); + let after_vv = vv!(PEER_A => doc_len as Counter, PEER_B => (fragments - 1) as Counter); + let diff_start = Instant::now(); + let diff_len = t.diff(&before_vv, &after_vv).count(); + let diff_elapsed = diff_start.elapsed(); + assert_eq!(t.rope.tree().iter().count(), 1 + 2 * (fragments - 1)); + println!( + "perf_update_insert_by_split_quadratic: doc_len={}, fragments={}, expected_fragment_updates={}, insert_elapsed={:?}, diff_items={}, diff_elapsed={:?}", + doc_len, fragments, expected_fragment_updates, elapsed, diff_len, diff_elapsed + ); + } + + #[test] + #[ignore] + fn perf_update_insert_by_split_quadratic_unknown() { + // Run with: + // LORO_PERF_FRAGMENTS=8192 cargo test -p loro-internal perf_update_insert_by_split_quadratic_unknown -- --ignored --nocapture + const CHUNK_LEN: usize = 256; + let fragments: usize = std::env::var("LORO_PERF_FRAGMENTS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(8192); + const PEER_B: PeerID = 2; + + let doc_len = CHUNK_LEN * fragments; + + let mut t = Tracker::new_with_unknown(); + t.checkout(&vv!()); + t.id_to_cursor.diagnose(); + + let start = Instant::now(); + for i in 0..(fragments - 1) { + let pos = (i + 1) * CHUNK_LEN + i; + let op_id = IdFull::new(PEER_B, i as Counter, i as Lamport); + let chunk = RichtextChunk::new_text( + (doc_len as u32 + i as u32)..(doc_len as u32 + i as u32 + 1), + ); + t.insert(op_id, pos, chunk); + } + + let elapsed = start.elapsed(); + println!( + "perf_update_insert_by_split_quadratic_unknown: doc_len={}, fragments={}, elapsed={:?}", + doc_len, fragments, elapsed + ); + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/container/richtext/tracker/crdt_rope.rs b/3rdparty/loro/crates/loro-internal/src/container/richtext/tracker/crdt_rope.rs new file mode 100644 index 0000000000..3e3576c5fc --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/container/richtext/tracker/crdt_rope.rs @@ -0,0 +1,953 @@ +use std::cmp::Ordering; + +use generic_btree::{ + rle::{CanRemove, HasLength, Sliceable}, + BTree, BTreeTrait, Cursor, FindResult, LeafIndex, Query, SplittedLeaves, +}; +use itertools::Itertools; +use loro_common::{Counter, HasCounter, HasCounterSpan, HasIdSpan, IdSpan, Lamport, ID}; +use smallvec::{smallvec, SmallVec}; + +use crate::container::richtext::{fugue_span::DiffStatus, FugueSpan, RichtextChunk, Status}; + +use super::UNKNOWN_PEER_ID; + +#[derive(Debug, Default, Clone)] +pub(super) struct CrdtRope { + pub(super) tree: BTree, +} + +pub(super) struct InsertResult { + #[allow(unused)] + pub content: FugueSpan, + pub leaf: LeafIndex, + pub splitted: SplittedLeaves, +} + +impl CrdtRope { + pub fn new() -> Self { + Self { tree: BTree::new() } + } + + #[inline(always)] + #[allow(unused)] + pub fn len(&self) -> usize { + self.tree.root_cache().len as usize + } + + #[inline(always)] + pub(super) fn tree(&self) -> &BTree { + &self.tree + } + + pub(super) fn active_real_span_at(&self, pos: usize) -> Option<(ID, usize)> { + if pos >= self.len() { + return None; + } + + let start = self + .tree + .query::(&(pos as i32))?; + let cursor = start.cursor; + let elem = self.tree.get_elem(cursor.leaf)?; + if !elem.is_activated() || cursor.offset >= elem.rle_len() { + return None; + } + + Some(( + elem.real_id().inc(cursor.offset as Counter), + elem.rle_len() - cursor.offset, + )) + } + + pub(super) fn insert( + &mut self, + pos: usize, + mut content: FugueSpan, + find_elem: impl Fn(ID) -> LeafIndex, + ) -> InsertResult { + if self.tree.is_empty() { + assert_eq!(pos, 0); + let leaf = self.tree.push(content).leaf; + return InsertResult { + content, + leaf, + splitted: SplittedLeaves::default(), + }; + } + + // tracing::span!(tracing::Level::INFO, "Inserting {} len={}", content.id, content.rle_len()); + + let pos = pos as i32; + let start = self.tree.query::(&pos).unwrap(); + + let (parent_right_leaf, in_between) = { + // calculate origin_left and origin_right + // origin_left is the alive op at `pos-1`, origin_right is the first non-future op between `pos-1` and `pos`. + + // `start` may point to a zero len node that's before the active index. + let origin_left = if start.cursor.offset == 0 { + // get left leaf node if offset == 0, so we can calculate the origin_left + if let Some(left) = self.tree.prev_elem(start.cursor) { + let left_node = self.tree.get_leaf(left.leaf.into()); + assert!(left_node.elem().rle_len() > 0); + Some( + left_node + .elem() + .id + .inc(left_node.elem().rle_len() as Counter - 1) + .id(), + ) + } else { + None + } + } else { + let left_node = self.tree.get_leaf(start.leaf().into()); + assert!(left_node.elem().rle_len() >= start.offset()); + Some(left_node.elem().id.inc(start.offset() as Counter - 1).id()) + }; + + let (origin_right, parent_right_leaf, in_between) = { + let mut in_between = Vec::new(); + let mut origin_right = None; + let mut parent_right_idx = None; + for iter in self.tree.iter_range(start.cursor..) { + if let Some(offset) = iter.start { + if offset >= iter.elem.rle_len() { + continue; + } + } + + if !iter.elem.status.future { + origin_right = + Some(iter.elem.id.inc(iter.start.unwrap_or(0) as Counter).id()); + let parent_right = match iter.start { + Some(offset) if offset > 0 => { + // It's guaranteed that origin_right's origin_left == this.origin_left. + // Because the first non-future node is just the leaf node in `start.cursor` node (iter.start.is_some()) + // Thus parent_right = origin_right + Some(origin_right) + } + _ => { + // Otherwise, we need to test whether origin_right's origin_left == this.origin_left + if iter.elem.origin_left.map(|x| x.to_id()) == origin_left { + Some(origin_right) + } else { + None + } + } + }; + parent_right_idx = parent_right.map(|_| iter.cursor().leaf); + break; + } + + // elem must be from future + in_between.push((iter.cursor().leaf, *iter.elem)); + } + + (origin_right, parent_right_idx, in_between) + }; + + content.origin_left = origin_left.map(|x| x.try_into().unwrap()); + content.origin_right = origin_right.map(|x| x.try_into().unwrap()); + (parent_right_leaf, in_between) + }; + + let mut insert_pos = start.cursor; + + if !in_between.is_empty() { + // find insert pos + let mut scanning = false; + let mut visited: SmallVec<[IdSpan; 4]> = Default::default(); + for (other_leaf, other_elem) in in_between.iter() { + // tracing::info!("Visiting {}", &other_elem.id); + let other_origin_left = other_elem.origin_left; + if other_origin_left != content.origin_left + && other_origin_left + .map(|left| visited.iter().all(|x| !x.contains_id(left.to_id()))) + .unwrap_or(true) + { + // The other_elem's origin_left must be at the left side of content's origin_left. + // So the content must be at the left side of other_elem. + + // tracing::info!("Break because the node's origin_left is at the left side of new_elem's origin left"); + break; + } + + visited.push(IdSpan::new( + other_elem.id.peer, + other_elem.id.counter, + other_elem.id.counter + other_elem.rle_len() as Counter, + )); + + if content.origin_left == other_origin_left { + if other_elem.origin_right == content.origin_right { + // tracing::info!("Same right parent"); + // Same right parent + if other_elem.id.peer > content.id.peer { + // tracing::info!("Break on larger peer"); + break; + } else { + scanning = false; + } + } else { + // tracing::info!("Different right parent"); + // Different right parent, we need to compare the right parents' position + + let other_parent_right_idx = + if let Some(other_origin_right) = other_elem.origin_right { + let elem_idx = find_elem(other_origin_right.to_id()); + let elem = self.tree.get_elem(elem_idx).unwrap(); + // It must be the start of the elem + assert_eq!(elem.id.id(), other_origin_right.to_id()); + if elem.origin_left == content.origin_left { + Some(elem_idx) + } else { + None + } + } else { + None + }; + + match self.cmp_pos(other_parent_right_idx, parent_right_leaf) { + Ordering::Less => { + // tracing::info!("Less"); + scanning = true; + } + Ordering::Equal if other_elem.id.peer > content.id.peer => { + // tracing::info!("Break on eq"); + break; + } + _ => { + // tracing::info!("Scanning"); + scanning = false; + } + } + } + } + + if !scanning { + insert_pos = Cursor { + leaf: *other_leaf, + offset: other_elem.rle_len(), + }; + // tracing::info!("updating insert pos {:?}", &insert_pos); + } + } + } + + // tracing::info!("Inserting at {:?}", insert_pos); + // + let (cursor, splitted) = self.tree.insert_by_path(insert_pos, content); + InsertResult { + content, + leaf: cursor.leaf, + splitted, + } + } + + /// Delete the span at `pos` with `len`. + /// + /// The `start_id` is used to mark the real_id in the [FugueSpan]. + /// + /// - If reversed is true, the deletion will be done in reversed order. + /// But the start_id always refers to the first delete op's id. + /// - If reversed is true, the returned `SplittedLeaves` will be in reversed order. + pub(super) fn delete( + &mut self, + mut start_id: ID, + pos: usize, + len: usize, + reversed: bool, + notify_deleted_span: &mut dyn FnMut(&FugueSpan), + ) -> SmallVec<[SplittedLeaves; 1]> { + if len == 0 { + return Default::default(); + } + + if reversed && len > 1 { + // TODO: PERF: optimize this + let mut ans = SmallVec::with_capacity(len); + for i in (0..len).rev() { + let a = self.delete( + start_id.inc(i as i32), + pos + i, + 1, + false, + notify_deleted_span, + ); + + ans.extend(a); + } + + return ans; + } + + let start = self + .tree + .query::(&(pos as i32)) + .unwrap(); + // avoid pointing to the end of the node + let start = start.cursor; + let elem = self.tree.get_elem_mut(start.leaf).unwrap(); + if elem.rle_len() >= start.offset + len { + // tracing::info!("len={} offset={} l={} ", elem.rle_len(), start.offset, len,); + let (_, splitted) = self.tree.update_leaf(start.leaf, |elem| { + let (a, b) = elem.update_with_split(start.offset..start.offset + len, |elem| { + assert!(elem.is_activated()); + debug_assert_eq!(len, elem.rle_len()); + notify_deleted_span(elem); + elem.status.delete_times += 1; + if elem.real_id.is_none() { + elem.real_id = Some(start_id.try_into().unwrap()); + } + + start_id = start_id.inc(elem.rle_len() as i32); + }); + + (true, a, b) + }); + + return smallvec![splitted]; + } + + let end = self + .tree + .query::(&((pos + len) as i32)) + .unwrap(); + smallvec![self.tree.update(start..end.cursor(), &mut |elem| { + if elem.is_activated() { + notify_deleted_span(elem); + elem.status.delete_times += 1; + if elem.real_id.is_none() { + elem.real_id = Some(start_id.try_into().unwrap()); + } + + start_id = start_id.inc(elem.rle_len() as i32); + Some(Cache { + len: -(elem.rle_len() as i32), + changed_num: 0, + }) + } else { + None + } + })] + } + + #[allow(unused)] + pub(crate) fn diagnose(&self) { + println!("crdt_rope number of tree nodes = {}", self.tree.node_len()); + } + + /// Update the leaf with given `id_span` + /// + /// Return the new leaf indexes that are created by splitting the old leaf nodes + pub(super) fn update( + &mut self, + mut updates: Vec, + on_diff_status: bool, + ) -> Vec { + updates.sort_by_key(|x| x.leaf); + let mut tree_update_info = Vec::with_capacity(updates.len()); + for (leaf, group) in &updates.into_iter().group_by(|x| x.leaf) { + let elem = self.tree.get_elem(leaf).unwrap(); + for u in group { + debug_assert_eq!(u.id_span.peer, elem.id.peer); + let start = (u.id_span.ctr_start() - elem.id.counter).max(0); + let end = (u.id_span.ctr_end() - elem.id.counter).max(0); + let len = elem.rle_len(); + + tree_update_info.push((leaf, (start as usize).min(len)..(end as usize).min(len), u)) + } + } + + self.tree + .update_leaves_with_arg_in_ranges(tree_update_info, |elem, arg| { + let status = if on_diff_status { + if elem.diff_status.is_none() { + elem.diff_status = Some(elem.status); + } + elem.diff_status.as_mut().unwrap() + } else { + &mut elem.status + }; + + if let Some(f) = arg.set_future { + status.future = f; + } + + status.delete_times += arg.delete_times_diff; + }) + } + + pub(super) fn clear_diff_status(&mut self) { + self.tree.update_cache_and_elem_with_filter( + |cache| { + let drill = cache.changed_num > 0; + cache.changed_num = 0; + drill + }, + |elem| { + elem.diff_status = None; + }, + ); + } + + pub(super) fn get_diff(&self) -> impl Iterator + '_ { + let mut last_pos = 0; + let mut iter = self + .tree + .iter_with_filter(|cache| (cache.changed_num > 0, cache.len)); + let mut next = None; + std::iter::from_fn(move || { + if let Some(next) = next.take() { + return Some(next); + } + + #[allow(clippy::while_let_on_iterator)] + while let Some((index, elem)) = iter.next() { + // The elements will not be changed by this method. + // This index is current index of the elem (calculated by `status` field rather than `diff_status` field) + match elem.diff() { + DiffStatus::NotChanged => {} + DiffStatus::Created => { + let rt = Some(CrdtRopeDelta::Insert { + chunk: elem.content, + id: elem.real_id.map(|x| x.to_id()).unwrap(), + lamport: if elem.id.peer == UNKNOWN_PEER_ID { + None + } else { + Some(elem.id.lamport) + }, + }); + if index > last_pos { + next = rt; + let len = index - last_pos; + // last pos = index, because the creation has not been applied to the elem + last_pos = index; + return Some(CrdtRopeDelta::Retain(len as usize)); + } else { + return rt; + } + } + DiffStatus::Deleted => { + let rt = Some(CrdtRopeDelta::Delete(elem.rle_len())); + if index > last_pos { + next = rt; + let len = index - last_pos; + // last pos = index + len, because the deletion has not been applied to the elem + last_pos = index + elem.rle_len() as i32; + return Some(CrdtRopeDelta::Retain(len as usize)); + } else { + last_pos = index + elem.rle_len() as i32; + return rt; + } + } + } + } + + None + }) + } + + fn cmp_pos( + &self, + parent_right: Option, + other_parent_right: Option, + ) -> Ordering { + match (parent_right, other_parent_right) { + (Some(a), Some(b)) => self + .tree + .compare_pos(Cursor { leaf: a, offset: 0 }, Cursor { leaf: b, offset: 0 }), + (Some(_), None) => Ordering::Less, + (None, Some(_)) => Ordering::Greater, + (None, None) => Ordering::Equal, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Copy)] +pub(crate) enum CrdtRopeDelta { + Retain(usize), + Insert { + chunk: RichtextChunk, + id: ID, + /// This is a optional field, because we may not know the correct lamport + /// for chunk id with UNKNOWN_PEER_ID. + /// + /// This case happens when the chunk is created by default placeholder and + /// the deletion happens that marks the chunk with its start_id. But it doesn't + /// know the correct lamport for the chunk. + lamport: Option, + }, + Delete(usize), +} + +#[derive(Debug, Clone, Copy)] +pub(super) struct CrdtRopeTrait; + +#[derive(Debug, Default, Clone, PartialEq, Eq, Copy)] +pub(super) struct Cache { + pub(super) len: i32, + pub(super) changed_num: i32, +} + +impl CanRemove for Cache { + fn can_remove(&self) -> bool { + self.len == 0 && self.changed_num == 0 + } +} + +impl BTreeTrait for CrdtRopeTrait { + type Elem = FugueSpan; + type Cache = Cache; + type CacheDiff = Cache; + const USE_DIFF: bool = true; + + #[inline(always)] + fn calc_cache_internal( + cache: &mut Self::Cache, + caches: &[generic_btree::Child], + ) -> Self::CacheDiff { + let new_len = caches.iter().map(|x| x.cache.len).sum(); + let new_changed_num = caches.iter().map(|x| x.cache.changed_num).sum(); + let len_diff = new_len - cache.len; + let changed_num_diff = new_changed_num - cache.changed_num; + cache.len = new_len; + cache.changed_num = new_changed_num; + Cache { + len: len_diff, + changed_num: changed_num_diff, + } + } + + #[inline(always)] + fn apply_cache_diff(cache: &mut Self::Cache, diff: &Self::CacheDiff) { + cache.len += diff.len; + cache.changed_num += diff.changed_num; + } + + #[inline(always)] + fn merge_cache_diff(diff1: &mut Self::CacheDiff, diff2: &Self::CacheDiff) { + diff1.len += diff2.len; + diff1.changed_num += diff2.changed_num; + } + + #[inline(always)] + fn get_elem_cache(elem: &Self::Elem) -> Self::Cache { + Cache { + len: elem.activated_len() as i32, + changed_num: if elem.diff_status.is_some() { 1 } else { 0 }, + } + } + + #[inline(always)] + fn new_cache_to_diff(cache: &Self::Cache) -> Self::CacheDiff { + *cache + } + + fn sub_cache(cache_lhs: &Self::Cache, cache_rhs: &Self::Cache) -> Self::CacheDiff { + Cache { + len: cache_lhs.len - cache_rhs.len, + changed_num: cache_lhs.changed_num - cache_rhs.changed_num, + } + } +} + +/// Query for start position, prefer left. +/// +/// If there are zero length spans (deleted, or spans from future) at the +/// active index, the query will return the position of the first non-zero length +/// content before them. +/// +/// NOTE: it may points to the end of a leaf node (with offset = rle_len) while the next leaf node is available. +struct ActiveLenQueryPreferLeft { + left: i32, +} + +impl Query for ActiveLenQueryPreferLeft { + type QueryArg = i32; + + fn init(target: &Self::QueryArg) -> Self { + debug_assert!(*target >= 0); + ActiveLenQueryPreferLeft { left: *target } + } + + fn find_node( + &mut self, + _: &Self::QueryArg, + child_caches: &[generic_btree::Child], + ) -> generic_btree::FindResult { + let mut left = self.left; + for (i, child) in child_caches.iter().enumerate() { + let cache = &child.cache; + if left <= cache.len { + // Prefer left. So if both `cache.len` and `left` equal zero, return here. + self.left = left; + return generic_btree::FindResult::new_found(i, left as usize); + } + + left -= cache.len; + } + + if let Some(last) = child_caches.last() { + left += last.cache.len; + self.left = left; + FindResult::new_missing(child_caches.len() - 1, left as usize) + } else { + // TODO: this should be impossible + unreachable!() + } + } + + fn confirm_elem( + &mut self, + _: &Self::QueryArg, + elem: &::Elem, + ) -> (usize, bool) { + if elem.is_activated() { + (self.left as usize, (self.left as usize) < elem.rle_len()) + } else { + // prefer left on zero length spans + (0, self.left == 0) + } + } +} + +/// Query for start position, prefer right +/// +/// If there are zero length spans (deleted, or spans from future) at the +/// active index, the query will return the position of the first non-zero length +/// content after them. +struct ActiveLenQueryPreferRight { + left: i32, +} + +impl Query for ActiveLenQueryPreferRight { + type QueryArg = i32; + + fn init(target: &Self::QueryArg) -> Self { + debug_assert!(*target >= 0); + Self { left: *target } + } + + fn find_node( + &mut self, + _: &Self::QueryArg, + child_caches: &[generic_btree::Child], + ) -> generic_btree::FindResult { + let mut left = self.left; + for (i, child) in child_caches.iter().enumerate() { + let cache = &child.cache; + if left < cache.len { + // Prefer left. So if both `cache.len` and `left` equal zero, return here. + self.left = left; + return generic_btree::FindResult::new_found(i, left as usize); + } + + left -= cache.len; + } + + if let Some(last) = child_caches.last() { + left += last.cache.len; + self.left = left; + FindResult::new_missing(child_caches.len() - 1, left as usize) + } else { + // TODO: this should be impossible + unreachable!() + } + } + + fn confirm_elem( + &mut self, + _: &Self::QueryArg, + elem: &::Elem, + ) -> (usize, bool) { + if elem.is_activated() { + (self.left as usize, (self.left as usize) < elem.rle_len()) + } else { + (self.left as usize, self.left == 0) + } + } +} + +/// This struct describe the ways to update a leaf's status +#[derive(Clone, Debug, Copy)] +pub(super) struct LeafUpdate { + pub leaf: LeafIndex, + /// `id_span` should only contains a subset of the leaf content + pub id_span: IdSpan, + /// if `set_future` is `None`, the `future` field will not be changed + pub set_future: Option, + pub delete_times_diff: i16, +} + +impl LeafUpdate { + #[allow(unused)] + fn apply_to(&self, s: &mut Status) { + s.delete_times += self.delete_times_diff; + if let Some(f) = self.set_future { + s.future = f; + } + } +} + +#[cfg(test)] +mod test { + use std::ops::Range; + + use loro_common::{CompactId, Counter, IdFull, PeerID, ID}; + + use crate::container::richtext::RichtextChunk; + + use super::*; + + fn span(id: u32, range: Range) -> FugueSpan { + FugueSpan::new( + IdFull::new(id as PeerID, 0 as Counter, 0), + RichtextChunk::new_text(range), + ) + } + + #[allow(unused)] + fn unknown_span(id: u32, len: usize) -> FugueSpan { + FugueSpan::new( + IdFull::new(id as PeerID, 0 as Counter, 0), + RichtextChunk::new_unknown(len as u32), + ) + } + + fn future_span(id: u32, range: Range) -> FugueSpan { + let mut fugue = FugueSpan::new( + IdFull::new(id as PeerID, 0 as Counter, 0), + RichtextChunk::new_text(range), + ); + + fugue.status.future = true; + fugue + } + + fn dead_span(id: u32, range: Range) -> FugueSpan { + let mut span = FugueSpan::new( + IdFull::new(id as PeerID, 0 as Counter, 0), + RichtextChunk::new_text(range), + ); + + span.status.delete_times += 1; + span + } + + #[test] + fn len_test() { + let mut rope = CrdtRope::new(); + rope.insert(0, span(0, 0..10), |_| panic!()); + assert_eq!(rope.len(), 10); + rope.insert(5, span(1, 0..10), |_| panic!()); + assert_eq!(rope.len(), 20); + rope.insert(20, span(1, 0..10), |_| panic!()); + assert_eq!(rope.len(), 30); + for i in 3..30 { + assert_eq!(rope.len(), i * 10); + rope.insert( + i, + span(i as u32, i as u32 * 10..(i as u32 + 1) * 10), + |_| panic!(), + ); + } + } + + #[test] + fn content_insert_middle() { + let mut rope = CrdtRope::new(); + rope.insert(0, span(0, 0..10), |_| panic!()); + rope.insert(5, span(1, 10..20), |_| panic!()); + let arr: Vec<_> = rope.tree.iter().collect(); + assert_eq!(arr.len(), 3); + assert_eq!(arr[0].rle_len(), 5); + assert_eq!(arr[1].rle_len(), 10); + assert_eq!(arr[2].rle_len(), 5); + + assert_eq!(arr[0].id.counter, 0); + assert_eq!(arr[0].id.peer, 0); + assert_eq!(arr[1].id.counter, 0); + assert_eq!(arr[1].id.peer, 1); + assert_eq!(arr[2].id.counter, 5); + assert_eq!(arr[2].id.peer, 0); + } + + #[test] + fn content_insert_should_ignore_tombstone() { + let mut rope = CrdtRope::new(); + rope.insert(0, span(0, 0..10), |_| panic!()); + // 0..10 + + rope.insert(5, dead_span(1, 10..20), |_| panic!()); + // 0..5, 10..20(dead), 5..10 + + rope.insert(10, span(0, 20..30), |_| panic!()); + // 0..5, 10..20(dead), 5..10, 20..30 + + let arr: Vec<_> = rope.tree.iter().collect(); + assert_eq!(arr.len(), 4); + assert_eq!(arr[0].rle_len(), 5); + assert_eq!(arr[1].rle_len(), 10); + assert_eq!(arr[1].activated_len(), 0); + assert_eq!(arr[2].rle_len(), 5); + assert_eq!(arr[3].rle_len(), 10); + } + + #[test] + fn get_origin_left_and_right() { + let mut rope = CrdtRope::new(); + rope.insert(0, span(0, 0..10), |_| panic!()); + let fugue = rope.insert(5, span(1, 10..20), |_| panic!()).content; + assert_eq!(fugue.origin_left, Some(CompactId::new(0, 4))); + assert_eq!(fugue.origin_right, Some(CompactId::new(0, 5))); + } + + #[test] + fn get_origin_left_and_right_among_tombstones() { + let mut rope = CrdtRope::new(); + rope.insert(0, span(0, 0..10), |_| panic!()); + assert_eq!(rope.len(), 10); + rope.delete(ID::NONE_ID, 5, 2, false, &mut |_| {}); + assert_eq!(rope.len(), 8); + let fugue = rope.insert(6, span(1, 10..20), |_| panic!()).content; + assert_eq!(fugue.origin_left, Some(CompactId::new(0, 7))); + assert_eq!(fugue.origin_right, Some(CompactId::new(0, 8))); + let fugue = rope.insert(5, span(1, 10..11), |_| panic!()).content; + assert_eq!(fugue.origin_left, Some(CompactId::new(0, 4))); + assert_eq!(fugue.origin_right, Some(CompactId::new(0, 5))); + } + + #[test] + fn should_ignore_future_spans_when_getting_origin_left() { + { + // insert future + let mut rope = CrdtRope::new(); + rope.insert(0, span(0, 0..10), |_| panic!()); + rope.insert(5, future_span(1, 10..20), |_| panic!()); + let fugue = rope.insert(5, span(1, 10..20), |_| panic!()).content; + assert_eq!(fugue.origin_left, Some(CompactId::new(0, 4))); + assert_eq!(fugue.origin_right, Some(CompactId::new(0, 5))); + } + { + // insert deleted + let mut rope = CrdtRope::new(); + rope.insert(0, span(0, 0..10), |_| panic!()); + rope.insert(5, dead_span(1, 10..20), |_| panic!()); + let fugue = rope.insert(5, span(1, 10..20), |_| panic!()).content; + assert_eq!(fugue.origin_left, Some(CompactId::new(0, 4))); + assert_eq!(fugue.origin_right, Some(CompactId::new(1, 0))); + } + } + + #[test] + fn update() { + let mut rope = CrdtRope::new(); + let result = rope.insert(0, span(0, 0..10), |_| panic!()); + let split = rope.update( + vec![LeafUpdate { + leaf: result.leaf, + id_span: IdSpan::new(0, 2, 8), + set_future: None, + delete_times_diff: 1, + }], + false, + ); + + assert_eq!(rope.len(), 4); + assert_eq!(split.len(), 2); + let split = rope.update( + vec![LeafUpdate { + leaf: split[0], + id_span: IdSpan::new(0, 2, 8), + set_future: None, + delete_times_diff: -1, + }], + false, + ); + + assert_eq!(rope.len(), 10); + assert_eq!(split.len(), 0); + } + + #[test] + fn checkout() { + let mut rope = CrdtRope::new(); + let result1 = rope.insert(0, span(0, 0..10), |_| panic!()); + let result2 = rope.insert(10, dead_span(1, 10..20), |_| panic!()); + rope.update( + vec![ + LeafUpdate { + leaf: result1.leaf, + id_span: IdSpan::new(0, 2, 8), + set_future: None, + delete_times_diff: 1, + }, + LeafUpdate { + leaf: result2.leaf, + id_span: IdSpan::new(1, 0, 3), + set_future: None, + delete_times_diff: -1, + }, + ], + true, + ); + let vec: Vec<_> = rope.get_diff().collect(); + assert_eq!( + vec![ + CrdtRopeDelta::Retain(2), + CrdtRopeDelta::Delete(6), + CrdtRopeDelta::Retain(2), + CrdtRopeDelta::Insert { + chunk: RichtextChunk::new_text(10..13), + id: ID::new(1, 0), + lamport: Some(0) + } + ], + vec, + ); + } + + #[test] + fn checkout_future() { + let mut rope = CrdtRope::new(); + let result = rope.insert(0, future_span(0, 0..10), |_| panic!()); + rope.update( + vec![LeafUpdate { + leaf: result.leaf, + id_span: IdSpan::new(0, 2, 10), + set_future: Some(false), + delete_times_diff: 0, + }], + true, + ); + let vec: Vec<_> = rope.get_diff().collect(); + assert_eq!( + vec![CrdtRopeDelta::Insert { + chunk: RichtextChunk::new_text(2..10), + id: ID::new(0, 2), + lamport: Some(2) + }], + vec, + ); + } + + #[test] + fn checkout_future_with_delete() { + let mut rope = CrdtRope::new(); + let result = rope.insert(0, future_span(0, 0..10), |_| panic!()); + rope.update( + vec![LeafUpdate { + leaf: result.leaf, + id_span: IdSpan::new(0, 2, 10), + set_future: Some(false), + delete_times_diff: 1, + }], + true, + ); + let vec: Vec<_> = rope.get_diff().collect(); + assert!(vec.is_empty()); + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/container/richtext/tracker/id_to_cursor.rs b/3rdparty/loro/crates/loro-internal/src/container/richtext/tracker/id_to_cursor.rs new file mode 100644 index 0000000000..d716069a30 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/container/richtext/tracker/id_to_cursor.rs @@ -0,0 +1,1222 @@ +use generic_btree::{ + rle::{HasLength, Mergeable}, + LeafIndex, +}; +use loro_common::{Counter, IdSpan, PeerID, ID}; +use rle::{HasLength as RHasLength, Mergable as RMergeable, Sliceable}; +use rustc_hash::FxHashMap; +use smallvec::smallvec; + +use self::insert_set::InsertSet; + +// If we make this too large, we may have too many cursors inside a fragment +// and trigger the worst case +const MAX_FRAGMENT_LEN: usize = 256; +#[cfg(not(test))] +const SMALL_SET_MAX_LEN: usize = 32; +#[cfg(test)] +const SMALL_SET_MAX_LEN: usize = 4; + +/// This struct maintains the mapping of Op `ID` to +/// +/// - `LeafIndex` from crdt_rope, if the Op is an Insert +/// - The IdSpan deleted by the Op, if the Op is a Delete +#[derive(Debug, Default)] +pub(super) struct IdToCursor { + map: FxHashMap>, +} + +static EMPTY_VEC: Vec = vec![]; +impl IdToCursor { + pub fn insert_without_split(&mut self, id: ID, cursor: Cursor) { + let list = self.map.entry(id.peer).or_default(); + list.push(Fragment { + counter: id.counter, + cursor, + }); + } + + pub fn insert(&mut self, id: ID, cursor: Cursor) { + let list = self.map.entry(id.peer).or_default(); + if let Some(last) = list.last_mut() { + let last_end = last.counter + last.cursor.rle_len() as Counter; + debug_assert!(last_end <= id.counter, "id:{}, {:#?}", id, &self); + if last_end == id.counter + && last.cursor.can_merge(&cursor) + && last.cursor.rle_len() + cursor.rle_len() < MAX_FRAGMENT_LEN + { + last.cursor.merge_right(&cursor); + return; + } + } + + if let Cursor::Insert(InsertSet::Small(set)) = cursor { + if set.len > MAX_FRAGMENT_LEN as u32 { + assert!(set.set.len() == 1); + let insert = set.set[0]; + let mut counter = id.counter; + for start in (0..set.len).step_by(MAX_FRAGMENT_LEN) { + let end = (start + MAX_FRAGMENT_LEN as u32).min(set.len); + let len = (end - start) as usize; + list.push(Fragment { + counter, + cursor: Cursor::new_insert(insert.leaf, len), + }); + counter += len as Counter; + } + } else { + list.push(Fragment { + counter: id.counter, + cursor: Cursor::Insert(InsertSet::Small(set)), + }); + } + } else { + list.push(Fragment { + counter: id.counter, + cursor, + }); + } + } + + /// Update the given id_span to the new_leaf + /// + /// id_span should be within the same `Cursor` and should be a `Insert` + pub fn update_insert(&mut self, id_span: IdSpan, new_leaf: LeafIndex) { + debug_assert!(!id_span.is_reversed()); + let list = self.map.get_mut(&id_span.peer).unwrap(); + let last = list.last().unwrap(); + debug_assert!(last.counter + last.cursor.rle_len() as Counter > id_span.counter.max()); + let mut index = match list.binary_search_by_key(&id_span.counter.start, |x| x.counter) { + Ok(index) => index, + Err(index) => index.saturating_sub(1), + }; + + let mut start_counter = id_span.counter.start; + while start_counter < id_span.counter.end + && index < list.len() + && start_counter < list[index].counter_end() + { + let fragment = &mut list[index]; + let from = (start_counter - fragment.counter) as usize; + let to = + ((id_span.counter.end - fragment.counter) as usize).min(fragment.cursor.rle_len()); + + fragment.cursor.update_insert(from, to, new_leaf); + start_counter += (to - from) as Counter; + index += 1; + } + + assert_eq!(start_counter, id_span.counter.end); + } + + pub fn update_insert_batch(&mut self, updates: &mut [(IdSpan, LeafIndex)]) { + if updates.is_empty() { + return; + } + + if updates.len() == 1 { + let (span, leaf) = updates[0]; + self.update_insert(span, leaf); + return; + } + + let mut per_peer: FxHashMap> = FxHashMap::default(); + for &(id_span, new_leaf) in updates.iter() { + per_peer + .entry(id_span.peer) + .or_default() + .push((id_span, new_leaf)); + } + + for (peer, peer_updates) in per_peer { + let Some(list) = self.map.get_mut(&peer) else { + continue; + }; + + let mut per_fragment: FxHashMap> = + FxHashMap::default(); + + for (id_span, new_leaf) in peer_updates { + debug_assert!(!id_span.is_reversed()); + let mut index = + match list.binary_search_by_key(&id_span.counter.start, |x| x.counter) { + Ok(index) => index, + Err(index) => index.saturating_sub(1), + }; + + let mut start_counter = id_span.counter.start; + while start_counter < id_span.counter.end + && index < list.len() + && start_counter < list[index].counter_end() + { + let fragment = &list[index]; + let from = (start_counter - fragment.counter) as usize; + let to = ((id_span.counter.end - fragment.counter) as usize) + .min(fragment.cursor.rle_len()); + + if from != to { + per_fragment + .entry(index) + .or_default() + .push((from, to, new_leaf)); + } + + start_counter += (to - from) as Counter; + index += 1; + } + + assert_eq!(start_counter, id_span.counter.end); + } + + let mut fragment_indexes: Vec = per_fragment.keys().copied().collect(); + fragment_indexes.sort_unstable(); + for index in fragment_indexes { + let updates = per_fragment.get(&index).unwrap(); + list[index].cursor.update_insert_many(updates); + } + } + } + + pub fn iter_all(&self) -> impl Iterator + '_ { + self.map.iter().flat_map(|(peer, list)| { + list.iter() + .flat_map(move |f| -> Box> { + match &f.cursor { + Cursor::Insert(set) => set.iter_all(*peer, f.counter), + Cursor::Delete(span) => { + let start_counter = f.counter; + let end_counter = f.counter + span.atom_len() as Counter; + let id_span = IdSpan::new(*peer, start_counter, end_counter); + Box::new(std::iter::once(IterCursor::Delete(id_span))) + } + Cursor::Move { from, to } => Box::new(std::iter::once(IterCursor::Move { + from_id: *from, + to_leaf: *to, + new_op_id: ID::new(*peer, f.counter), + })), + } + }) + }) + } + + pub fn iter(&self, mut iter_id_span: IdSpan) -> impl Iterator + '_ { + iter_id_span.normalize_(); + let list = self.map.get(&iter_id_span.peer).unwrap_or(&EMPTY_VEC); + // Index in the list + let mut index = 0; + let mut insert_set_iter: Option>> = None; + + if !list.is_empty() { + index = match list.binary_search_by_key(&iter_id_span.counter.start, |x| x.counter) { + Ok(index) => index, + Err(index) => index.saturating_sub(1), + }; + } + + std::iter::from_fn(move || loop { + if index >= list.len() { + return None; + } + + // Always iterate the insert set's iterator first if it exists + // It's on the top of the stack + if let Some(iter) = insert_set_iter.as_mut() { + let Some(next) = iter.next() else { + index += 1; + insert_set_iter = None; + continue; + }; + + return Some(next); + } + + let f = &list[index]; + let iter_counter = f.counter; + if iter_counter >= iter_id_span.counter.end { + return None; + } + + if iter_counter + f.cursor.rle_len() as Counter <= iter_id_span.counter.start { + index += 1; + continue; + } + + match &f.cursor { + Cursor::Insert(set) => { + insert_set_iter = Some(set.iter_range( + ID::new(iter_id_span.peer, iter_counter), + iter_id_span.counter.start, + iter_id_span.counter.end, + )); + continue; + } + Cursor::Delete(span) => { + index += 1; + let from = (iter_id_span.counter.start - iter_counter) + .max(0) + .min(span.atom_len() as Counter); + let to = (iter_id_span.counter.end - iter_counter) + .max(0) + .min(span.atom_len() as Counter); + if from == to { + continue; + } + + return Some(IterCursor::Delete(span.slice(from as usize, to as usize))); + } + Cursor::Move { from, to } => { + index += 1; + let op_id = ID::new(iter_id_span.peer, f.counter); + return Some(IterCursor::Move { + from_id: *from, + to_leaf: *to, + new_op_id: op_id, + }); + } + } + }) + } + + pub fn get_insert(&self, id: ID) -> Option { + let list = self.map.get(&id.peer)?; + let index = match list.binary_search_by_key(&id.counter, |x| x.counter) { + Ok(index) => index, + Err(index) => index - 1, + }; + + list[index] + .cursor + .get_insert((id.counter - list[index].counter) as usize) + } + + #[allow(unused)] + pub fn diagnose(&self) { + let fragment_num = self.map.iter().map(|x| x.1.len()).sum::(); + let insert_pieces = self + .map + .iter() + .flat_map(|x| x.1.iter()) + .map(|x| match &x.cursor { + Cursor::Insert(set) => set.len(), + Cursor::Delete(_) => 0, + Cursor::Move { .. } => 0, + }) + .sum::(); + let max_insert_len = self + .map + .iter() + .map(|x| { + x.1.iter() + .filter_map(|x| match &x.cursor { + Cursor::Insert(set) => Some(set.len()), + _ => None, + }) + .max() + .unwrap_or(0) + }) + .max() + .unwrap_or(0); + println!( + "fragments:{}, insert_pieces:{}, max_insert_len:{}", + fragment_num, insert_pieces, max_insert_len + ); + } +} + +#[derive(Debug)] +pub(super) struct Fragment { + pub(super) counter: Counter, + pub(super) cursor: Cursor, +} + +impl PartialEq for Fragment { + fn eq(&self, other: &Self) -> bool { + self.counter == other.counter + } +} + +impl Eq for Fragment {} + +impl PartialOrd for Fragment { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for Fragment { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.counter.cmp(&other.counter) + } +} + +impl Fragment { + fn counter_end(&self) -> Counter { + self.counter + self.cursor.rle_len() as Counter + } +} + +#[derive(Debug, Clone, Copy)] +pub(super) enum IterCursor { + Insert { + leaf: LeafIndex, + id_span: IdSpan, + }, + // deleted id_span, the start may be greater than the end + Delete(IdSpan), + // Move from `from ID` to `to LeafIndex` with `op_id` + Move { + from_id: ID, + to_leaf: LeafIndex, + new_op_id: ID, + }, +} + +#[derive(Debug)] +pub(super) enum Cursor { + Insert(InsertSet), + Delete(IdSpan), + Move { + from: ID, + to: LeafIndex, + // to id is the same as the current op_id + }, +} + +mod insert_set { + + use super::*; + use generic_btree::{ + rle::{CanRemove, TryInsert}, + BTree, BTreeTrait, LengthFinder, UseLengthFinder, + }; + use smallvec::SmallVec; + + #[derive(Debug)] + pub(crate) enum InsertSet { + Small(SmallInsertSet), + Large(LargeInsertSet), + } + + impl InsertSet { + pub(crate) fn new(leaf: LeafIndex, len: usize) -> Self { + Self::Small(SmallInsertSet { + set: smallvec![Insert { + leaf, + len: len as u32 + }], + len: len as u32, + }) + } + + pub(crate) fn update(&mut self, from: usize, to: usize, new_leaf: LeafIndex) { + self.upgrade_if_needed(); + match self { + Self::Small(set) => { + set.update(from, to, new_leaf); + } + Self::Large(set) => { + set.update(from, to, new_leaf); + } + } + } + + pub(crate) fn update_many(&mut self, updates: &[(usize, usize, LeafIndex)]) { + if updates.is_empty() { + return; + } + + if updates.len() == 1 { + let (from, to, leaf) = updates[0]; + self.update(from, to, leaf); + return; + } + + let len = self.len(); + if len > MAX_FRAGMENT_LEN { + for &(from, to, leaf) in updates { + self.update(from, to, leaf); + } + return; + } + + let mut dense: SmallVec<[LeafIndex; MAX_FRAGMENT_LEN]> = SmallVec::with_capacity(len); + match self { + InsertSet::Small(set) => { + for insert in set.set.iter() { + for _ in 0..insert.len { + dense.push(insert.leaf); + } + } + } + InsertSet::Large(set) => { + for insert in set.tree.iter() { + for _ in 0..insert.len { + dense.push(insert.leaf); + } + } + } + } + debug_assert_eq!(dense.len(), len); + + for &(from, to, leaf) in updates { + debug_assert!(from <= to); + debug_assert!(to <= len); + for i in from..to { + dense[i] = leaf; + } + } + + let mut new_set: SmallVec<[Insert; 1]> = SmallVec::new(); + if !dense.is_empty() { + let mut cur_leaf = dense[0]; + let mut cur_len: usize = 1; + for &leaf in dense.iter().skip(1) { + if leaf == cur_leaf { + cur_len += 1; + } else { + new_set.push(Insert { + leaf: cur_leaf, + len: cur_len as u32, + }); + cur_leaf = leaf; + cur_len = 1; + } + } + new_set.push(Insert { + leaf: cur_leaf, + len: cur_len as u32, + }); + } + + *self = InsertSet::Small(SmallInsertSet { + set: new_set, + len: len as u32, + }); + } + + pub(crate) fn get_insert(&self, pos: usize) -> Option { + match self { + Self::Small(set) => { + let mut index = 0; + for insert in set.set.iter() { + if index + insert.len as usize > pos { + return Some(insert.leaf); + } + index += insert.len as usize; + } + + unreachable!() + } + Self::Large(set) => set.get_insert(pos), + } + } + + pub(crate) fn len(&self) -> usize { + match self { + Self::Small(set) => set.len as usize, + Self::Large(set) => *set.tree.root_cache() as usize, + } + } + + pub(crate) fn iter_all( + &self, + peer: PeerID, + counter: Counter, + ) -> Box + '_> { + match self { + InsertSet::Small(set) => { + let mut offset = 0; + Box::new(set.set.iter().map(move |elem| { + let ans = IterCursor::Insert { + leaf: elem.leaf, + id_span: IdSpan::new( + peer, + counter + offset as Counter, + counter + offset as Counter + elem.len as Counter, + ), + }; + + offset += elem.len; + ans + })) + } + InsertSet::Large(set) => { + let mut offset = 0; + Box::new(set.tree.iter().map(move |elem| { + let ans = IterCursor::Insert { + leaf: elem.leaf, + id_span: IdSpan::new( + peer, + counter + offset as Counter, + counter + offset as Counter + elem.len as Counter, + ), + }; + + offset += elem.len; + ans + })) + } + } + } + + /// Iterate the given target span range, the start id of the InsertSet is cur_id + pub(crate) fn iter_range( + &self, + cur_id: ID, + target_start_counter: i32, + target_end_counter: i32, + ) -> Box + '_> { + match self { + InsertSet::Small(set) => { + let mut offset = 0; + Box::new(set.set.iter().filter_map(move |elem| { + let id_span = IdSpan::new( + cur_id.peer, + (cur_id.counter + offset as Counter) + .max(target_start_counter) + .min(target_end_counter), + (cur_id.counter + offset as Counter + elem.len as Counter) + .max(target_start_counter) + .min(target_end_counter), + ); + + offset += elem.len; + if id_span.atom_len() == 0 { + return None; + } + + let ans = IterCursor::Insert { + leaf: elem.leaf, + id_span, + }; + + Some(ans) + })) + } + InsertSet::Large(set) => { + // let mut offset = 0; + // Box::new(set.tree.iter().filter_map(move |elem| { + // let id_span = IdSpan::new( + // cur_id.peer, + // (cur_id.counter + offset as Counter) + // .max(target_start_counter) + // .min(target_end_counter), + // (cur_id.counter + offset as Counter + elem.len as Counter) + // .max(target_start_counter) + // .min(target_end_counter), + // ); + + // offset += elem.len; + // if id_span.atom_len() == 0 { + // return None; + // } + + // let ans = IterCursor::Insert { + // leaf: elem.leaf, + // id_span, + // }; + + // Some(ans) + // })); + let offset = (target_start_counter - cur_id.counter).max(0); + let (start, mut start_counter) = if offset > 0 { + match set.tree.query::(&(offset as usize)) { + Some(start) => ( + std::ops::Bound::Included(start.cursor), + // NOTE: Can this be wrong? + target_start_counter - start.cursor.offset as Counter, + ), + _ => (std::ops::Bound::Unbounded, cur_id.counter), + } + } else { + (std::ops::Bound::Unbounded, cur_id.counter) + }; + + Box::new( + set.tree + .iter_range((start, std::ops::Bound::Unbounded)) + .map(move |b| { + let id_span = IdSpan::new( + cur_id.peer, + (start_counter) + .max(target_start_counter) + .min(target_end_counter), + (start_counter + b.elem.rle_len() as Counter) + .max(target_start_counter) + .min(target_end_counter), + ); + + start_counter += b.elem.rle_len() as Counter; + if id_span.atom_len() == 0 { + return None; + } + + let ans = IterCursor::Insert { + leaf: b.elem.leaf, + id_span, + }; + + Some(ans) + }) + .take_while(|x| x.is_some()) + .map(|x| x.unwrap()), + ) + } + } + } + + fn upgrade_if_needed(&mut self) { + match self { + InsertSet::Small(set) => { + if set.set.len() < SMALL_SET_MAX_LEN { + return; + } + + let set = std::mem::take(&mut set.set); + let tree_set = LargeInsertSet::init_from_small(set); + *self = InsertSet::Large(tree_set); + } + InsertSet::Large(_) => {} + } + } + } + + #[derive(Debug)] + pub(crate) struct SmallInsertSet { + pub set: SmallVec<[Insert; 1]>, + pub len: u32, + } + + impl SmallInsertSet { + fn update(&mut self, from: usize, to: usize, new_leaf: LeafIndex) { + if from == to { + return; + } + + let mut cur_scan_index: usize = 0; + let old_set = std::mem::take(&mut self.set); + let mut new_set = SmallVec::with_capacity(old_set.len() + 2); + let mut new_leaf_inserted = false; + for insert in old_set.iter() { + if new_leaf_inserted { + let end = cur_scan_index + insert.len as usize; + if end <= to { + cur_scan_index = end; + continue; + } + + if cur_scan_index >= to { + new_set.push(*insert); + } else { + new_set.push(Insert { + leaf: insert.leaf, + len: (end - to) as u32, + }); + } + + cur_scan_index = end; + continue; + } + + if cur_scan_index + insert.len as usize <= from { + new_set.push(*insert); + cur_scan_index += insert.len as usize; + } else { + debug_assert!(!new_leaf_inserted); + let elem_end = cur_scan_index + insert.len as usize; + if cur_scan_index < from { + new_set.push(Insert { + leaf: insert.leaf, + len: (from - cur_scan_index) as u32, + }); + cur_scan_index = from; + } + + if elem_end > to { + new_set.push(Insert { + leaf: new_leaf, + len: (to - cur_scan_index) as u32, + }); + new_set.push(Insert { + leaf: insert.leaf, + len: (elem_end - to) as u32, + }); + } else { + new_set.push(Insert { + leaf: new_leaf, + len: (to - cur_scan_index) as u32, + }); + } + + new_leaf_inserted = true; + cur_scan_index = elem_end; + } + } + + self.set = new_set; + debug_assert_eq!( + self.len, + self.set.iter().map(|x| x.len as usize).sum::() as u32 + ); + } + } + + pub(super) struct InsertSetBTreeTrait; + impl CanRemove for Insert { + fn can_remove(&self) -> bool { + self.len == 0 + } + } + + impl generic_btree::rle::Mergeable for Insert { + fn can_merge(&self, rhs: &Self) -> bool { + self.leaf == rhs.leaf + } + + fn merge_right(&mut self, rhs: &Self) { + self.len += rhs.len; + } + + fn merge_left(&mut self, left: &Self) { + self.len += left.len; + } + } + + impl HasLength for Insert { + fn rle_len(&self) -> usize { + self.len as usize + } + } + + impl generic_btree::rle::Sliceable for Insert { + fn _slice(&self, range: std::ops::Range) -> Self { + Insert { + leaf: self.leaf, + len: range.len() as u32, + } + } + } + + impl TryInsert for Insert { + fn try_insert(&mut self, _: usize, elem: Self) -> Result<(), Self> { + if self.leaf == elem.leaf { + self.len += elem.len; + Ok(()) + } else { + Err(elem) + } + } + } + + impl BTreeTrait for InsertSetBTreeTrait { + type Elem = Insert; + + type Cache = i32; + + type CacheDiff = i32; + + fn calc_cache_internal( + cache: &mut Self::Cache, + caches: &[generic_btree::Child], + ) -> Self::CacheDiff { + let new: Self::Cache = caches.iter().map(|x| *x.cache()).sum(); + let diff = new - *cache; + *cache = new; + diff + } + + fn apply_cache_diff(cache: &mut Self::Cache, diff: &Self::CacheDiff) { + *cache += *diff; + } + + fn merge_cache_diff(diff1: &mut Self::CacheDiff, diff2: &Self::CacheDiff) { + *diff1 += *diff2; + } + + fn get_elem_cache(elem: &Self::Elem) -> Self::Cache { + elem.len as i32 + } + + fn new_cache_to_diff(cache: &Self::Cache) -> Self::CacheDiff { + *cache + } + + fn sub_cache(cache_lhs: &Self::Cache, cache_rhs: &Self::Cache) -> Self::CacheDiff { + *cache_lhs - *cache_rhs + } + } + + impl UseLengthFinder for InsertSetBTreeTrait { + fn get_len(cache: &i32) -> usize { + *cache as usize + } + } + + #[derive(Debug)] + pub(crate) struct LargeInsertSet { + tree: Box>, + } + + impl LargeInsertSet { + pub fn new() -> Self { + Self { + tree: Box::new(BTree::new()), + } + } + + fn update(&mut self, from: usize, to: usize, new_leaf: LeafIndex) { + if from == to { + return; + } + + let Some(from) = self.tree.query::(&from) else { + return; + }; + let to = self + .tree + .query::(&to) + .map(|x| x.cursor) + .or_else(|| self.tree.end_cursor()) + .unwrap(); + + self.tree.update(from.cursor..to, &mut |x| { + x.leaf = new_leaf; + None + }); + } + + fn get_insert(&self, pos: usize) -> Option { + let c = self.tree.query::(&pos)?.cursor; + Some(self.tree.get_elem(c.leaf)?.leaf) + } + + fn init_from_small(set: SmallVec<[Insert; 1]>) -> LargeInsertSet { + let mut tree_set = LargeInsertSet::new(); + for item in set.into_iter() { + tree_set.tree.push(item); + } + + tree_set + } + } +} + +#[derive(Debug, Clone, Copy)] +pub(super) struct Insert { + leaf: LeafIndex, + len: u32, +} + +impl Cursor { + pub fn new_insert(leaf: LeafIndex, len: usize) -> Self { + Self::Insert(InsertSet::new(leaf, len)) + } + + pub fn new_move(leaf: LeafIndex, from_id: ID) -> Self { + Self::Move { + to: leaf, + from: from_id, + } + } + + #[allow(unused)] + pub fn new_delete(id_span: IdSpan) -> Self { + Self::Delete(id_span) + } + + fn update_insert(&mut self, from: usize, to: usize, new_leaf: LeafIndex) { + // tracing::info!( + // "set_insert: from={}, to={}, new_leaf={:?}", + // from, + // to, + // &new_leaf + // ); + + assert!(from <= to); + assert!(to <= self.rle_len()); + match self { + Self::Insert(set) => { + set.update(from, to, new_leaf); + } + Self::Move { .. } => { + unreachable!("update_insert on Move") + } + _ => unreachable!(), + } + } + + fn update_insert_many(&mut self, updates: &[(usize, usize, LeafIndex)]) { + match self { + Self::Insert(set) => { + set.update_many(updates); + } + Self::Move { .. } => { + unreachable!("update_insert_many on Move") + } + _ => unreachable!(), + } + } + + fn get_insert(&self, pos: usize) -> Option { + if pos >= self.rle_len() { + return None; + } + + match self { + Cursor::Insert(set) => set.get_insert(pos), + Cursor::Move { to, .. } => { + assert!(pos == 0); + Some(*to) + } + Cursor::Delete(_) => unreachable!(), + } + } +} + +impl HasLength for Cursor { + fn rle_len(&self) -> usize { + match self { + Cursor::Insert(set) => set.len(), + Cursor::Delete(d) => d.atom_len(), + Cursor::Move { .. } => 1, + } + } +} + +impl Mergeable for Cursor { + fn can_merge(&self, rhs: &Self) -> bool { + match (self, rhs) { + (Self::Insert(InsertSet::Small(a)), Self::Insert(InsertSet::Small(b))) => { + a.set.last().unwrap().leaf == b.set.first().unwrap().leaf && b.len == 1 + } + (Self::Delete(a), Self::Delete(b)) => a.is_mergable(b, &()), + _ => false, + } + } + + fn merge_right(&mut self, rhs: &Self) { + match (self, rhs) { + (Self::Insert(InsertSet::Small(a)), Self::Insert(InsertSet::Small(b))) => { + assert!(b.len == 1); + a.set.last_mut().unwrap().len += b.set.first().unwrap().len; + a.len += b.len; + } + (Self::Delete(a), Self::Delete(b)) => { + a.merge(b, &()); + } + _ => unreachable!(), + } + } + + fn merge_left(&mut self, _: &Self) { + unreachable!(); + } +} + +#[cfg(test)] +mod test { + use std::{any::Any, ops::Range, panic::AssertUnwindSafe}; + + use super::*; + use generic_btree::{ + rle::{CanRemove, TryInsert}, + BTree, BTreeTrait, UseLengthFinder, + }; + use smallvec::smallvec; + + #[derive(Debug, Clone)] + struct DummyElem; + + impl generic_btree::rle::HasLength for DummyElem { + fn rle_len(&self) -> usize { + 1 + } + } + + impl generic_btree::rle::Sliceable for DummyElem { + fn _slice(&self, range: Range) -> Self { + assert_eq!(range.start, 0); + assert_eq!(range.end, 1); + DummyElem + } + + fn split(&mut self, _pos: usize) -> Self { + unreachable!() + } + } + + impl generic_btree::rle::Mergeable for DummyElem { + fn can_merge(&self, _rhs: &Self) -> bool { + false + } + + fn merge_right(&mut self, _rhs: &Self) { + unreachable!() + } + + fn merge_left(&mut self, _left: &Self) { + unreachable!() + } + } + + impl TryInsert for DummyElem { + fn try_insert(&mut self, _pos: usize, elem: Self) -> Result<(), Self> { + Err(elem) + } + } + + impl CanRemove for DummyElem { + fn can_remove(&self) -> bool { + false + } + } + + struct DummyTree; + + impl BTreeTrait for DummyTree { + type Elem = DummyElem; + type Cache = isize; + type CacheDiff = isize; + const USE_DIFF: bool = true; + + fn calc_cache_internal( + cache: &mut Self::Cache, + caches: &[generic_btree::Child], + ) -> Self::CacheDiff { + let new_cache = caches.iter().map(|child| child.cache).sum::(); + let diff = new_cache - *cache; + *cache = new_cache; + diff + } + + fn apply_cache_diff(cache: &mut Self::Cache, diff: &Self::CacheDiff) { + *cache += diff; + } + + fn merge_cache_diff(diff1: &mut Self::CacheDiff, diff2: &Self::CacheDiff) { + *diff1 += diff2; + } + + fn get_elem_cache(_elem: &Self::Elem) -> Self::Cache { + 1 + } + + fn new_cache_to_diff(cache: &Self::Cache) -> Self::CacheDiff { + *cache + } + + fn sub_cache(cache_lhs: &Self::Cache, cache_rhs: &Self::Cache) -> Self::CacheDiff { + cache_lhs - cache_rhs + } + } + + impl UseLengthFinder for DummyTree { + fn get_len(cache: &isize) -> usize { + *cache as usize + } + } + + fn dummy_leaf() -> LeafIndex { + let mut tree = BTree::::new(); + tree.push(DummyElem).leaf + } + + fn dummy_leaves(n: usize) -> Vec { + let mut tree = BTree::::new(); + (0..n).map(|_| tree.push(DummyElem).leaf).collect() + } + + fn panic_message(payload: Box) -> String { + if let Some(msg) = payload.downcast_ref::() { + msg.clone() + } else if let Some(msg) = payload.downcast_ref::<&str>() { + msg.to_string() + } else { + "".to_string() + } + } + + #[test] + fn test_id_to_cursor() { + let _map = IdToCursor::default(); + } + + #[test] + fn zero_len_fragment_panics_with_left_0_right_3() { + let peer = 1; + let mut map = IdToCursor::default(); + map.map.insert( + peer, + vec![ + Fragment { + counter: 0, + cursor: Cursor::Insert(insert_set::InsertSet::Small( + insert_set::SmallInsertSet { + set: smallvec![], + len: 0, + }, + )), + }, + Fragment { + counter: 3, + cursor: Cursor::new_insert(dummy_leaf(), 1), + }, + ], + ); + + let err = std::panic::catch_unwind(AssertUnwindSafe(|| { + map.update_insert(IdSpan::new(peer, 0, 3), dummy_leaf()); + })) + .expect_err("expected zero-length fragment to trip the invariant"); + let msg = panic_message(err); + assert!(msg.contains("left == right"), "{msg}"); + assert!(msg.contains("left: 0"), "{msg}"); + assert!(msg.contains("right: 3"), "{msg}"); + } + + #[test] + fn zero_width_small_update_keeps_insert_set_non_empty() { + let leaf = dummy_leaf(); + let mut set = insert_set::InsertSet::new(leaf, 3); + set.update(0, 0, dummy_leaf()); + + match set { + insert_set::InsertSet::Small(set) => { + assert_eq!(set.len, 3); + assert_eq!(set.set.len(), 1); + assert_eq!(set.set[0].len, 3); + } + insert_set::InsertSet::Large(_) => panic!("expected small insert set"), + } + } + + #[test] + fn large_update_can_replace_the_tail_range() { + let leaves = dummy_leaves(4); + let mut set = insert_set::InsertSet::Small(insert_set::SmallInsertSet { + set: smallvec![ + Insert { + leaf: leaves[0], + len: 100, + }, + Insert { + leaf: leaves[1], + len: 100, + }, + Insert { + leaf: leaves[2], + len: 100, + } + ], + len: 300, + }); + set.update(0, 1, leaves[3]); + let new_leaf = leaves[3]; + set.update_many(&[(200, 300, new_leaf)]); + + assert_ne!(set.get_insert(199), Some(new_leaf)); + for i in 200..300 { + assert_eq!(set.get_insert(i), Some(new_leaf)); + } + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/container/tree/bool_rle_vec.rs b/3rdparty/loro/crates/loro-internal/src/container/tree/bool_rle_vec.rs new file mode 100644 index 0000000000..c3609d7230 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/container/tree/bool_rle_vec.rs @@ -0,0 +1,442 @@ +// TODO: use boolrle to encode the has_effects array +#![allow(dead_code)] +use std::io::{Read, Write}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct BoolRleVec { + /// For the number of the nth item (0-indexed): + /// - if n is odd, the item is the run length of consecutive `true` values + /// - if n is even, the item is the run length of consecutive `false` values + rle_vec: Vec, + len: usize, +} + +impl BoolRleVec { + pub fn push(&mut self, value: bool) { + self.len += 1; + if self.rle_vec.is_empty() { + self.rle_vec.push(0); + } + + let last_index = self.rle_vec.len() - 1; + let is_last_run_true = last_index % 2 == 1; + + if (value && is_last_run_true) || (!value && !is_last_run_true) { + // If the new value matches the last run, increment its length + self.rle_vec[last_index] += 1; + } else { + // If the new value doesn't match, start a new run + self.rle_vec.push(1); + } + } + + pub fn pop(&mut self) -> Option { + while let Some(last) = self.rle_vec.last() { + if *last == 0 { + self.rle_vec.pop(); + } else { + break; + } + } + + if self.rle_vec.is_empty() { + return None; + } + + self.len -= 1; + let last_index = self.rle_vec.len() - 1; + let is_last_run_true = last_index % 2 == 1; + self.rle_vec[last_index] -= 1; + if self.rle_vec[last_index] == 0 { + self.rle_vec.pop(); + } + + Some(is_last_run_true) + } + + pub fn drop_last_n(&mut self, n: usize) { + if n > self.len { + panic!("Attempted to drop more elements than exist in the vector"); + } + + let mut remaining = n; + while remaining > 0 { + if let Some(last) = self.rle_vec.last_mut() { + if *last <= remaining as u32 { + remaining -= *last as usize; + self.rle_vec.pop(); + } else { + *last -= remaining as u32; + remaining = 0; + } + } else { + break; + } + } + + self.len -= n; + + // Remove any trailing zero-length runs + while self.rle_vec.last().is_some_and(|&x| x == 0) { + self.rle_vec.pop(); + } + } + + pub fn merge(&mut self, other: &BoolRleVec) { + if self.is_empty() { + self.rle_vec = other.rle_vec.clone(); + self.len = other.len; + return; + } + + if other.is_empty() { + return; + } + + // Align the end of self with the start of other + let self_last_run_true = self.rle_vec.len().is_multiple_of(2); + if self_last_run_true { + self.rle_vec.extend_from_slice(&other.rle_vec); + } else { + *self.rle_vec.last_mut().unwrap() += other.rle_vec[0]; + self.rle_vec.extend_from_slice(&other.rle_vec[1..]); + } + + self.len += other.len; + } + + pub fn len(&self) -> usize { + self.len + } + + pub fn is_empty(&self) -> bool { + self.len == 0 + } + + pub fn iter(&self) -> BoolRleVecIter<'_> { + BoolRleVecIter { + rle_vec: self, + index: 0, + offset: 0, + } + } + + pub fn new() -> Self { + Self { + rle_vec: Default::default(), + len: 0, + } + } + + pub fn encode(&self, writer: &mut W) -> Result<(), std::io::Error> { + leb128::write::unsigned(writer, self.rle_vec.len() as u64)?; + for item in &self.rle_vec { + leb128::write::unsigned(writer, *item as u64)?; + } + + Ok(()) + } + + pub fn decode(reader: &mut R) -> Result { + let len = leb128::read::unsigned(reader) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))? + as usize; + let mut rle_vec = Vec::with_capacity(len); + let mut total_len = 0; + for _ in 0..len { + let v = leb128::read::unsigned(reader) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))? + as u32; + rle_vec.push(v); + total_len += v; + } + + Ok(Self { + rle_vec, + len: total_len as usize, + }) + } +} + +pub(crate) struct BoolRleVecIter<'a> { + rle_vec: &'a BoolRleVec, + index: usize, + offset: u32, +} + +impl Iterator for BoolRleVecIter<'_> { + type Item = bool; + + fn next(&mut self) -> Option { + if self.index >= self.rle_vec.rle_vec.len() { + return None; + } + + let run_length = self.rle_vec.rle_vec[self.index]; + if run_length == 0 { + self.index += 1; + self.offset = 0; + return self.next(); + } + + let value = self.index % 2 == 1; + self.offset += 1; + if self.offset >= run_length { + self.index += 1; + self.offset = 0; + } + + Some(value) + } + + fn nth(&mut self, n: usize) -> Option { + self.offset += n as u32; + while self.index < self.rle_vec.rle_vec.len() + && self.offset >= self.rle_vec.rle_vec[self.index] + { + self.offset -= self.rle_vec.rle_vec[self.index]; + self.index += 1; + } + + self.next() + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_bool_rle_vec() { + let truth = [true, true, false, false, true, true]; + let mut rle_vec = BoolRleVec::new(); + for t in truth.iter() { + rle_vec.push(*t); + } + + let iter = rle_vec.iter(); + for (a, b) in truth.iter().zip(iter) { + assert_eq!(*a, b); + } + + let mut encoded = Vec::new(); + rle_vec.encode(&mut encoded).unwrap(); + let decoded = BoolRleVec::decode(&mut encoded.as_slice()).unwrap(); + assert_eq!(rle_vec, decoded); + } + + #[test] + fn test_bool_rle_vec_skip() { + let truth = [true, true, false, false, true, true]; + let mut rle_vec = BoolRleVec::new(); + for t in truth.iter() { + rle_vec.push(*t); + } + + let iter = rle_vec.iter(); + for (a, b) in truth.iter().skip(3).zip(iter.skip(3)) { + assert_eq!(*a, b); + } + } + + #[test] + fn test_bool_rle_vec_empty() { + let rle_vec = BoolRleVec::new(); + assert_eq!(rle_vec.len(), 0); + assert!(rle_vec.iter().next().is_none()); + } + + #[test] + fn test_bool_rle_vec_single_element() { + let mut rle_vec = BoolRleVec::new(); + rle_vec.push(true); + assert_eq!(rle_vec.len(), 1); + assert_eq!(rle_vec.iter().next(), Some(true)); + } + + #[test] + fn test_bool_rle_vec_alternating() { + let mut rle_vec = BoolRleVec::new(); + rle_vec.push(true); + rle_vec.push(false); + rle_vec.push(true); + rle_vec.push(false); + assert_eq!(rle_vec.len(), 4); + assert_eq!( + rle_vec.iter().collect::>(), + vec![true, false, true, false] + ); + } + + #[test] + fn test_bool_rle_vec_long_run() { + let mut rle_vec = BoolRleVec::new(); + for _ in 0..1000 { + rle_vec.push(true); + } + rle_vec.push(false); + assert_eq!(rle_vec.len(), 1001); + assert_eq!(rle_vec.iter().filter(|&x| x).count(), 1000); + assert_eq!(rle_vec.iter().filter(|&x| !x).count(), 1); + } + + #[test] + fn test_bool_rle_vec_nth() { + let mut rle_vec = BoolRleVec::new(); + rle_vec.push(true); + rle_vec.push(true); + rle_vec.push(false); + rle_vec.push(false); + rle_vec.push(true); + let mut iter = rle_vec.iter(); + assert_eq!(iter.nth(2), Some(false)); + assert_eq!(iter.nth(1), Some(true)); + assert_eq!(iter.next(), None); + } + + #[test] + fn test_bool_rle_vec_iter_skip() { + let mut rle_vec = BoolRleVec::new(); + rle_vec.push(true); + rle_vec.push(true); + rle_vec.push(false); + rle_vec.push(false); + rle_vec.push(true); + rle_vec.push(false); + + // Test skipping zero elements + let iter = rle_vec.iter(); + assert_eq!( + iter.collect::>(), + vec![true, true, false, false, true, false] + ); + + // Test skipping some elements + let iter = rle_vec.iter().skip(2); + assert_eq!(iter.collect::>(), vec![false, false, true, false]); + + // Test skipping all elements + let mut iter = rle_vec.iter().skip(6); + assert_eq!(iter.next(), None); + + // Test skipping more than available elements + let mut iter = rle_vec.iter().skip(10); + assert_eq!(iter.next(), None); + + // Test skipping with a long run + let mut long_rle_vec = BoolRleVec::new(); + for _ in 0..1000 { + long_rle_vec.push(true); + } + long_rle_vec.push(false); + long_rle_vec.push(true); + + let iter = long_rle_vec.iter().skip(999); + assert_eq!(iter.collect::>(), vec![true, false, true]); + } + + #[test] + fn test_bool_rle_vec_merge() { + // Test merging two empty vectors + let mut vec1 = BoolRleVec::new(); + let vec2 = BoolRleVec::new(); + vec1.merge(&vec2); + assert!(vec1.is_empty()); + + // Test merging an empty vector with a non-empty one + let mut vec1 = BoolRleVec::new(); + let mut vec2 = BoolRleVec::new(); + vec2.push(true); + vec2.push(false); + vec1.merge(&vec2); + assert_eq!(vec1.iter().collect::>(), vec![true, false]); + + // Test merging two non-empty vectors + let mut vec1 = BoolRleVec::new(); + let mut vec2 = BoolRleVec::new(); + vec1.push(true); + vec1.push(true); + vec2.push(false); + vec2.push(true); + vec1.merge(&vec2); + assert_eq!( + vec1.iter().collect::>(), + vec![true, true, false, true] + ); + + // Test merging with alignment (last run of vec1 matches first run of vec2) + let mut vec1 = BoolRleVec::new(); + let mut vec2 = BoolRleVec::new(); + vec1.push(true); + vec1.push(false); + vec2.push(false); + vec2.push(true); + vec1.merge(&vec2); + assert_eq!( + vec1.iter().collect::>(), + vec![true, false, false, true] + ); + + // Test merging with long runs + let mut vec1 = BoolRleVec::new(); + let mut vec2 = BoolRleVec::new(); + for _ in 0..100 { + vec1.push(true); + } + for _ in 0..100 { + vec2.push(false); + } + vec1.merge(&vec2); + assert_eq!(vec1.len(), 200); + assert_eq!(vec1.rle_vec.len(), 3); + assert_eq!(vec1.iter().take(100).filter(|&x| x).count(), 100); + assert_eq!(vec1.iter().skip(100).filter(|&x| !x).count(), 100); + } + + #[test] + fn test_drop_last_n() { + // Test dropping from a single run + let mut vec = BoolRleVec::new(); + for _ in 0..5 { + vec.push(true); + } + vec.drop_last_n(3); + assert_eq!(vec.iter().collect::>(), vec![true, true]); + + // Test dropping across multiple runs + let mut vec = BoolRleVec::new(); + vec.push(true); + vec.push(true); + vec.push(false); + vec.push(false); + vec.push(true); + vec.drop_last_n(3); + assert_eq!(vec.iter().collect::>(), vec![true, true]); + + // Test dropping entire vector + let mut vec = BoolRleVec::new(); + vec.push(true); + vec.push(false); + vec.drop_last_n(2); + assert!(vec.is_empty()); + + // Test dropping from long runs + let mut vec = BoolRleVec::new(); + for _ in 0..100 { + vec.push(true); + } + for _ in 0..100 { + vec.push(false); + } + vec.drop_last_n(150); + assert_eq!(vec.len(), 50); + assert!(vec.iter().all(|x| x)); + + // Test dropping zero elements + let mut vec = BoolRleVec::new(); + vec.push(true); + vec.push(false); + vec.drop_last_n(0); + assert_eq!(vec.iter().collect::>(), vec![true, false]); + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/container/tree/mod.rs b/3rdparty/loro/crates/loro-internal/src/container/tree/mod.rs new file mode 100644 index 0000000000..d9f97a1064 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/container/tree/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod bool_rle_vec; +pub(crate) mod tree_ds; +pub(crate) mod tree_op; diff --git a/3rdparty/loro/crates/loro-internal/src/container/tree/tree_ds.rs b/3rdparty/loro/crates/loro-internal/src/container/tree/tree_ds.rs new file mode 100644 index 0000000000..01912995e0 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/container/tree/tree_ds.rs @@ -0,0 +1,115 @@ +#![allow(unused)] + +use loro_common::{IdFull, IdLp, TreeID}; +use rustc_hash::FxHashMap; + +use crate::VersionVector; + +use super::bool_rle_vec::BoolRleVec; + +pub(crate) struct TreeLinearHistory { + ops: Vec, + has_effect: BoolRleVec, + target_to_op_idx: FxHashMap, +} + +struct TreeOpWrap { + op: TreeOp, + last_effective_update_on_target: Option, +} + +pub(crate) struct TreeOp { + pub id: IdFull, + pub target: TreeID, +} + +pub(crate) struct PoppedTreeOp { + pub op: TreeOp, + pub has_effect: bool, + pub last_update_on_target: Option, +} + +impl TreeLinearHistory { + pub fn new() -> Self { + Self { + ops: Vec::new(), + has_effect: BoolRleVec::new(), + target_to_op_idx: FxHashMap::default(), + } + } + + pub fn from_vec(ops: Vec, has_effect: BoolRleVec) -> Self { + let mut this = Self::new(); + for (op, e) in ops.into_iter().zip(has_effect.iter()) { + this.push(op, e); + } + + this + } + + pub fn push(&mut self, op: TreeOp, has_effect: bool) { + if has_effect { + let last = self.target_to_op_idx.insert(op.target, self.ops.len()); + self.ops.push(TreeOpWrap { + op, + last_effective_update_on_target: last, + }); + } else { + self.ops.push(TreeOpWrap { + op, + last_effective_update_on_target: None, + }); + } + + self.has_effect.push(has_effect); + } + + pub fn pop_util(&mut self, threshold: IdLp) -> Vec { + let index = match self + .ops + .binary_search_by(|op_wrap| op_wrap.op.id.idlp().cmp(&threshold)) + { + Ok(index) => index, + Err(index) => index, + }; + + self._pop_util(index) + } + + // Some of them need to be pushed back later if they are inside the vv + #[must_use] + pub fn pop_until_all_inside_vv(&mut self, vv: &VersionVector) -> Vec { + for (i, op_wrap) in self.ops.iter().enumerate() { + if vv.get(&op_wrap.op.id.peer).unwrap() <= &op_wrap.op.id.counter { + return self._pop_util(i); + } + } + + vec![] + } + + fn _pop_util(&mut self, index: usize) -> Vec { + todo!() + } + + pub fn last_update_on(&self, target: TreeID) -> Option { + self.target_to_op_idx + .get(&target) + .and_then(|&idx| self.ops.get(idx)) + .map(|op_wrap| op_wrap.op.id) + } + + pub fn greatest_idlp(&self) -> Option { + self.ops.last().map(|op| op.op.id.idlp()) + } + + #[inline] + pub fn len(&self) -> usize { + self.ops.len() + } + + #[inline] + pub fn get_bool_rle_vec(&self) -> &BoolRleVec { + &self.has_effect + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/container/tree/tree_op.rs b/3rdparty/loro/crates/loro-internal/src/container/tree/tree_op.rs new file mode 100644 index 0000000000..5f7fe3c209 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/container/tree/tree_op.rs @@ -0,0 +1,75 @@ +use fractional_index::FractionalIndex; +use loro_common::TreeID; +use rle::{HasLength, Mergable}; +use serde::{Deserialize, Serialize}; + +use crate::state::TreeParentId; + +/// The operation of movable tree. +/// +/// In the movable tree, there are three actions: +/// - **Create**: target tree id will be generated by [`Transaction`], and parent tree id is `None`. +/// - **Move**: move target tree node a child node of the specified parent node. +/// - **Delete**: move target tree node to [`loro_common::DELETED_TREE_ROOT`]. +/// +/// +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub enum TreeOp { + Create { + target: TreeID, + parent: Option, + position: FractionalIndex, + }, + Move { + target: TreeID, + parent: Option, + position: FractionalIndex, + }, + Delete { + target: TreeID, + }, +} + +impl TreeOp { + pub(crate) fn target(&self) -> TreeID { + match self { + TreeOp::Create { target, .. } => *target, + TreeOp::Move { target, .. } => *target, + TreeOp::Delete { target, .. } => *target, + } + } + + #[allow(unused)] + pub(crate) fn parent(&self) -> Option { + match self { + TreeOp::Create { parent, .. } => *parent, + TreeOp::Move { parent, .. } => *parent, + TreeOp::Delete { .. } => Some(TreeID::delete_root()), + } + } + + pub(crate) fn parent_id(&self) -> TreeParentId { + match self { + TreeOp::Create { parent, .. } => TreeParentId::from(*parent), + TreeOp::Move { parent, .. } => TreeParentId::from(*parent), + TreeOp::Delete { .. } => TreeParentId::Deleted, + } + } + + pub(crate) fn fractional_index(&self) -> Option { + match self { + TreeOp::Create { position, .. } | TreeOp::Move { position, .. } => { + Some(position.clone()) + } + TreeOp::Delete { .. } => None, + } + } +} + +impl HasLength for TreeOp { + fn content_len(&self) -> usize { + 1 + } +} + +impl Mergable for TreeOp {} diff --git a/3rdparty/loro/crates/loro-internal/src/cursor.rs b/3rdparty/loro/crates/loro-internal/src/cursor.rs new file mode 100644 index 0000000000..f3c3558534 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/cursor.rs @@ -0,0 +1,97 @@ +use loro_common::{ContainerID, ID}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct Cursor { + // It's option because it's possible that the given container is empty. + pub id: Option, + pub container: ContainerID, + /// The target position is at the left, middle, or right of the given id. + /// + /// Side info can help to model the selection + pub side: Side, + /// The position of the cursor in the container when the cursor is created. + /// For text, this is the unicode codepoint index + /// This value is not encoded + pub(crate) origin_pos: usize, +} + +#[derive(Default, Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum Side { + Left = -1, + #[default] + Middle = 0, + Right = 1, +} + +impl Side { + pub fn from_i32(i: i32) -> Option { + match i { + -1 => Some(Side::Left), + 0 => Some(Side::Middle), + 1 => Some(Side::Right), + _ => None, + } + } + + pub fn to_i32(&self) -> i32 { + *self as i32 + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PosQueryResult { + pub update: Option, + pub current: AbsolutePosition, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AbsolutePosition { + pub pos: usize, + /// The target position is at the left, middle, or right of the given pos. + pub side: Side, +} + +#[derive(Debug, Clone, Copy, thiserror::Error)] +pub enum CannotFindRelativePosition { + #[error("Cannot find relative position. The container is deleted.")] + ContainerDeleted, + #[error("Cannot find relative position. It may be that the given id is deleted and the relative history is cleared.")] + HistoryCleared, + #[error("Cannot find relative position. The id is not found.")] + IdNotFound, +} + +impl Cursor { + pub fn new(id: Option, container: ContainerID, side: Side, origin_pos: usize) -> Self { + Self { + id, + container, + side, + origin_pos, + } + } + + pub fn encode(&self) -> Vec { + postcard::to_allocvec(self).unwrap() + } + + pub fn decode(data: &[u8]) -> Result { + postcard::from_bytes(data) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum PosType { + /// The index is based on the length of the text in bytes(UTF-8). + Bytes, + /// The index is based on the length of the text in Unicode Code Points. + Unicode, + /// The index is based on the length of the text in UTF-16 code units. + Utf16, + /// The index is based on the length of the text in events. + /// It is determined by the `wasm` feature. + Event, + /// The index is based on the entity index. + Entity, +} diff --git a/3rdparty/loro/crates/loro-internal/src/dag.rs b/3rdparty/loro/crates/loro-internal/src/dag.rs new file mode 100644 index 0000000000..9b14434eff --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/dag.rs @@ -0,0 +1,1417 @@ +//! DAG (Directed Acyclic Graph) is a common data structure in distributed system. +//! +//! This mod contains the DAGs in our CRDT. It's not a general DAG, it has some specific properties that +//! we used to optimize the speed: +//! - Each node has lamport clock. +//! - Each node has its ID (client_id, counter). +//! - We use ID to refer to node rather than content addressing (hash) +//! +use std::{ + borrow::Cow, + collections::{BinaryHeap, HashMap}, + fmt::Debug, +}; + +use loro_common::IdSpanVector; +use rle::{HasLength, Sliceable}; +use rustc_hash::{FxHashMap, FxHashSet}; +mod iter; +mod mermaid; +#[cfg(feature = "test_utils")] +mod test; +#[cfg(feature = "test_utils")] +pub use test::{fuzz_alloc_tree, Interaction}; + +use crate::{ + change::Lamport, + diff_calc::DiffMode, + id::{Counter, PeerID, ID}, + span::{CounterSpan, HasId, HasIdSpan, HasLamport, HasLamportSpan, IdSpan}, + version::{Frontiers, VersionVector, VersionVectorDiff}, +}; + +use self::{ + iter::{iter_dag, iter_dag_with_vv, DagCausalIter, DagIterator, DagIteratorVV}, + mermaid::dag_to_mermaid, +}; + +pub(crate) trait DagNode: HasLamport + HasId + HasLength + Debug + Sliceable { + fn deps(&self) -> &Frontiers; + + #[allow(unused)] + #[inline] + fn get_lamport_from_counter(&self, c: Counter) -> Lamport { + self.lamport() + c as Lamport - self.id_start().counter as Lamport + } +} + +/// Dag (Directed Acyclic Graph). +/// +/// We have following invariance in DAG +/// - All deps' lamports are smaller than current node's lamport +pub(crate) trait Dag: Debug { + type Node: DagNode; + + fn get(&self, id: ID) -> Option; + #[allow(unused)] + fn frontier(&self) -> &Frontiers; + fn vv(&self) -> &VersionVector; + fn contains(&self, id: ID) -> bool; +} + +pub(crate) trait DagUtils: Dag { + fn find_common_ancestor(&self, a_id: &Frontiers, b_id: &Frontiers) -> (Frontiers, DiffMode); + /// Slow, should probably only use on dev + #[allow(unused)] + fn get_vv(&self, id: ID) -> VersionVector; + #[allow(unused)] + fn find_path(&self, from: &Frontiers, to: &Frontiers) -> VersionVectorDiff; + fn iter_causal(&self, from: Frontiers, target: IdSpanVector) -> DagCausalIter<'_, Self> + where + Self: Sized; + #[allow(unused)] + fn iter(&self) -> DagIterator<'_, Self::Node> + where + Self: Sized; + #[allow(unused)] + fn iter_with_vv(&self) -> DagIteratorVV<'_, Self::Node> + where + Self: Sized; + #[allow(unused)] + fn mermaid(&self) -> String + where + Self: Sized; +} + +impl DagUtils for T { + #[inline] + fn find_common_ancestor(&self, a_id: &Frontiers, b_id: &Frontiers) -> (Frontiers, DiffMode) { + // TODO: perf: make it also return the spans to reach common_ancestors + find_common_ancestor(&|id| self.get(id), a_id, b_id) + } + + #[inline] + fn get_vv(&self, id: ID) -> VersionVector { + get_version_vector(&|id| self.get(id), id) + } + + fn find_path(&self, from: &Frontiers, to: &Frontiers) -> VersionVectorDiff { + let mut ans = VersionVectorDiff::default(); + if from == to { + return ans; + } + + if from.len() == 1 && to.len() == 1 { + let from = from.as_single().unwrap(); + let to = to.as_single().unwrap(); + if from.peer == to.peer { + let from_span = self.get(from).unwrap(); + let to_span = self.get(to).unwrap(); + if from_span.id_start() == to_span.id_start() { + if from.counter < to.counter { + ans.forward.insert( + from.peer, + CounterSpan::new(from.counter + 1, to.counter + 1), + ); + } else { + ans.retreat.insert( + from.peer, + CounterSpan::new(to.counter + 1, from.counter + 1), + ); + } + return ans; + } + + if from_span.deps().len() == 1 + && to_span.contains_id(from_span.deps().as_single().unwrap()) + { + ans.retreat.insert( + from.peer, + CounterSpan::new(to.counter + 1, from.counter + 1), + ); + return ans; + } + + if to_span.deps().len() == 1 + && from_span.contains_id(to_span.deps().as_single().unwrap()) + { + ans.forward.insert( + from.peer, + CounterSpan::new(from.counter + 1, to.counter + 1), + ); + return ans; + } + } + } + + _find_common_ancestor( + &|v| self.get(v), + from, + to, + &mut |span, node_type| match node_type { + NodeType::A => ans.merge_left(span), + NodeType::B => ans.merge_right(span), + NodeType::Shared => { + ans.subtract_start_left(span); + ans.subtract_start_right(span); + } + }, + true, + ); + + ans + } + + #[inline(always)] + fn iter_with_vv(&self) -> DagIteratorVV<'_, Self::Node> + where + Self: Sized, + { + iter_dag_with_vv(self) + } + + #[inline(always)] + fn iter_causal(&self, from: Frontiers, target: IdSpanVector) -> DagCausalIter<'_, Self> + where + Self: Sized, + { + DagCausalIter::new(self, from, target) + } + + #[inline(always)] + fn iter(&self) -> DagIterator<'_, Self::Node> + where + Self: Sized, + { + iter_dag(self) + } + + /// You can visualize and generate img link at https://mermaid.live/ + #[inline] + fn mermaid(&self) -> String + where + Self: Sized, + { + dag_to_mermaid(self) + } +} + +#[allow(dead_code)] +fn get_version_vector<'a, Get, D>(get: &'a Get, id: ID) -> VersionVector +where + Get: Fn(ID) -> Option, + D: DagNode + 'a, +{ + let mut vv = VersionVector::new(); + let mut visited: FxHashSet = FxHashSet::default(); + vv.insert(id.peer, id.counter + 1); + let node = get(id).unwrap(); + + if node.deps().is_empty() { + return vv; + } + + let mut stack = Vec::with_capacity(node.deps().len()); + for dep in node.deps().iter() { + stack.push(dep); + } + + while let Some(node_id) = stack.pop() { + let node = get(node_id).unwrap(); + let node_id_start = node.id_start(); + vv.try_update_last(node_id); + if !visited.contains(&node_id_start) { + for dep in node.deps().iter() { + if !visited.contains(&dep) { + stack.push(dep); + } + } + + visited.insert(node_id_start); + } + } + + vv +} + +#[derive(Debug, PartialEq, Eq)] +struct OrdIdSpan<'a> { + id: ID, + lamport: Lamport, + len: usize, + deps: Cow<'a, Frontiers>, +} + +impl HasLength for OrdIdSpan<'_> { + fn content_len(&self) -> usize { + self.len + } +} + +impl HasId for OrdIdSpan<'_> { + fn id_start(&self) -> ID { + self.id + } +} + +impl HasLamport for OrdIdSpan<'_> { + fn lamport(&self) -> Lamport { + self.lamport + } +} + +impl PartialOrd for OrdIdSpan<'_> { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for OrdIdSpan<'_> { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.lamport_last() + .cmp(&other.lamport_last()) + .then(self.id.peer.cmp(&other.id.peer)) + // If they have the same last id, we want the shorter one to be greater; + // Otherwise, find_common_ancestor won't work correctly. Because we may + // lazily load the dag node, so sometimes the longer one should be broken + // into smaller pieces but it's already pushed to the queue. + .then(other.len.cmp(&self.len)) + } +} + +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy)] +enum NodeType { + A, + B, + Shared, +} + +impl<'a> OrdIdSpan<'a> { + #[inline] + fn from_dag_node(id: ID, get: &'a F) -> Option> + where + D: DagNode + 'a, + F: Fn(ID) -> Option, + { + let span = get(id)?; + let span_id = span.id_start(); + Some(OrdIdSpan { + id: span_id, + lamport: span.lamport(), + deps: Cow::Owned(span.deps().clone()), + len: (id.counter - span_id.counter) as usize + 1, + }) + } + + #[inline] + fn get_min(&self) -> OrdIdSpan<'a> { + OrdIdSpan { + id: self.id, + lamport: self.lamport, + deps: Cow::Owned(Default::default()), + len: 1, + } + } +} + +#[inline(always)] +fn find_common_ancestor<'a, F, D>( + get: &'a F, + a_id: &Frontiers, + b_id: &Frontiers, +) -> (Frontiers, DiffMode) +where + D: DagNode + 'a, + F: Fn(ID) -> Option, +{ + if b_id.is_empty() { + return (Default::default(), DiffMode::Checkout); + } + + _find_common_ancestor_new(get, a_id, b_id) +} + +/// - deep whether keep searching until the min of non-shared node is found +fn _find_common_ancestor<'a, F, D, G>( + get: &'a F, + a_ids: &Frontiers, + b_ids: &Frontiers, + notify: &mut G, + find_path: bool, +) -> FxHashMap +where + D: DagNode + 'a, + F: Fn(ID) -> Option, + G: FnMut(IdSpan, NodeType), +{ + let mut ans: FxHashMap = Default::default(); + let mut queue: BinaryHeap<(OrdIdSpan, NodeType)> = BinaryHeap::new(); + for id in a_ids.iter() { + queue.push((OrdIdSpan::from_dag_node(id, get).unwrap(), NodeType::A)); + } + for id in b_ids.iter() { + queue.push((OrdIdSpan::from_dag_node(id, get).unwrap(), NodeType::B)); + } + let mut visited: HashMap = FxHashMap::default(); + // invariants in this method: + // + // - visited's (client, counters) are subset of max(version_vector_a, version_vector_b) + // - visited's node type reflecting whether we found the shared node of this client + // - ans's client id never repeat + // - nodes with the same id will only be visited once + // - we may visit nodes that are before the common ancestors + + // type count in the queue. if both are zero, we can stop + let mut a_count = a_ids.len(); + let mut b_count = b_ids.len(); + let mut min = None; + while let Some((node, mut node_type)) = queue.pop() { + match node_type { + NodeType::A => a_count -= 1, + NodeType::B => b_count -= 1, + NodeType::Shared => {} + } + + if node_type != NodeType::Shared { + if let Some(min) = &mut min { + let node_start = node.get_min(); + if node_start < *min { + *min = node_start; + } + } else { + min = Some(node.get_min()) + } + } + + // pop the same node in the queue + while let Some((other_node, other_type)) = queue.peek() { + if node.id_span() == other_node.id_span() { + if node_type == *other_type { + match node_type { + NodeType::A => a_count -= 1, + NodeType::B => b_count -= 1, + NodeType::Shared => {} + } + } else { + if node_type != NodeType::Shared { + if visited.get(&node.id.peer).map(|(_, t)| *t) != Some(NodeType::Shared) { + ans.insert(node.id.peer, other_node.id_last().counter); + } + node_type = NodeType::Shared; + } + match other_type { + NodeType::A => a_count -= 1, + NodeType::B => b_count -= 1, + NodeType::Shared => {} + } + } + + queue.pop(); + } else { + break; + } + } + + // detect whether client is visited by other + if let Some((ctr, visited_type)) = visited.get_mut(&node.id.peer) { + debug_assert!(*ctr >= node.id_last().counter); + if *visited_type == NodeType::Shared { + node_type = NodeType::Shared; + } else if *visited_type != node_type { + // if node_type is shared, ans should already contains it or its descendance + if node_type != NodeType::Shared { + ans.insert(node.id.peer, node.id_last().counter); + } + *visited_type = NodeType::Shared; + node_type = NodeType::Shared; + } + } else { + visited.insert(node.id.peer, (node.id_last().counter, node_type)); + } + + // if this is not shared, the end of the span must be only reachable from A, or only reachable from B. + // but the begin of the span may be reachable from both A and B + notify(node.id_span(), node_type); + + match node_type { + NodeType::A => a_count += node.deps.len(), + NodeType::B => b_count += node.deps.len(), + NodeType::Shared => {} + } + + if a_count == 0 && b_count == 0 && (min.is_none() || &node <= min.as_ref().unwrap()) { + if node_type != NodeType::Shared { + ans.clear(); + } + + break; + } + + for dep_id in node.deps.as_ref().iter() { + queue.push((OrdIdSpan::from_dag_node(dep_id, get).unwrap(), node_type)); + } + + if node_type != NodeType::Shared { + if queue.is_empty() { + ans.clear(); + break; + } + if node.deps.is_empty() && !find_path { + if node.len == 1 { + ans.clear(); + break; + } + + match node_type { + NodeType::A => a_count += 1, + NodeType::B => b_count += 1, + NodeType::Shared => {} + } + + queue.push(( + OrdIdSpan { + deps: Cow::Owned(Default::default()), + id: node.id, + len: 1, + lamport: node.lamport, + }, + node_type, + )); + } + } + } + + ans +} + +fn _find_common_ancestor_new<'a, F, D>( + get: &'a F, + left: &Frontiers, + right: &Frontiers, +) -> (Frontiers, DiffMode) +where + D: DagNode + 'a, + F: Fn(ID) -> Option, +{ + if right.is_empty() { + return (Default::default(), DiffMode::Checkout); + } + + if left.is_empty() { + if right.len() == 1 { + let mut node_id = right.as_single().unwrap(); + let mut node = get(node_id).unwrap(); + while node.deps().len() == 1 { + node_id = node.deps().as_single().unwrap(); + let Some(next) = get(node_id) else { + return (Default::default(), DiffMode::ImportGreaterUpdates); + }; + node = next; + } + + if node.deps().is_empty() { + return (Default::default(), DiffMode::Linear); + } + } + + return (Default::default(), DiffMode::ImportGreaterUpdates); + } + + if left.len() == 1 && right.len() == 1 { + let left = left.as_single().unwrap(); + let right = right.as_single().unwrap(); + if left.peer == right.peer { + let left_span = get(left).unwrap(); + let right_span = get(right).unwrap(); + if left_span.id_start() == right_span.id_start() { + if left.counter < right.counter { + return (left.into(), DiffMode::Linear); + } else { + return (right.into(), DiffMode::Checkout); + } + } + + if left_span.deps().len() == 1 + && right_span.contains_id(left_span.deps().as_single().unwrap()) + { + return (right.into(), DiffMode::Checkout); + } + + if right_span.deps().len() == 1 + && left_span.contains_id(right_span.deps().as_single().unwrap()) + { + return (left.into(), DiffMode::Linear); + } + } + } + + let mut is_linear = left.len() <= 1 && right.len() == 1; + let mut is_right_greater = true; + let mut has_unmatched_branch = false; + let mut ans: Frontiers = Default::default(); + + fn ids_to_ord_id_spans<'a, D: DagNode + 'a, F: Fn(ID) -> Option>( + ids: &Frontiers, + get: &'a F, + ) -> Option>> { + let mut ans = Vec::with_capacity(ids.len()); + for id in ids.iter() { + if let Some(node) = OrdIdSpan::from_dag_node(id, get) { + ans.push(node); + } else { + return None; + } + } + + Some(ans) + } + + fn deps_to_ord_id_spans<'a, D: DagNode + 'a, F: Fn(ID) -> Option>( + node: &OrdIdSpan<'a>, + get: &'a F, + ) -> Option>> { + let mut deps = ids_to_ord_id_spans(node.deps.as_ref(), get)?; + if node.id.counter > 0 { + let prev = node.id.inc(-1); + if let Some(prev) = OrdIdSpan::from_dag_node(prev, get) { + if !deps.iter().any(|dep| dep.contains_id(prev.id_last())) { + deps.push(prev); + } + } + } + + Some(deps) + } + + fn shrink_ancestor_frontiers<'a, D: DagNode + 'a, F: Fn(ID) -> Option>( + ids: &Frontiers, + get: &'a F, + ) -> Frontiers { + if ids.len() <= 1 { + return ids.clone(); + } + + let mut ids = ids_to_ord_id_spans(ids, get).expect("common ancestors should be in dag"); + ids.sort_unstable(); + let mut frontiers = Vec::with_capacity(ids.len()); + for id in ids.iter().rev() { + let mut should_insert = true; + for frontier in frontiers.iter().rev() { + if contains_in_ancestors(get, *frontier, id) { + should_insert = false; + break; + } + } + + if should_insert { + frontiers.push(id.id_last()); + } + } + + frontiers.into_iter().collect() + } + + fn has_trimmed_history_deps<'a, D: DagNode + 'a, F: Fn(ID) -> Option>( + ids: &Frontiers, + get: &'a F, + ) -> bool { + ids.iter().any(|id| { + let Some(node) = OrdIdSpan::from_dag_node(id, get) else { + return true; + }; + ids_to_ord_id_spans(node.deps.as_ref(), get).is_none() + }) + } + + fn contains_in_ancestors<'a, D: DagNode + 'a, F: Fn(ID) -> Option>( + get: &'a F, + frontier: ID, + target: &OrdIdSpan<'_>, + ) -> bool { + let mut visited = FxHashSet::default(); + let mut pending = Vec::new(); + let Some(node) = OrdIdSpan::from_dag_node(frontier, get) else { + return false; + }; + pending.push(node); + while let Some(node) = pending.pop() { + if node.contains_id(target.id_last()) { + return true; + } + + if node.lamport_last() < target.lamport_last() { + continue; + } + + if !visited.insert(node.id_start()) { + continue; + } + + if let Some(deps) = deps_to_ord_id_spans(&node, get) { + for dep in deps { + pending.push(dep); + } + } + } + + false + } + + let mut queue: BinaryHeap<(OrdIdSpan, NodeType)> = BinaryHeap::new(); + for span in ids_to_ord_id_spans(left, get).unwrap() { + queue.push((span, NodeType::A)); + } + + for span in ids_to_ord_id_spans(right, get).unwrap() { + queue.push((span, NodeType::B)); + } + + while let Some((mut node, mut node_type)) = queue.pop() { + while let Some((other_node, other_type)) = queue.peek() { + if node == *other_node || node.id_last() == other_node.id_last() { + if node_type != *other_type { + node_type = NodeType::Shared; + } + + queue.pop(); + } else { + break; + } + } + + if node_type == NodeType::Shared { + ans.push(node.id_last()); + continue; + } + + if queue.is_empty() { + has_unmatched_branch = true; + is_right_greater = false; + break; + } + + // if node_type is A, then the left side is greater or parallel to the right side + if node_type == NodeType::A { + is_right_greater = false; + } + + if let Some(other) = queue.peek() { + if node.contains_id(other.0.id_last()) && node_type != other.1 { + node.len = (other.0.id_last().counter - node.id.counter + 1) as usize; + queue.push((node, node_type)); + continue; + } + + if node.len > 1 { + node.len = if other.0.lamport_last() >= node.lamport { + (other.0.lamport_last() - node.lamport + 1).min(node.len as u32 - 1) as usize + } else { + 1 + }; + queue.push((node, node_type)); + continue; + } + } + + if let Some(deps) = deps_to_ord_id_spans(&node, get) { + if !deps.is_empty() { + for dep in deps { + queue.push((dep, node_type)); + } + is_linear = false; + continue; + } + } else { + // The dependency is on trimmed shallow history. The exact ancestor is + // not representable in the current DAG, so fall back to a conservative + // checkout base. + has_unmatched_branch = true; + is_right_greater = false; + continue; + } + + { + // Some checkout calculators still require replaying from a base that + // includes every branch whose operation positions may affect the diff. + // In non-linear checkout mode, an earlier common ancestor is a valid + // conservative base even when it is not the mathematical LCA. + has_unmatched_branch = true; + is_right_greater = false; + continue; + } + } + + ans = shrink_ancestor_frontiers(&ans, get); + if has_unmatched_branch && !has_trimmed_history_deps(&ans, get) { + ans = Default::default(); + } + + let mode = if is_right_greater { + if ans.len() <= 1 { + debug_assert_eq!(&ans, left); + } + + if is_linear { + debug_assert!(ans.len() <= 1); + DiffMode::Linear + } else { + DiffMode::ImportGreaterUpdates + } + } else { + DiffMode::Checkout + }; + + (ans, mode) +} + +pub fn remove_included_frontiers(frontiers: &mut VersionVector, new_change_deps: &[ID]) { + for dep in new_change_deps.iter() { + if let Some(last) = frontiers.get_last(dep.peer) { + if last <= dep.counter { + frontiers.remove(&dep.peer); + } + } + } +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use loro_common::{HasId, HasIdSpan}; + use rand::{rngs::StdRng, Rng, SeedableRng}; + + use super::*; + + #[derive(Clone, Debug)] + struct TestNode { + id: ID, + len: usize, + lamport: Lamport, + deps: Frontiers, + } + + impl DagNode for TestNode { + fn deps(&self) -> &Frontiers { + &self.deps + } + } + + impl HasId for TestNode { + fn id_start(&self) -> ID { + self.id + } + } + + impl HasLamport for TestNode { + fn lamport(&self) -> Lamport { + self.lamport + } + } + + impl HasLength for TestNode { + fn content_len(&self) -> usize { + self.len + } + } + + impl Sliceable for TestNode { + fn slice(&self, from: usize, to: usize) -> Self { + Self { + id: self.id.inc(from as i32), + len: to - from, + lamport: self.lamport + from as Lamport, + deps: if from == 0 { + self.deps.clone() + } else { + self.id.inc(from as i32 - 1).into() + }, + } + } + } + + #[derive(Debug)] + struct TestDag { + nodes: BTreeMap, + vv: VersionVector, + frontier: Frontiers, + } + + impl TestDag { + fn new(nodes: impl IntoIterator, frontier: Frontiers) -> Self { + let mut vv = VersionVector::default(); + let nodes = nodes + .into_iter() + .map(|node| { + vv.set_end(node.id_end()); + (node.id_start(), node) + }) + .collect(); + Self { + nodes, + vv, + frontier, + } + } + } + + impl Dag for TestDag { + type Node = TestNode; + + fn get(&self, id: ID) -> Option { + self.nodes + .range(..=id) + .rev() + .find(|(_, node)| node.contains_id(id)) + .map(|(_, node)| node.clone()) + } + + fn frontier(&self) -> &Frontiers { + &self.frontier + } + + fn vv(&self) -> &VersionVector { + &self.vv + } + + fn contains(&self, id: ID) -> bool { + self.get(id).is_some() + } + } + + fn node( + peer: PeerID, + counter: Counter, + len: usize, + lamport: Lamport, + deps: Frontiers, + ) -> TestNode { + TestNode { + id: ID::new(peer, counter), + len, + lamport, + deps, + } + } + + fn ancestors_of_frontiers(dag: &TestDag, frontiers: &Frontiers) -> FxHashSet { + let mut ans = FxHashSet::default(); + for id in frontiers.iter() { + collect_ancestors(dag, id, &mut ans); + } + ans + } + + fn collect_ancestors(dag: &TestDag, id: ID, ans: &mut FxHashSet) { + let mut stack = vec![id]; + let mut visited_targets = FxHashSet::default(); + while let Some(id) = stack.pop() { + if !visited_targets.insert(id) { + continue; + } + + for node in dag.nodes.values() { + if node.id_start().peer != id.peer || node.id_start().counter > id.counter { + continue; + } + + let end = node.id_last().counter.min(id.counter); + for counter in node.id_start().counter..=end { + ans.insert(ID::new(node.id_start().peer, counter)); + } + + for dep in node.deps().iter() { + stack.push(dep); + } + } + } + } + + fn is_ancestor(dag: &TestDag, ancestor: ID, descendant: ID) -> bool { + let mut ancestors = FxHashSet::default(); + collect_ancestors(dag, descendant, &mut ancestors); + ancestors.contains(&ancestor) + } + + fn maximal_frontiers(dag: &TestDag, ids: impl IntoIterator) -> Frontiers { + let mut ids = Vec::::from_iter(ids); + ids.sort_by_key(|id| dag.get(*id).unwrap().get_lamport_from_counter(id.counter)); + let mut frontiers = Vec::new(); + for id in ids.into_iter().rev() { + if frontiers + .iter() + .any(|frontier| is_ancestor(dag, id, *frontier)) + { + continue; + } + + frontiers.retain(|frontier| !is_ancestor(dag, *frontier, id)); + frontiers.push(id); + } + + frontiers.into_iter().collect() + } + + fn oracle_common_ancestor(dag: &TestDag, left: &Frontiers, right: &Frontiers) -> Frontiers { + let left_ancestors = ancestors_of_frontiers(dag, left); + let right_ancestors = ancestors_of_frontiers(dag, right); + maximal_frontiers( + dag, + left_ancestors + .into_iter() + .filter(|id| right_ancestors.contains(id)), + ) + } + + fn assert_common_ancestor_valid_against_oracle( + dag: &TestDag, + left: &Frontiers, + right: &Frontiers, + ) { + let (actual, mode) = dag.find_common_ancestor(left, right); + let expected = oracle_common_ancestor(dag, left, right); + let left_ancestors = ancestors_of_frontiers(dag, left); + let right_ancestors = ancestors_of_frontiers(dag, right); + for id in actual.iter() { + assert!( + left_ancestors.contains(&id) && right_ancestors.contains(&id), + "actual LCA id {id} must be common: left={left:?} right={right:?} actual={actual:?} expected={expected:?} mode={mode:?}\ndag={dag:?}", + ); + } + + for a in actual.iter() { + for b in actual.iter() { + if a != b { + assert!( + !is_ancestor(dag, a, b), + "actual LCA must be a minimal frontier set: left={left:?} right={right:?} actual={actual:?} expected={expected:?} mode={mode:?}\ndag={dag:?}", + ); + } + } + } + + if !matches!(mode, DiffMode::Checkout) { + assert_eq!( + actual, expected, + "non-checkout mode should use the maximal common ancestor: left={left:?} right={right:?} mode={mode:?}\ndag={dag:?}", + ); + assert_eq!( + &actual, left, + "non-checkout mode must use left as LCA: left={left:?} right={right:?} mode={mode:?}" + ); + for id in left.iter() { + assert!( + right_ancestors.contains(&id), + "non-checkout mode requires right to include left id {id}; left={left:?} right={right:?} mode={mode:?}", + ); + } + } + } + + fn all_ids(dag: &TestDag) -> Vec { + dag.nodes + .values() + .flat_map(|node| { + (0..node.content_len()).map(|offset| node.id_start().inc(offset as Counter)) + }) + .collect() + } + + fn random_frontiers(dag: &TestDag, ids: &[ID], rng: &mut impl Rng) -> Frontiers { + if ids.is_empty() || rng.gen_bool(0.1) { + return Frontiers::default(); + } + + let len = rng.gen_range(1..=ids.len().min(4)); + maximal_frontiers(dag, (0..len).map(|_| ids[rng.gen_range(0..ids.len())])) + } + + fn random_dag(seed: u64, node_count: usize) -> TestDag { + let mut rng = StdRng::seed_from_u64(seed); + let mut next_counter = [0 as Counter; 8]; + let mut nodes = Vec::new(); + let mut existing_ids = Vec::new(); + for i in 0..node_count { + let peer = rng.gen_range(1..next_counter.len() as PeerID); + let counter = next_counter[peer as usize]; + let len = rng.gen_range(1..=3); + next_counter[peer as usize] += len as Counter; + let dep_count = if existing_ids.is_empty() { + 0 + } else { + rng.gen_range(0..=existing_ids.len().min(3)) + }; + let deps = maximal_frontiers( + &TestDag::new(nodes.clone(), Frontiers::default()), + (0..dep_count).map(|_| existing_ids[rng.gen_range(0..existing_ids.len())]), + ); + let node = node(peer, counter, len, i as Lamport * 4, deps); + existing_ids.extend((0..len).map(|offset| node.id_start().inc(offset as Counter))); + nodes.push(node); + } + + let dag = TestDag::new(nodes, Frontiers::default()); + let frontier = maximal_frontiers(&dag, existing_ids); + TestDag::new(dag.nodes.into_values(), frontier) + } + + fn layered_merge_dag() -> TestDag { + let root = node(1, 0, 3, 0, Frontiers::default()); + let left = node(1, 3, 2, 4, ID::new(1, 2).into()); + let right = node(2, 0, 3, 5, ID::new(1, 1).into()); + let late_right = node(2, 3, 2, 9, ID::new(2, 2).into()); + let third = node(3, 0, 2, 6, ID::new(1, 2).into()); + let merge_left_right = node( + 4, + 0, + 1, + 12, + Frontiers::from([left.id_last(), right.id_last()]), + ); + let merge_all = node( + 5, + 0, + 2, + 16, + Frontiers::from([ + merge_left_right.id_last(), + late_right.id_last(), + third.id_last(), + ]), + ); + let independent = node(6, 0, 2, 20, Frontiers::default()); + let final_merge = node( + 7, + 0, + 1, + 25, + Frontiers::from([merge_all.id_last(), independent.id_last()]), + ); + + TestDag::new( + vec![ + root, + left, + right, + late_right, + third, + merge_left_right, + merge_all, + independent, + final_merge.clone(), + ], + final_merge.id_last().into(), + ) + } + + #[test] + fn common_ancestor_handles_empty_linear_same_span_and_parent_child_cases() { + let first = node(1, 0, 2, 0, Frontiers::default()); + let second = node(1, 2, 2, 2, ID::new(1, 1).into()); + let dag = TestDag::new(vec![first, second], ID::new(1, 3).into()); + + assert_eq!( + dag.find_common_ancestor(&Frontiers::default(), &ID::new(1, 3).into()), + (Frontiers::default(), DiffMode::Linear) + ); + assert_eq!( + dag.find_common_ancestor(&ID::new(1, 3).into(), &Frontiers::default()), + (Frontiers::default(), DiffMode::Checkout) + ); + assert_eq!( + dag.find_common_ancestor(&ID::new(1, 0).into(), &ID::new(1, 1).into()), + (ID::new(1, 0).into(), DiffMode::Linear) + ); + assert_eq!( + dag.find_common_ancestor(&ID::new(1, 1).into(), &ID::new(1, 0).into()), + (ID::new(1, 0).into(), DiffMode::Checkout) + ); + assert_eq!( + dag.find_common_ancestor(&ID::new(1, 1).into(), &ID::new(1, 3).into()), + (ID::new(1, 1).into(), DiffMode::Linear) + ); + } + + #[test] + fn common_ancestor_left_empty_stops_linear_scan_at_missing_shallow_dependency() { + let visible = node(1, 1, 1, 1, ID::new(1, 0).into()); + let dag = TestDag::new(vec![visible], ID::new(1, 1).into()); + + assert_eq!( + dag.find_common_ancestor(&Frontiers::default(), &ID::new(1, 1).into()), + (Frontiers::default(), DiffMode::ImportGreaterUpdates) + ); + } + + #[test] + fn common_ancestor_of_parallel_branches_is_shared_dependency() { + let root = node(1, 0, 1, 0, Frontiers::default()); + let left = node(2, 0, 1, 1, root.id.into()); + let right = node(3, 0, 1, 2, root.id.into()); + let merge = node(4, 0, 1, 3, Frontiers::from([left.id, right.id])); + let dag = TestDag::new( + vec![root.clone(), left.clone(), right.clone(), merge.clone()], + merge.id.into(), + ); + + let (ancestor, mode) = dag.find_common_ancestor(&left.id.into(), &right.id.into()); + assert_eq!(ancestor, root.id.into()); + assert_eq!(mode, DiffMode::Checkout); + + let (ancestor, _) = dag.find_common_ancestor(&root.id.into(), &merge.id.into()); + assert_eq!(ancestor, root.id.into()); + } + + #[test] + fn common_ancestor_falls_back_before_independent_branch() { + let left = node(1, 0, 1, 0, Frontiers::default()); + let independent = node(2, 0, 1, 1, Frontiers::default()); + let merge = node(3, 0, 1, 2, Frontiers::from([left.id, independent.id])); + let dag = TestDag::new( + vec![left.clone(), independent, merge.clone()], + merge.id.into(), + ); + + let (ancestor, mode) = dag.find_common_ancestor(&left.id.into(), &merge.id.into()); + assert_eq!(ancestor, Frontiers::default()); + assert_eq!(mode, DiffMode::Checkout); + } + + #[test] + fn common_ancestor_falls_back_before_unmatched_branch_with_multiple_left_frontiers() { + let left_a = node(1, 0, 1, 0, Frontiers::default()); + let left_b = node(2, 0, 1, 1, Frontiers::default()); + let independent = node(3, 0, 1, 2, Frontiers::default()); + let left_frontiers = Frontiers::from([left_a.id, left_b.id]); + let merge = node( + 4, + 0, + 1, + 3, + Frontiers::from([left_a.id, left_b.id, independent.id]), + ); + let dag = TestDag::new( + vec![left_a.clone(), left_b.clone(), independent, merge.clone()], + merge.id.into(), + ); + + let (ancestor, mode) = dag.find_common_ancestor(&left_frontiers, &merge.id.into()); + assert_eq!(ancestor, Frontiers::default()); + assert_eq!(mode, DiffMode::Checkout); + } + + #[test] + fn common_ancestor_marks_cross_peer_direct_dependency_as_greater_update() { + let left = node(1, 0, 1, 0, Frontiers::default()); + let right = node(2, 0, 1, 1, left.id.into()); + let dag = TestDag::new(vec![left.clone(), right.clone()], right.id.into()); + + let (ancestor, mode) = dag.find_common_ancestor(&left.id.into(), &right.id.into()); + assert_eq!(ancestor, left.id.into()); + assert_eq!(mode, DiffMode::ImportGreaterUpdates); + } + + #[test] + fn common_ancestor_falls_back_when_right_adds_concurrent_branch_from_shared_root() { + let root = node(1, 0, 1, 0, Frontiers::default()); + let left = node(2, 0, 1, 1, root.id.into()); + let concurrent = node(3, 0, 1, 2, root.id.into()); + let merge = node(4, 0, 1, 3, Frontiers::from([left.id, concurrent.id])); + let dag = TestDag::new( + vec![root, left.clone(), concurrent, merge.clone()], + merge.id.into(), + ); + + let (ancestor, mode) = dag.find_common_ancestor(&left.id.into(), &merge.id.into()); + assert_eq!(ancestor, Frontiers::default()); + assert_eq!(mode, DiffMode::Checkout); + } + + #[test] + fn common_ancestor_keeps_target_when_checking_out_to_ancestor_with_extra_branch() { + let root = node(1, 0, 1, 0, Frontiers::default()); + let left = node(2, 0, 2, 1, root.id.into()); + let right = node(3, 0, 2, 3, root.id.into()); + let extra = node( + 4, + 0, + 1, + 5, + Frontiers::from([left.id_last(), right.id_last()]), + ); + let dag = TestDag::new( + vec![root, left.clone(), right.clone(), extra.clone()], + extra.id.into(), + ); + let target = Frontiers::from([ID::new(2, 0), ID::new(3, 0)]); + let current = Frontiers::from([extra.id, left.id_last(), right.id_last()]); + + let (ancestor, mode) = dag.find_common_ancestor(¤t, &target); + assert_eq!(ancestor, target); + assert_eq!(mode, DiffMode::Checkout); + } + + #[test] + fn common_ancestor_does_not_keep_ancestor_of_shared_descendant() { + let root = node(1, 0, 1, 0, Frontiers::default()); + let shared = node(2, 0, 1, 1, root.id.into()); + let left_only = node(3, 0, 1, 2, root.id.into()); + let right_only = node(4, 0, 1, 3, root.id.into()); + let dag = TestDag::new( + vec![root, shared.clone(), left_only.clone(), right_only.clone()], + Frontiers::from([shared.id, left_only.id, right_only.id]), + ); + + let (ancestor, mode) = dag.find_common_ancestor( + &Frontiers::from([shared.id, left_only.id]), + &Frontiers::from([shared.id, right_only.id]), + ); + assert_eq!(ancestor, shared.id.into()); + assert_eq!(mode, DiffMode::Checkout); + } + + #[test] + fn common_ancestor_valid_against_slow_oracle_on_random_dags() { + for seed in 0..128 { + let mut rng = StdRng::seed_from_u64(seed); + let dag = random_dag(seed, rng.gen_range(1..=18)); + let ids = all_ids(&dag); + for _ in 0..64 { + let left = random_frontiers(&dag, &ids, &mut rng); + let right = random_frontiers(&dag, &ids, &mut rng); + assert_common_ancestor_valid_against_oracle(&dag, &left, &right); + } + } + } + + #[test] + fn common_ancestor_valid_against_slow_oracle_on_all_pairs_in_small_random_dags() { + for seed in 1000..1020 { + let dag = random_dag(seed, 8); + let ids = all_ids(&dag); + let mut frontiers = vec![Frontiers::default()]; + frontiers.extend(ids.iter().copied().map(Frontiers::from)); + for chunk in ids.chunks(3) { + frontiers.push(maximal_frontiers(&dag, chunk.iter().copied())); + } + + for left in frontiers.iter() { + for right in frontiers.iter() { + assert_common_ancestor_valid_against_oracle(&dag, left, right); + } + } + } + } + + #[test] + fn common_ancestor_valid_against_slow_oracle_on_layered_merge_dag() { + let dag = layered_merge_dag(); + let ids = all_ids(&dag); + let mut frontiers = vec![Frontiers::default(), dag.frontier().clone()]; + frontiers.extend(ids.iter().copied().map(Frontiers::from)); + frontiers.extend([ + Frontiers::from([ID::new(1, 4), ID::new(2, 2)]), + Frontiers::from([ID::new(1, 4), ID::new(3, 1)]), + Frontiers::from([ID::new(4, 0), ID::new(2, 4), ID::new(3, 1)]), + Frontiers::from([ID::new(5, 1), ID::new(6, 1)]), + ]); + + for left in frontiers.iter() { + for right in frontiers.iter() { + assert_common_ancestor_valid_against_oracle(&dag, left, right); + } + } + } + + #[test] + fn common_ancestor_valid_against_slow_oracle_on_branchy_random_dags() { + for seed in 2000..2064 { + let mut rng = StdRng::seed_from_u64(seed); + let dag = random_dag(seed, rng.gen_range(20..=36)); + let ids = all_ids(&dag); + for _ in 0..96 { + let left = random_frontiers(&dag, &ids, &mut rng); + let right = random_frontiers(&dag, &ids, &mut rng); + assert_common_ancestor_valid_against_oracle(&dag, &left, &right); + } + } + } + + #[test] + fn find_path_reports_forward_and_retreat_spans_for_linear_and_branch_paths() { + let first = node(1, 0, 2, 0, Frontiers::default()); + let second = node(1, 2, 2, 2, ID::new(1, 1).into()); + let side = node(2, 0, 1, 1, ID::new(1, 1).into()); + let dag = TestDag::new( + vec![first, second, side], + Frontiers::from([ID::new(1, 3), ID::new(2, 0)]), + ); + + let forward = dag.find_path(&ID::new(1, 0).into(), &ID::new(1, 3).into()); + assert!(forward.retreat.is_empty()); + assert_eq!(forward.forward.get(&1), Some(&CounterSpan::new(1, 4))); + + let retreat = dag.find_path(&ID::new(1, 3).into(), &ID::new(1, 0).into()); + assert!(retreat.forward.is_empty()); + assert_eq!(retreat.retreat.get(&1), Some(&CounterSpan::new(1, 4))); + + let branch = dag.find_path(&ID::new(2, 0).into(), &ID::new(1, 3).into()); + assert_eq!(branch.retreat.get(&2), Some(&CounterSpan::new(0, 1))); + assert_eq!(branch.forward.get(&1), Some(&CounterSpan::new(2, 4))); + } + + #[test] + fn get_version_vector_walks_dependencies_once_and_tracks_dep_ids_inside_spans() { + let base = node(1, 0, 3, 0, Frontiers::default()); + let left = node(2, 0, 1, 4, ID::new(1, 1).into()); + let right = node(3, 0, 1, 5, ID::new(1, 2).into()); + let merge = node(4, 0, 1, 6, Frontiers::from([left.id, right.id])); + let dag = TestDag::new(vec![base, left, right, merge.clone()], merge.id.into()); + + let vv = dag.get_vv(merge.id); + assert_eq!(vv.get_last(1), Some(2)); + assert_eq!(vv.get_last(2), Some(0)); + assert_eq!(vv.get_last(3), Some(0)); + assert_eq!(vv.get_last(4), Some(0)); + } + + #[test] + fn remove_included_frontiers_drops_only_dependencies_at_or_after_recorded_frontiers() { + let mut vv = VersionVector::default(); + vv.set_last(ID::new(1, 3)); + vv.set_last(ID::new(2, 4)); + vv.set_last(ID::new(3, 1)); + + remove_included_frontiers(&mut vv, &[ID::new(1, 3), ID::new(2, 2), ID::new(4, 0)]); + + assert_eq!(vv.get_last(1), None); + assert_eq!(vv.get_last(2), Some(4)); + assert_eq!(vv.get_last(3), Some(1)); + } + + #[test] + fn ord_id_span_orders_by_last_lamport_peer_and_shorter_overlapping_span() { + let short = OrdIdSpan { + id: ID::new(1, 1), + lamport: 1, + len: 1, + deps: Cow::Owned(Frontiers::default()), + }; + let long = OrdIdSpan { + id: ID::new(1, 0), + lamport: 0, + len: 2, + deps: Cow::Owned(Frontiers::default()), + }; + let later_peer = OrdIdSpan { + id: ID::new(2, 0), + lamport: 1, + len: 1, + deps: Cow::Owned(Frontiers::default()), + }; + + assert!(short > long); + assert!(later_peer > short); + assert_eq!(long.get_min().len, 1); + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/dag/iter.rs b/3rdparty/loro/crates/loro-internal/src/dag/iter.rs new file mode 100644 index 0000000000..4c824b5b83 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/dag/iter.rs @@ -0,0 +1,631 @@ +use crate::version::Frontiers; +use num::Zero; +use std::{collections::BTreeMap, ops::Range}; + +use super::*; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +struct IdHeapItem { + id: ID, + lamport: Lamport, +} + +impl PartialOrd for IdHeapItem { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for IdHeapItem { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.lamport.cmp(&other.lamport).reverse() + } +} + +#[allow(dead_code)] +pub(crate) fn iter_dag_with_vv>(dag: &D) -> DagIteratorVV<'_, T> { + DagIteratorVV { + dag, + vv_map: Default::default(), + heap: BinaryHeap::new(), + } +} + +#[allow(dead_code)] +pub(crate) fn iter_dag(dag: &dyn Dag) -> DagIterator<'_, T> { + DagIterator { + dag, + visited: VersionVector::new(), + heap: BinaryHeap::new(), + } +} + +pub struct DagIterator<'a, T> { + dag: &'a dyn Dag, + /// Because all deps' lamports are smaller than current node's lamport. + /// We can use the lamport to sort the nodes so that each node's deps are processed before itself. + /// + /// The ids in this heap are start ids of nodes. It won't be a id pointing to the middle of a node. + heap: BinaryHeap, + visited: VersionVector, +} + +/// Should only use it on debug, because it's slow and likely to use lots of mem +impl Iterator for DagIterator<'_, T> { + type Item = T; + + fn next(&mut self) -> Option { + if self.visited.is_empty() { + if self.dag.vv().is_empty() { + return None; + } + + for (&client_id, _) in self.dag.vv().iter() { + if let Some(node) = self.dag.get(ID::new(client_id, 0)) { + self.heap.push(IdHeapItem { + id: ID::new(client_id, 0), + lamport: node.lamport(), + }); + } + + self.visited.insert(client_id, 0); + } + } + + if !self.heap.is_empty() { + let item = self.heap.pop().unwrap(); + let id = item.id; + let node = self.dag.get(id).unwrap(); + // push next node from the same client to the heap + let next_id = id.inc(node.content_len() as i32); + if self.dag.contains(next_id) { + let next_node = self.dag.get(next_id).unwrap(); + self.heap.push(IdHeapItem { + id: next_id, + lamport: next_node.lamport(), + }); + } + + return Some(node); + } + + None + } +} + +pub(crate) struct DagIteratorVV<'a, T> { + dag: &'a dyn Dag, + /// we should keep every nodes starting id inside this map + vv_map: FxHashMap, + /// Because all deps' lamports are smaller than current node's lamport. + /// We can use the lamport to sort the nodes so that each node's deps are processed before itself. + /// + /// The ids in this heap are start ids of nodes. It won't be a id pointing to the middle of a node. + heap: BinaryHeap, +} + +/// Should only use it on debug, because it's slow and likely to use lots of mem +impl Iterator for DagIteratorVV<'_, T> { + type Item = (T, VersionVector); + + fn next(&mut self) -> Option { + if self.vv_map.is_empty() { + if self.dag.vv().is_empty() { + return None; + } + + for (&client_id, _) in self.dag.vv().iter() { + let vv = VersionVector::new(); + if let Some(node) = self.dag.get(ID::new(client_id, 0)) { + if node.lamport() == 0 { + self.vv_map.insert(ID::new(client_id, 0), vv.clone()); + } + + self.heap.push(IdHeapItem { + id: ID::new(client_id, 0), + lamport: node.lamport(), + }); + } + } + } + + if !self.heap.is_empty() { + let item = self.heap.pop().unwrap(); + let id = item.id; + let node = self.dag.get(id).unwrap(); + debug_assert_eq!(id, node.id_start()); + let mut vv = { + // calculate vv + let mut vv: Option = None; + for dep_id in node.deps().iter() { + let dep = self.dag.get(dep_id).unwrap(); + let dep_vv = self.vv_map.get(&dep.id_start()).unwrap(); + if let Some(vv) = vv.as_mut() { + vv.merge(dep_vv); + } else { + vv = Some(dep_vv.clone()); + } + + if dep.id_start() != dep_id { + vv.as_mut().unwrap().set_last(dep_id); + } + } + + vv.unwrap_or_default() + }; + + vv.try_update_last(id); + self.vv_map.insert(id, vv.clone()); + + // push next node from the same client to the heap + let next_id = id.inc(node.content_len() as i32); + if self.dag.contains(next_id) { + let next_node = self.dag.get(next_id).unwrap(); + self.heap.push(IdHeapItem { + id: next_id, + lamport: next_node.lamport(), + }); + } + + return Some((node, vv)); + } + + None + } +} + +/// Visit every span in the target IdSpanVector. +/// It's guaranteed that the spans are visited in causal order, and each span is visited only once. +/// When visiting a span, we will checkout to the version where the span was created +pub(crate) struct DagCausalIter<'a, Dag> { + dag: &'a Dag, + frontier: Frontiers, + target: IdSpanVector, + /// how many dependencies are inside target for each id + out_degrees: FxHashMap, + succ: BTreeMap>, + stack: Vec, +} + +#[derive(Debug)] +pub(crate) struct IterReturn { + /// data is a reference, it need to be sliced by the counter_range to get the underlying data + pub data: T, + /// data[slice] is the data we want to return + #[allow(unused)] + pub slice: Range, +} + +impl<'a, T: DagNode, D: Dag + Debug> DagCausalIter<'a, D> { + pub fn new(dag: &'a D, from: Frontiers, target: IdSpanVector) -> Self { + let mut out_degrees: FxHashMap = FxHashMap::default(); + let mut succ: BTreeMap> = BTreeMap::default(); + let mut stack = Vec::new(); + let mut q = vec![]; + for id in target.iter() { + if id.1.content_len() > 0 { + let id = id.id_start(); + q.push(id); + } + } + + // traverse all nodes, calculate the out_degrees + // if out_degree is 0, then it can be iterated directly + while let Some(id) = q.pop() { + let client = id.peer; + let node = dag.get(id).unwrap(); + let deps = node.deps(); + out_degrees.insert( + id, + deps.iter() + .filter(|&dep| { + if let Some(span) = target.get(&dep.peer) { + let included_in_target = + dep.counter >= span.min() && dep.counter <= span.max(); + if included_in_target { + succ.entry(dep).or_default().push(id); + } + included_in_target + } else { + false + } + }) + .count(), + ); + let target_span = target.get(&client).unwrap(); + let last_counter = node.id_last().counter; + if target_span.max() > last_counter { + q.push(ID::new(client, last_counter + 1)) + } + } + + // Enforce per-peer counter order. Within a single peer, the iterator + // must visit tracked node-starts in ascending counter order even when + // the nodes are concurrent (i.e. don't declare each other as deps). + // Without this, two same-peer nodes can both have zero in-degree and + // end up on the stack at once; the LIFO pop order would then violate + // the iterator's invariant that each popped id equals the current + // target_span.min() for its peer. + let mut by_peer: FxHashMap> = FxHashMap::default(); + for id in out_degrees.keys() { + by_peer.entry(id.peer).or_default().push(id.counter); + } + for (peer, mut counters) in by_peer { + if counters.len() < 2 { + continue; + } + counters.sort_unstable(); + for pair in counters.windows(2) { + let prev = ID::new(peer, pair[0]); + let curr = ID::new(peer, pair[1]); + if let Some(deg) = out_degrees.get_mut(&curr) { + *deg += 1; + } + succ.entry(prev).or_default().push(curr); + } + } + + out_degrees.retain(|k, v| { + if v.is_zero() { + stack.push(*k); + return false; + } + true + }); + + Self { + dag, + frontier: from, + target, + out_degrees, + succ, + stack, + } + } +} + +impl> Iterator for DagCausalIter<'_, D> { + type Item = IterReturn; + + fn next(&mut self) -> Option { + loop { + if self.stack.is_empty() { + debug_assert_eq!( + 0, + self.target + .iter() + .map(|x| x.1.content_len() as i32) + .sum::() + ); + return None; + } + + let node_id = self.stack.pop().unwrap(); + let (node, slice_from, slice_end, next_same_peer) = { + let target_span = self.target.get_mut(&node_id.peer).unwrap(); + if node_id.counter != target_span.min() { + debug_assert!( + node_id.counter < target_span.min(), + "{} {:?}", + node_id, + target_span + ); + continue; + } + + // node_id may point into the middle of the node, we need to slice + let node = self.dag.get(node_id).unwrap(); + // node start_id may be smaller than node_id + let counter = node.id_span().counter; + let slice_from = if counter.start < target_span.start { + target_span.start - counter.start + } else { + 0 + }; + let slice_end = if counter.end < target_span.end { + counter.end - counter.start + } else { + target_span.end - counter.start + }; + + if slice_end <= slice_from { + debug_assert_eq!(slice_end, slice_from, "{node_id} {:?}", target_span); + continue; + } + + let consumed_last_counter = counter.start + slice_end - 1; + target_span.set_start(consumed_last_counter + 1); + let next_same_peer = if target_span.content_len() > 0 { + Some(ID::new(node_id.peer, target_span.min())) + } else { + None + }; + + (node, slice_from, slice_end, next_same_peer) + }; + + if let Some(next_id) = next_same_peer { + if !self.out_degrees.contains_key(&next_id) && !self.stack.contains(&next_id) { + self.stack.push(next_id); + } + } + + // NOTE: we expect user to update the tracker, to apply node, after visiting the node + self.frontier = Frontiers::from_id(node.id_start().inc(slice_end - 1)); + + let current_peer = node_id.peer; + let mut keys = Vec::new(); + let mut heap = BinaryHeap::new(); + // The in-degree of the successor node minus 1, and if it becomes 0, it is added to the heap + for (key, succ) in self.succ.range(node.id_start()..node.id_end()) { + keys.push(*key); + for succ_id in succ.iter() { + self.out_degrees.entry(*succ_id).and_modify(|i| *i -= 1); + if let Some(in_degree) = self.out_degrees.get(succ_id) { + if in_degree.is_zero() { + heap.push((succ_id.peer != current_peer, *succ_id)); + self.out_degrees.remove(succ_id); + } + } + } + } + // Nodes that have been traversed are removed from the graph to avoid being covered by other node ranges again + keys.into_iter().for_each(|k| { + self.succ.remove(&k); + }); + while let Some(id) = heap.pop() { + self.stack.push(id.1) + } + + return Some(IterReturn { + data: node, + slice: slice_from..slice_end, + }); + } + } +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use super::*; + use crate::{ + change::Lamport, + id::ID, + span::{CounterSpan, HasId, HasLamport}, + version::VersionVector, + }; + use rle::{HasLength, Sliceable}; + + #[derive(Debug, Clone)] + struct DummyNode { + id: ID, + lamport: Lamport, + len: usize, + deps: Frontiers, + } + + impl DagNode for DummyNode { + fn deps(&self) -> &Frontiers { + &self.deps + } + } + + impl Sliceable for DummyNode { + fn slice(&self, from: usize, to: usize) -> Self { + Self { + id: self.id.inc(from as i32), + lamport: self.lamport + from as Lamport, + len: to - from, + deps: if from > 0 { + self.id.inc(from as i32 - 1).into() + } else { + self.deps.clone() + }, + } + } + } + + impl HasLamport for DummyNode { + fn lamport(&self) -> Lamport { + self.lamport + } + } + + impl HasId for DummyNode { + fn id_start(&self) -> ID { + self.id + } + } + + impl HasLength for DummyNode { + fn content_len(&self) -> usize { + self.len + } + } + + #[derive(Debug)] + struct DummyDag { + nodes: BTreeMap, + vv: VersionVector, + frontier: Frontiers, + } + + impl Dag for DummyDag { + type Node = DummyNode; + + fn get(&self, id: ID) -> Option { + self.nodes + .range(..=id) + .next_back() + .filter(|(_, node)| node.contains_id(id)) + .map(|(_, node)| node.clone()) + } + + fn frontier(&self) -> &Frontiers { + &self.frontier + } + + fn vv(&self) -> &VersionVector { + &self.vv + } + + fn contains(&self, id: ID) -> bool { + self.nodes + .range(..=id) + .next_back() + .is_some_and(|(_, node)| node.contains_id(id)) + } + } + + #[test] + fn stale_iterator_state_repairs_missing_same_peer_continuation() { + let first = DummyNode { + id: ID::new(1, 0), + lamport: 0, + len: 5, + deps: Frontiers::default(), + }; + let second = DummyNode { + id: ID::new(1, 5), + lamport: 5, + len: 5, + deps: ID::new(1, 4).into(), + }; + let mut vv = VersionVector::default(); + vv.set_end(second.id_end()); + let dag = DummyDag { + frontier: second.id_last().into(), + nodes: [(first.id_start(), first), (second.id_start(), second)] + .into_iter() + .collect(), + vv, + }; + + let mut target = IdSpanVector::default(); + target.insert(1, CounterSpan::new(0, 7)); + let mut iter = DagCausalIter { + dag: &dag, + frontier: Frontiers::default(), + target, + out_degrees: Default::default(), + succ: BTreeMap::default(), + // This models the stale iterator state after `new()` saw one large + // node and queued only the first peer segment. + stack: vec![ID::new(1, 0)], + }; + + let first = iter.next().expect("first segment should be yielded"); + assert_eq!(first.data.id_start(), ID::new(1, 0)); + assert_eq!(first.slice, 0..5); + + let second = iter.next().expect("second segment should be re-queued"); + assert_eq!(second.data.id_start(), ID::new(1, 5)); + assert_eq!(second.slice, 0..2); + + assert!(iter.next().is_none()); + } + + /// Regression: before the fix, `DagCausalIter::new` would push both + /// `(peer=1, counter=0)` and `(peer=1, counter=5)` onto the initial + /// stack because neither node has any *in-target* dependency. The LIFO + /// pop would then surface `(peer=1, counter=5)` first while + /// `target_span.min()` is still `0`, tripping the causal-order + /// invariant (historically: "slice_end > slice_from" / "counter < + /// target_span.min()"). With the fix the iter synthesizes a per-peer + /// ordering edge so the second node is only released after the first is + /// consumed. + #[test] + fn two_concurrent_same_peer_nodes_visit_in_counter_order() { + let first = DummyNode { + id: ID::new(1, 0), + lamport: 10, + len: 5, + deps: Frontiers::default(), + }; + // `second` is concurrent with `first` from peer 1's perspective: + // its only dep lives outside the target span (peer 2), so the BFS + // records zero in-degree and would historically race to the stack + // alongside `first`. + let second = DummyNode { + id: ID::new(1, 5), + lamport: 20, + len: 5, + deps: ID::new(2, 0).into(), + }; + let side = DummyNode { + id: ID::new(2, 0), + lamport: 0, + len: 1, + deps: Frontiers::default(), + }; + + let mut vv = VersionVector::default(); + vv.set_end(first.id_end()); + vv.set_end(second.id_end()); + vv.set_end(side.id_end()); + + let dag = DummyDag { + frontier: second.id_last().into(), + nodes: [ + (first.id_start(), first), + (second.id_start(), second), + (side.id_start(), side), + ] + .into_iter() + .collect(), + vv, + }; + + // Target only peer 1. `second`'s dep on (peer 2, 0) is thus + // *outside* the target vector — the exact condition that made both + // same-peer nodes reach the stack concurrently before the fix. + let mut target = IdSpanVector::default(); + target.insert(1, CounterSpan::new(0, 10)); + + let mut iter = DagCausalIter::new(&dag, Frontiers::default(), target); + + let a = iter.next().expect("first peer-1 segment"); + assert_eq!(a.data.id_start(), ID::new(1, 0)); + assert_eq!(a.slice, 0..5); + + let b = iter.next().expect("second peer-1 segment"); + assert_eq!(b.data.id_start(), ID::new(1, 5)); + assert_eq!(b.slice, 0..5); + + assert!(iter.next().is_none()); + } + + #[test] + fn larger_node_only_advances_consumed_slice() { + let node = DummyNode { + id: ID::new(1, 0), + lamport: 0, + len: 10, + deps: Frontiers::default(), + }; + let mut vv = VersionVector::default(); + vv.set_end(node.id_end()); + let dag = DummyDag { + frontier: node.id_last().into(), + nodes: [(node.id_start(), node)].into_iter().collect(), + vv, + }; + + let mut target = IdSpanVector::default(); + target.insert(1, CounterSpan::new(0, 7)); + let mut iter = DagCausalIter { + dag: &dag, + frontier: Frontiers::default(), + target, + out_degrees: Default::default(), + succ: BTreeMap::default(), + stack: vec![ID::new(1, 0)], + }; + + let item = iter.next().expect("target slice should be yielded"); + assert_eq!(item.slice, 0..7); + assert_eq!(iter.target.get(&1).unwrap().content_len(), 0); + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/dag/mermaid.rs b/3rdparty/loro/crates/loro-internal/src/dag/mermaid.rs new file mode 100644 index 0000000000..f8d7f4032a --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/dag/mermaid.rs @@ -0,0 +1,320 @@ +use super::*; +#[allow(dead_code)] +struct BreakPoints { + break_points: FxHashMap>, + /// start ID to ID. The target ID may be in the middle of an op. + /// + /// only includes links across different clients + links: FxHashMap>, +} + +#[allow(dead_code)] +struct Output { + clients: FxHashMap>, + /// start ID to start ID. + /// + /// only includes links across different clients + links: FxHashMap>, +} + +#[allow(dead_code)] +fn to_str(output: Output) -> String { + let mut s = String::new(); + let mut indent_level = 0; + macro_rules! new_line { + () => { + s += "\n"; + for _ in 0..indent_level { + s += " "; + } + }; + } + s += "flowchart RL"; + indent_level += 1; + new_line!(); + for (client_id, spans) in output.clients.iter() { + s += format!("subgraph peer{client_id}").as_str(); + new_line!(); + let mut is_first = true; + for id_span in spans.iter().rev() { + if !is_first { + s += " --> " + } + + is_first = false; + s += format!( + "{}-{}(\"c{}: [{}, {})\")", + id_span.peer, + id_span.counter.start, + id_span.peer, + id_span.counter.start, + id_span.counter.end, + ) + .as_str(); + } + + new_line!(); + s += "end"; + new_line!(); + new_line!(); + } + + for (id_from, id_tos) in output.links.iter() { + for id_to in id_tos.iter() { + s += format!( + "{}-{} --> {}-{}", + id_from.peer, id_from.counter, id_to.peer, id_to.counter + ) + .as_str(); + new_line!(); + } + } + + s +} + +#[allow(dead_code)] +fn break_points_to_output(input: BreakPoints) -> Output { + let mut output = Output { + clients: FxHashMap::default(), + links: FxHashMap::default(), + }; + let breaks: FxHashMap> = input + .break_points + .into_iter() + .map(|(client_id, set)| { + let mut vec: Vec = set.iter().cloned().collect(); + vec.sort(); + (client_id, vec) + }) + .collect(); + for (client_id, break_points) in breaks.iter() { + let mut spans = Vec::with_capacity(break_points.len()); + for (from, to) in break_points.iter().zip(break_points.iter().skip(1)) { + spans.push(IdSpan::new(*client_id, *from, *to)); + } + output.clients.insert(*client_id, spans); + } + + for (id_from, id_tos) in input.links.iter() { + for id_to in id_tos { + let client_breaks = breaks.get(&id_to.peer).unwrap(); + match client_breaks.binary_search(&id_to.counter) { + Ok(_) => { + output.links.entry(*id_from).or_default().push(*id_to); + } + Err(index) => { + output + .links + .entry(*id_from) + .or_default() + .push(ID::new(id_to.peer, client_breaks[index - 1])); + } + } + } + } + output +} + +#[allow(dead_code)] +fn get_dag_break_points(dag: &impl Dag) -> BreakPoints { + let mut break_points = BreakPoints { + break_points: FxHashMap::default(), + links: FxHashMap::default(), + }; + + for node in dag.iter() { + let id = node.id_start(); + let set = break_points.break_points.entry(id.peer).or_default(); + set.insert(id.counter); + set.insert(id.counter + node.content_len() as Counter); + for dep in node.deps().iter() { + if dep.peer == id.peer { + continue; + } + + break_points + .break_points + .entry(dep.peer) + .or_default() + .insert(dep.counter); + break_points.links.entry(id).or_default().push(dep); + } + } + + break_points +} + +#[allow(dead_code)] +pub(crate) fn dag_to_mermaid(dag: &impl Dag) -> String { + to_str(break_points_to_output(get_dag_break_points(dag))) +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use loro_common::{HasId, HasIdSpan}; + use rle::{HasLength, Sliceable}; + + use super::*; + + #[derive(Clone, Debug)] + struct TestNode { + id: ID, + len: usize, + lamport: Lamport, + deps: Frontiers, + } + + impl DagNode for TestNode { + fn deps(&self) -> &Frontiers { + &self.deps + } + } + + impl HasId for TestNode { + fn id_start(&self) -> ID { + self.id + } + } + + impl HasLamport for TestNode { + fn lamport(&self) -> Lamport { + self.lamport + } + } + + impl HasLength for TestNode { + fn content_len(&self) -> usize { + self.len + } + } + + impl Sliceable for TestNode { + fn slice(&self, _from: usize, _to: usize) -> Self { + self.clone() + } + } + + #[derive(Debug)] + struct TestDag { + nodes: BTreeMap, + vv: VersionVector, + frontier: Frontiers, + } + + impl TestDag { + fn new(nodes: impl IntoIterator, frontier: Frontiers) -> Self { + let mut vv = VersionVector::default(); + let nodes = nodes + .into_iter() + .map(|node| { + vv.set_end(node.id_end()); + (node.id_start(), node) + }) + .collect(); + Self { + nodes, + vv, + frontier, + } + } + } + + impl Dag for TestDag { + type Node = TestNode; + + fn get(&self, id: ID) -> Option { + self.nodes + .range(..=id) + .next_back() + .filter(|(_, node)| node.contains_id(id)) + .map(|(_, node)| node.clone()) + } + + fn frontier(&self) -> &Frontiers { + &self.frontier + } + + fn vv(&self) -> &VersionVector { + &self.vv + } + + fn contains(&self, id: ID) -> bool { + self.get(id).is_some() + } + } + + fn node( + peer: PeerID, + counter: Counter, + len: usize, + lamport: Lamport, + deps: Frontiers, + ) -> TestNode { + TestNode { + id: ID::new(peer, counter), + len, + lamport, + deps, + } + } + + #[test] + fn break_points_to_output_splits_spans_and_retargets_middle_links_to_span_starts() { + let mut break_points = FxHashMap::default(); + break_points.insert(1, FxHashSet::from_iter([0, 3])); + break_points.insert(2, FxHashSet::from_iter([0, 5, 10])); + let from = ID::new(1, 0); + let mut links = FxHashMap::default(); + links.insert(from, vec![ID::new(2, 7)]); + + let output = break_points_to_output(BreakPoints { + break_points, + links, + }); + + assert_eq!(output.clients.get(&1).unwrap(), &vec![IdSpan::new(1, 0, 3)]); + assert_eq!( + output.clients.get(&2).unwrap(), + &vec![IdSpan::new(2, 0, 5), IdSpan::new(2, 5, 10)] + ); + assert_eq!(output.links.get(&from).unwrap(), &vec![ID::new(2, 5)]); + } + + #[test] + fn dag_to_mermaid_includes_peer_subgraphs_split_spans_and_cross_peer_edges() { + let first = node(1, 0, 2, 0, Frontiers::default()); + let second = node(2, 0, 1, 3, ID::new(1, 1).into()); + let dag = TestDag::new(vec![first, second], ID::new(2, 0).into()); + + let graph = dag_to_mermaid(&dag); + + assert!(graph.starts_with("flowchart RL")); + assert!(graph.contains("subgraph peer1")); + assert!(graph.contains("subgraph peer2")); + assert!(graph.contains("1-0(\"c1: [0, 1)\")")); + assert!(graph.contains("1-1(\"c1: [1, 2)\")")); + assert!(graph.contains("2-0(\"c2: [0, 1)\")")); + assert!(graph.contains("2-0 --> 1-1")); + } + + #[test] + fn to_str_renders_all_links_without_requiring_hash_map_order() { + let output = Output { + clients: FxHashMap::from_iter([ + (1, vec![IdSpan::new(1, 0, 1), IdSpan::new(1, 1, 2)]), + (2, vec![IdSpan::new(2, 0, 1)]), + ]), + links: FxHashMap::from_iter([(ID::new(2, 0), vec![ID::new(1, 0), ID::new(1, 1)])]), + }; + + let graph = to_str(output); + + assert!(graph.contains("subgraph peer1")); + assert!(graph.contains("1-1(\"c1: [1, 2)\") --> 1-0(\"c1: [0, 1)\")")); + assert!(graph.contains("2-0 --> 1-0")); + assert!(graph.contains("2-0 --> 1-1")); + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/dag/test.excalidraw b/3rdparty/loro/crates/loro-internal/src/dag/test.excalidraw new file mode 100644 index 0000000000..eb90d196e3 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/dag/test.excalidraw @@ -0,0 +1,4334 @@ +{ + "type": "excalidraw", + "version": 2, + "source": "https://marketplace.visualstudio.com/items?itemName=pomdtr.excalidraw-editor", + "elements": [ + { + "type": "text", + "version": 101, + "versionNonce": 883063978, + "isDeleted": false, + "id": "lhg64zlt-7Bh50s_bwPDh", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 263.48630115928825, + "y": -113.63915404458658, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 247, + "height": 25, + "seed": 127706155, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [], + "updated": 1666058888070, + "link": null, + "locked": false, + "fontSize": 20, + "fontFamily": 1, + "text": "Common Ancestors Tests", + "baseline": 18, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "Common Ancestors Tests" + }, + { + "type": "rectangle", + "version": 315, + "versionNonce": 978352886, + "isDeleted": false, + "id": "UFtnx7sfhV0uKBPc8bkjR", + "fillStyle": "cross-hatch", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 234.87301990928825, + "y": 593.3803772054134, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 56.71875, + "height": 53.90625, + "seed": 771228869, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "DvFA3B6P4HJIrrYLNfwWC", + "type": "arrow" + } + ], + "updated": 1666058888070, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 308, + "versionNonce": 759256938, + "isDeleted": false, + "id": "ndstLpUvl8l8gieSJBPWM", + "fillStyle": "cross-hatch", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 236.31442615928825, + "y": 492.0014709554134, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 51.8828125, + "height": 48.4921875, + "seed": 411378725, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "DvFA3B6P4HJIrrYLNfwWC", + "type": "arrow" + }, + { + "id": "lN2WY1kuFxjMAVkpFsGwW", + "type": "arrow" + }, + { + "id": "RU8Wyhtc02QcaaG6pQTMJ", + "type": "arrow" + } + ], + "updated": 1666058888070, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 532, + "versionNonce": 1135717430, + "isDeleted": false, + "id": "DvFA3B6P4HJIrrYLNfwWC", + "fillStyle": "cross-hatch", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 262.53317615928825, + "y": 547.0131897054134, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 1, + "height": 39.765625, + "seed": 653744005, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888070, + "link": null, + "locked": false, + "startBinding": { + "elementId": "ndstLpUvl8l8gieSJBPWM", + "focus": -0.03958466448687474, + "gap": 6.51953125 + }, + "endBinding": { + "elementId": "UFtnx7sfhV0uKBPc8bkjR", + "focus": -0.08757842935147449, + "gap": 6.6015625 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -1, + 39.765625 + ] + ] + }, + { + "type": "rectangle", + "version": 245, + "versionNonce": 1253497386, + "isDeleted": false, + "id": "URqECeM4WGZqAFI0abbR6", + "fillStyle": "cross-hatch", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 2, + "opacity": 100, + "angle": 0, + "x": 237.63473865928825, + "y": 382.3725647054134, + "strokeColor": "#e67700", + "backgroundColor": "transparent", + "width": 52.140625, + "height": 43.83203125, + "seed": 1133256421, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "lN2WY1kuFxjMAVkpFsGwW", + "type": "arrow" + }, + { + "id": "rsU49QVI8HgyJBtjILKZT", + "type": "arrow" + }, + { + "id": "4Xhxr95Fo3KM0W5_NblnS", + "type": "arrow" + } + ], + "updated": 1666058888070, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 551, + "versionNonce": 73053558, + "isDeleted": false, + "id": "lN2WY1kuFxjMAVkpFsGwW", + "fillStyle": "cross-hatch", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "x": 266.18161365928825, + "y": 434.3569397054134, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 3.11328125, + "height": 51.01953125, + "seed": 886110789, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888070, + "link": null, + "locked": false, + "startBinding": { + "elementId": "URqECeM4WGZqAFI0abbR6", + "focus": -0.15730547947414078, + "gap": 8.15234375 + }, + "endBinding": { + "elementId": "ndstLpUvl8l8gieSJBPWM", + "focus": -0.03906854855773698, + "gap": 6.625 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -3.11328125, + 51.01953125 + ] + ] + }, + { + "type": "rectangle", + "version": 229, + "versionNonce": 200148202, + "isDeleted": false, + "id": "WtyUQLscbyCgdCnYK_jNr", + "fillStyle": "cross-hatch", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 2, + "opacity": 100, + "angle": 0, + "x": 333.02536365928825, + "y": 382.2006897054134, + "strokeColor": "#e67700", + "backgroundColor": "transparent", + "width": 46.50390625, + "height": 47.48046875, + "seed": 329476517, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "RU8Wyhtc02QcaaG6pQTMJ", + "type": "arrow" + }, + { + "id": "ceD-EHtPmAHrld2ReqW1d", + "type": "arrow" + }, + { + "id": "gi633X0Z9XWBOZyYzVt0P", + "type": "arrow" + }, + { + "id": "aGZg71ALGFz6MuRal-TTz", + "type": "arrow" + }, + { + "id": "EE1SzT8racuV0SEacrH5H", + "type": "arrow" + } + ], + "updated": 1666058888070, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 214, + "versionNonce": 1972410038, + "isDeleted": false, + "id": "lSqDDXX9cdXC4LCluuQmS", + "fillStyle": "cross-hatch", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 2, + "opacity": 100, + "angle": 0, + "x": 419.02926990928825, + "y": 382.3374084554134, + "strokeColor": "#e67700", + "backgroundColor": "transparent", + "width": 50.58203125, + "height": 43.453125, + "seed": 19983621, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "KpyIcLVhnWpIY6J9Q8M3s", + "type": "arrow" + }, + { + "id": "Th-vLn9JQqbE80X8Pl1-U", + "type": "arrow" + }, + { + "id": "OzG5v9FzTWSKCdywOoGQL", + "type": "arrow" + } + ], + "updated": 1666058888070, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 456, + "versionNonce": 1517410218, + "isDeleted": false, + "id": "RU8Wyhtc02QcaaG6pQTMJ", + "fillStyle": "cross-hatch", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 325.62692615928825, + "y": 435.4702209554134, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 47.83203125, + "height": 47.4453125, + "seed": 485950565, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888070, + "link": null, + "locked": false, + "startBinding": { + "elementId": "WtyUQLscbyCgdCnYK_jNr", + "focus": 0.01865831404761865, + "gap": 7.3984375 + }, + "endBinding": { + "elementId": "ndstLpUvl8l8gieSJBPWM", + "focus": -0.358531455976611, + "gap": 9.0859375 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -47.83203125, + 47.4453125 + ] + ] + }, + { + "type": "arrow", + "version": 420, + "versionNonce": 536367094, + "isDeleted": false, + "id": "ceD-EHtPmAHrld2ReqW1d", + "fillStyle": "cross-hatch", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 345.79098865928825, + "y": 438.3452209554134, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 20.67578125, + "height": 99.421875, + "seed": 985870277, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888070, + "link": null, + "locked": false, + "startBinding": { + "elementId": "WtyUQLscbyCgdCnYK_jNr", + "focus": 0.6110590696052777, + "gap": 8.6640625 + }, + "endBinding": { + "elementId": "1Swh32YEcsSztjU6TVYMO", + "focus": 0.1403757111100622, + "gap": 6.3203125 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 20.67578125, + 99.421875 + ] + ] + }, + { + "type": "rectangle", + "version": 191, + "versionNonce": 913134186, + "isDeleted": false, + "id": "1Swh32YEcsSztjU6TVYMO", + "fillStyle": "cross-hatch", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 347.37692615928825, + "y": 544.0874084554134, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 43.8828125, + "height": 51.65625, + "seed": 1488121637, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "ceD-EHtPmAHrld2ReqW1d", + "type": "arrow" + } + ], + "updated": 1666058888070, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 181, + "versionNonce": 798417206, + "isDeleted": false, + "id": "7rpEPw0XEWSpd4zpGIL-i", + "fillStyle": "cross-hatch", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 456.69723865928825, + "y": 555.2280334554134, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 36.53125, + "height": 48.55078125, + "seed": 1429455493, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "gi633X0Z9XWBOZyYzVt0P", + "type": "arrow" + }, + { + "id": "KpyIcLVhnWpIY6J9Q8M3s", + "type": "arrow" + } + ], + "updated": 1666058888070, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 441, + "versionNonce": 1655403818, + "isDeleted": false, + "id": "gi633X0Z9XWBOZyYzVt0P", + "fillStyle": "cross-hatch", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 371.72848865928825, + "y": 442.6420959554134, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 115.453125, + "height": 106.890625, + "seed": 1694706149, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888070, + "link": null, + "locked": false, + "startBinding": { + "elementId": "WtyUQLscbyCgdCnYK_jNr", + "focus": 0.49474380369903664, + "gap": 12.9609375 + }, + "endBinding": { + "elementId": "7rpEPw0XEWSpd4zpGIL-i", + "focus": 1.0023527144670679, + "gap": 5.6953125 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 115.453125, + 106.890625 + ] + ] + }, + { + "type": "arrow", + "version": 435, + "versionNonce": 131156598, + "isDeleted": false, + "id": "KpyIcLVhnWpIY6J9Q8M3s", + "fillStyle": "cross-hatch", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 444.05661365928825, + "y": 433.2905334554134, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 36.18359375, + "height": 109.609375, + "seed": 1227793733, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888071, + "link": null, + "locked": false, + "startBinding": { + "elementId": "lSqDDXX9cdXC4LCluuQmS", + "focus": 0.30532272017379974, + "gap": 7.5 + }, + "endBinding": { + "elementId": "7rpEPw0XEWSpd4zpGIL-i", + "focus": 0.6606224867711618, + "gap": 12.328125 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 36.18359375, + 109.609375 + ] + ] + }, + { + "type": "rectangle", + "version": 442, + "versionNonce": 278347754, + "isDeleted": false, + "id": "CFSGkbwHAPWwmUPvyrNSU", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 246.38864490928825, + "y": 194.93506470541342, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 41.6796875, + "height": 41.82421874999999, + "seed": 1158919333, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "aGZg71ALGFz6MuRal-TTz", + "type": "arrow" + }, + { + "id": "WCsuKPIiyyv46Ci9BE954", + "type": "arrow" + } + ], + "updated": 1666058888071, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 201, + "versionNonce": 1466731446, + "isDeleted": false, + "id": "PX8R3GWFK-y4FAPfGXDWQ", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 241.00192615928825, + "y": -23.361810294586576, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 39, + "height": 43.90625, + "seed": 1415567365, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "vYg0q3zocHglNFtQ-Rzfb", + "type": "text" + }, + { + "id": "vYg0q3zocHglNFtQ-Rzfb", + "type": "text" + }, + { + "id": "g1I2bmNZBEIbE5bPbc9PO", + "type": "arrow" + }, + { + "type": "text", + "id": "vYg0q3zocHglNFtQ-Rzfb" + } + ], + "updated": 1666058888071, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 56, + "versionNonce": 205238954, + "isDeleted": false, + "id": "vYg0q3zocHglNFtQ-Rzfb", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 246.00192615928825, + "y": -13.908685294586576, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 29, + "height": 23, + "seed": 554733413, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [], + "updated": 1666058888071, + "link": null, + "locked": false, + "fontSize": 20, + "fontFamily": 1, + "text": "A", + "baseline": 17, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "PX8R3GWFK-y4FAPfGXDWQ", + "originalText": "A" + }, + { + "type": "rectangle", + "version": 259, + "versionNonce": 747480310, + "isDeleted": false, + "id": "wkdtJfpB-XRjEMes0NHIG", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 338.02926990928825, + "y": -23.916497794586576, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 39, + "height": 43.90625, + "seed": 963861189, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "B0jO-HvbxLWHssI5X1XjY", + "type": "text" + }, + { + "id": "X8GhhoUwWv39DpajqYRWn", + "type": "arrow" + }, + { + "type": "text", + "id": "B0jO-HvbxLWHssI5X1XjY" + } + ], + "updated": 1666058888071, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 117, + "versionNonce": 1749273962, + "isDeleted": false, + "id": "B0jO-HvbxLWHssI5X1XjY", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 343.02926990928825, + "y": -13.463372794586576, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 29, + "height": 23, + "seed": 103135781, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [], + "updated": 1666058888071, + "link": null, + "locked": false, + "fontSize": 20, + "fontFamily": 1, + "text": "B", + "baseline": 17, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "wkdtJfpB-XRjEMes0NHIG", + "originalText": "B" + }, + { + "type": "arrow", + "version": 305, + "versionNonce": 146380342, + "isDeleted": false, + "id": "aGZg71ALGFz6MuRal-TTz", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 267.8641585018744, + "y": 249.68115845541348, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 88.60298159582703, + "height": 125.39453124999994, + "seed": 1387909, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888071, + "link": null, + "locked": false, + "startBinding": { + "elementId": "CFSGkbwHAPWwmUPvyrNSU", + "focus": 0.6533881812518985, + "gap": 12.921875000000057 + }, + "endBinding": { + "elementId": "WtyUQLscbyCgdCnYK_jNr", + "focus": 0.5496088514101133, + "gap": 7.125 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 88.60298159582703, + 125.39453124999994 + ] + ] + }, + { + "type": "arrow", + "version": 364, + "versionNonce": 704286762, + "isDeleted": false, + "id": "Th-vLn9JQqbE80X8Pl1-U", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 285.62692615928825, + "y": 148.7884978967083, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 155.87566467544525, + "height": 226.02937930870513, + "seed": 645870821, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888071, + "link": null, + "locked": false, + "startBinding": { + "elementId": "hWhmUfN-whJHDhqK7HkDw", + "focus": -0.3630659075617793, + "gap": 5.17578125 + }, + "endBinding": { + "elementId": "lSqDDXX9cdXC4LCluuQmS", + "focus": 0.4308256364516943, + "gap": 7.51953125 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 155.87566467544525, + 226.02937930870513 + ] + ] + }, + { + "type": "rectangle", + "version": 231, + "versionNonce": 1960140662, + "isDeleted": false, + "id": "hWhmUfN-whJHDhqK7HkDw", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 240.19333240928825, + "y": 108.20459595541342, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 40.2578125, + "height": 45.4921875, + "seed": 888229957, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "WCsuKPIiyyv46Ci9BE954", + "type": "arrow" + }, + { + "id": "Th-vLn9JQqbE80X8Pl1-U", + "type": "arrow" + }, + { + "id": "g1I2bmNZBEIbE5bPbc9PO", + "type": "arrow" + }, + { + "id": "SJqqF6rk1YusFfgDKDZIX", + "type": "arrow" + } + ], + "updated": 1666058888071, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 452, + "versionNonce": 1967848170, + "isDeleted": false, + "id": "WCsuKPIiyyv46Ci9BE954", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 268.16418593279104, + "y": 159.16553345541342, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 0.11351614941020216, + "height": 32.2109375, + "seed": 424214437, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888071, + "link": null, + "locked": false, + "startBinding": { + "elementId": "hWhmUfN-whJHDhqK7HkDw", + "focus": -0.38312080421234723, + "gap": 5.46875 + }, + "endBinding": { + "elementId": "CFSGkbwHAPWwmUPvyrNSU", + "focus": 0.05429266252944664, + "gap": 3.55859375 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 0.11351614941020216, + 32.2109375 + ] + ] + }, + { + "type": "rectangle", + "version": 94, + "versionNonce": 354812086, + "isDeleted": false, + "id": "sHYBFDblul5QqJjxXun--", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 369.89255115928825, + "y": 81.69287720541342, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 40.76953125, + "height": 44.6484375, + "seed": 699646725, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "EE1SzT8racuV0SEacrH5H", + "type": "arrow" + }, + { + "id": "X8GhhoUwWv39DpajqYRWn", + "type": "arrow" + }, + { + "id": "T2wLMX7gk4q4Wlq65lvOL", + "type": "arrow" + } + ], + "updated": 1666058888071, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 71, + "versionNonce": 952110506, + "isDeleted": false, + "id": "MIEh2o7onmi2XVMBPNwtl", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 486.58786365928825, + "y": 145.29443970541342, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 50.74609375, + "height": 53.9140625, + "seed": 447683173, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "rsU49QVI8HgyJBtjILKZT", + "type": "arrow" + }, + { + "id": "T2wLMX7gk4q4Wlq65lvOL", + "type": "arrow" + }, + { + "id": "EtBC7WdsIkjxflHPwhLE3", + "type": "arrow" + } + ], + "updated": 1666058888071, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 87, + "versionNonce": 1028684278, + "isDeleted": false, + "id": "h7rDKyrC7qd8W2VaDHOVo", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 589.7050511592882, + "y": 230.97412720541342, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 60.6328125, + "height": 52.44140625, + "seed": 889750981, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "OzG5v9FzTWSKCdywOoGQL", + "type": "arrow" + }, + { + "id": "EtBC7WdsIkjxflHPwhLE3", + "type": "arrow" + } + ], + "updated": 1666058888071, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 170, + "versionNonce": 1501874282, + "isDeleted": false, + "id": "OzG5v9FzTWSKCdywOoGQL", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 601.5175511592882, + "y": 292.5288147054134, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 141.015625, + "height": 86.015625, + "seed": 1272808741, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888071, + "link": null, + "locked": false, + "startBinding": { + "elementId": "h7rDKyrC7qd8W2VaDHOVo", + "focus": -0.5378118988727681, + "gap": 9.11328125 + }, + "endBinding": { + "elementId": "lSqDDXX9cdXC4LCluuQmS", + "focus": -0.42120469532718297, + "gap": 3.79296875 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -141.015625, + 86.015625 + ] + ] + }, + { + "type": "arrow", + "version": 156, + "versionNonce": 72993590, + "isDeleted": false, + "id": "rsU49QVI8HgyJBtjILKZT", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 506.93161365928825, + "y": 207.66162720541342, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 236.51953125, + "height": 171.1328125, + "seed": 1922799749, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888071, + "link": null, + "locked": false, + "startBinding": { + "elementId": "MIEh2o7onmi2XVMBPNwtl", + "focus": -0.7011101066042149, + "gap": 8.453125 + }, + "endBinding": { + "elementId": "URqECeM4WGZqAFI0abbR6", + "focus": -0.5061734503271241, + "gap": 3.578125 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -236.51953125, + 171.1328125 + ] + ] + }, + { + "type": "arrow", + "version": 203, + "versionNonce": 1600400170, + "isDeleted": false, + "id": "EE1SzT8racuV0SEacrH5H", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 394.04880115928825, + "y": 132.08350220541342, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 29.9609375, + "height": 246.046875, + "seed": 1929396197, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888071, + "link": null, + "locked": false, + "startBinding": { + "elementId": "sHYBFDblul5QqJjxXun--", + "focus": -0.3111741555994148, + "gap": 5.7421875 + }, + "endBinding": { + "elementId": "WtyUQLscbyCgdCnYK_jNr", + "focus": 0.1692275109296219, + "gap": 4.0703125 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -29.9609375, + 246.046875 + ] + ] + }, + { + "type": "arrow", + "version": 152, + "versionNonce": 1487336566, + "isDeleted": false, + "id": "X8GhhoUwWv39DpajqYRWn", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 357.83786365928825, + "y": 26.669439705413424, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 28.1484375, + "height": 42.72265625, + "seed": 51968837, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888071, + "link": null, + "locked": false, + "startBinding": { + "elementId": "wkdtJfpB-XRjEMes0NHIG", + "focus": 0.5463571651085232, + "gap": 6.6796875 + }, + "endBinding": { + "elementId": "sHYBFDblul5QqJjxXun--", + "focus": 0.5277962544041825, + "gap": 12.30078125 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 28.1484375, + 42.72265625 + ] + ] + }, + { + "type": "arrow", + "version": 145, + "versionNonce": 550057450, + "isDeleted": false, + "id": "T2wLMX7gk4q4Wlq65lvOL", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 420.33786365928825, + "y": 109.02100220541342, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 88.5234375, + "height": 26.375, + "seed": 994265765, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888071, + "link": null, + "locked": false, + "startBinding": { + "elementId": "sHYBFDblul5QqJjxXun--", + "focus": -0.13918168745815823, + "gap": 9.67578125 + }, + "endBinding": { + "elementId": "MIEh2o7onmi2XVMBPNwtl", + "focus": 1.041000043955478, + "gap": 9.8984375 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 88.5234375, + 26.375 + ] + ] + }, + { + "type": "arrow", + "version": 170, + "versionNonce": 335345078, + "isDeleted": false, + "id": "EtBC7WdsIkjxflHPwhLE3", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 548.1972386592882, + "y": 171.43506470541342, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 72.3203125, + "height": 55.29296875, + "seed": 1277996549, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888071, + "link": null, + "locked": false, + "startBinding": { + "elementId": "MIEh2o7onmi2XVMBPNwtl", + "focus": -0.6152607989398555, + "gap": 10.86328125 + }, + "endBinding": { + "elementId": "h7rDKyrC7qd8W2VaDHOVo", + "focus": 0.6244234688285908, + "gap": 4.24609375 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 72.3203125, + 55.29296875 + ] + ] + }, + { + "type": "arrow", + "version": 342, + "versionNonce": 1858005162, + "isDeleted": false, + "id": "g1I2bmNZBEIbE5bPbc9PO", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 256.4971091669645, + "y": 29.618658455413424, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 4.3475249654532035, + "height": 65.71875, + "seed": 1959961957, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888071, + "link": null, + "locked": false, + "startBinding": { + "elementId": "PX8R3GWFK-y4FAPfGXDWQ", + "focus": 0.2891038770717789, + "gap": 9.07421875 + }, + "endBinding": { + "elementId": "hWhmUfN-whJHDhqK7HkDw", + "focus": 0.13304920019960997, + "gap": 12.8671875 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 4.3475249654532035, + 65.71875 + ] + ] + }, + { + "type": "rectangle", + "version": 144, + "versionNonce": 260099830, + "isDeleted": false, + "id": "anui-58DIlGAbjV4UqtEB", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 186.06833240928825, + "y": 259.5795959554134, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 35.375, + "height": 48.05859375, + "seed": 948089029, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "SJqqF6rk1YusFfgDKDZIX", + "type": "arrow" + }, + { + "id": "4Xhxr95Fo3KM0W5_NblnS", + "type": "arrow" + } + ], + "updated": 1666058888071, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 175, + "versionNonce": 1089487722, + "isDeleted": false, + "id": "SJqqF6rk1YusFfgDKDZIX", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 229.78317615928825, + "y": 140.61475220541342, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 26.859375, + "height": 109.1171875, + "seed": 1423593509, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888071, + "link": null, + "locked": false, + "startBinding": { + "elementId": "hWhmUfN-whJHDhqK7HkDw", + "focus": 1.0945410835846927, + "gap": 10.41015625 + }, + "endBinding": { + "elementId": "anui-58DIlGAbjV4UqtEB", + "focus": -0.38855870595300823, + "gap": 9.84765625 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -26.859375, + 109.1171875 + ] + ] + }, + { + "type": "arrow", + "version": 180, + "versionNonce": 934772790, + "isDeleted": false, + "id": "4Xhxr95Fo3KM0W5_NblnS", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 209.10348865928825, + "y": 316.2319397054134, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 36.80859375, + "height": 60.609375, + "seed": 2009531269, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888072, + "link": null, + "locked": false, + "startBinding": { + "elementId": "anui-58DIlGAbjV4UqtEB", + "focus": 0.4480882059017431, + "gap": 8.59375 + }, + "endBinding": { + "elementId": "URqECeM4WGZqAFI0abbR6", + "focus": -0.028542558229941315, + "gap": 5.53125 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 36.80859375, + 60.609375 + ] + ] + }, + { + "type": "text", + "version": 131, + "versionNonce": 1638417962, + "isDeleted": false, + "id": "YlP9Bsi-02bMH0FiCjimk", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1338.8156749765176, + "y": -115.61648283257028, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 95, + "height": 25, + "seed": 1642740997, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [], + "updated": 1666058888072, + "link": null, + "locked": false, + "fontSize": 20, + "fontFamily": 1, + "text": "Find Path", + "baseline": 18, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "Find Path" + }, + { + "type": "rectangle", + "version": 461, + "versionNonce": 545655158, + "isDeleted": false, + "id": "ar7IzDq-viaTUF4A87b_3", + "fillStyle": "cross-hatch", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 2, + "opacity": 100, + "angle": 0, + "x": 1350.2562583548645, + "y": 297.740958380131, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 37.91699739062982, + "height": 94.91248590244665, + "seed": 118309546, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "OLV3s1miKo9A3zrGxFock", + "type": "arrow" + }, + { + "id": "0vxegaqxPquMPgd_r72TA", + "type": "arrow" + }, + { + "id": "unCTd-_GGU2aGYrHtH0Xs", + "type": "arrow" + }, + { + "id": "XWSRH57gG_wU8RgnjMLPE", + "type": "arrow" + } + ], + "updated": 1666058888072, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 334, + "versionNonce": 433189098, + "isDeleted": false, + "id": "VjZnajfB7LKe5DMAoo0Do", + "fillStyle": "cross-hatch", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 2, + "opacity": 100, + "angle": 0, + "x": 1427.2168589704975, + "y": 345.30969428257765, + "strokeColor": "#e67700", + "backgroundColor": "transparent", + "width": 50.58203125, + "height": 43.453125, + "seed": 845372778, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "4EwKyIvTMtoyxqVHUbeGd", + "type": "arrow" + }, + { + "id": "E2mIJ2Sl4cVsW3XGconh3", + "type": "arrow" + }, + { + "id": "TggNAReZxluYM_0WrlwOr", + "type": "arrow" + } + ], + "updated": 1666058888072, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 1123, + "versionNonce": 422668982, + "isDeleted": false, + "id": "0vxegaqxPquMPgd_r72TA", + "fillStyle": "cross-hatch", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1364.6693370711946, + "y": 401.4742259036285, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 72.00486815355634, + "height": 146.2109087738151, + "seed": 698884138, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888072, + "link": null, + "locked": false, + "startBinding": { + "focus": -0.3965191378313845, + "gap": 8.159100701385455, + "elementId": "PDhm6OLJ3n51SVKIISduG" + }, + "endBinding": { + "focus": -0.13518700591321753, + "gap": 5.03237207597158, + "elementId": "MDw8O0rja4i4fRNIEVJ3V" + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -72.00486815355634, + 146.2109087738151 + ] + ] + }, + { + "type": "rectangle", + "version": 307, + "versionNonce": 1559774122, + "isDeleted": false, + "id": "MDw8O0rja4i4fRNIEVJ3V", + "fillStyle": "cross-hatch", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1260.210836693423, + "y": 552.7175067534151, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 43.8828125, + "height": 51.65625, + "seed": 1220971242, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "0vxegaqxPquMPgd_r72TA", + "type": "arrow" + } + ], + "updated": 1666058888072, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 222, + "versionNonce": 1079944182, + "isDeleted": false, + "id": "i1kjJhOQKEyXNdqTQ4aN7", + "fillStyle": "cross-hatch", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1497.8783853089833, + "y": 548.2801914995628, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 36.53125, + "height": 48.55078125, + "seed": 910794154, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "unCTd-_GGU2aGYrHtH0Xs", + "type": "arrow" + }, + { + "id": "4EwKyIvTMtoyxqVHUbeGd", + "type": "arrow" + } + ], + "updated": 1666058888072, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 944, + "versionNonce": 1967537770, + "isDeleted": false, + "id": "unCTd-_GGU2aGYrHtH0Xs", + "fillStyle": "cross-hatch", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1392.5188356116166, + "y": 402.286327945094, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 137.47301625611294, + "height": 141.3501859235961, + "seed": 257710186, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888072, + "link": null, + "locked": false, + "startBinding": { + "focus": 0.49474380369903687, + "gap": 10.567710822520771, + "elementId": "ar7IzDq-viaTUF4A87b_3" + }, + "endBinding": { + "focus": 1.0023527144670679, + "gap": 4.643677630872599, + "elementId": "i1kjJhOQKEyXNdqTQ4aN7" + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 137.47301625611294, + 141.3501859235961 + ] + ] + }, + { + "type": "arrow", + "version": 718, + "versionNonce": 403879222, + "isDeleted": false, + "id": "4EwKyIvTMtoyxqVHUbeGd", + "fillStyle": "cross-hatch", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1455.0832738563568, + "y": 396.26281928257765, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 63.74251748189454, + "height": 139.68924721698505, + "seed": 331987754, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888072, + "link": null, + "locked": false, + "startBinding": { + "focus": 0.3053227201737998, + "gap": 7.5, + "elementId": "VjZnajfB7LKe5DMAoo0Do" + }, + "endBinding": { + "focus": 0.6606224867711619, + "gap": 12.328125, + "elementId": "i1kjJhOQKEyXNdqTQ4aN7" + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 63.74251748189454, + 139.68924721698505 + ] + ] + }, + { + "type": "rectangle", + "version": 593, + "versionNonce": 797368618, + "isDeleted": false, + "id": "RTqCReH94hrfFOfKvX3Xb", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1237.3418614357154, + "y": 239.7958129496575, + "strokeColor": "#d9480f", + "backgroundColor": "transparent", + "width": 41.6796875, + "height": 41.82421874999999, + "seed": 1375520234, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "Gm1LhBSST9OO99UkA6y8X", + "type": "arrow" + }, + { + "id": "PIZ_GY_wTp8sNyDtJZcnP", + "type": "arrow" + } + ], + "updated": 1666058888072, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 242, + "versionNonce": 431136374, + "isDeleted": false, + "id": "v3ufybsWQdzjQGQnJYauJ", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1282.1830728089833, + "y": -30.30965225043724, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 39, + "height": 43.90625, + "seed": 1392539818, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "cVT4Yqnzt5w3srqK1yI08", + "type": "text" + }, + { + "id": "cVT4Yqnzt5w3srqK1yI08", + "type": "text" + }, + { + "id": "ScvEbRxtwuaacH97zK60u", + "type": "arrow" + }, + { + "type": "text", + "id": "cVT4Yqnzt5w3srqK1yI08" + } + ], + "updated": 1666058888072, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 97, + "versionNonce": 119876586, + "isDeleted": false, + "id": "cVT4Yqnzt5w3srqK1yI08", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1287.1830728089833, + "y": -20.85652725043724, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 29, + "height": 23, + "seed": 1182782314, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [], + "updated": 1666058888072, + "link": null, + "locked": false, + "fontSize": 20, + "fontFamily": 1, + "text": "A", + "baseline": 17, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "v3ufybsWQdzjQGQnJYauJ", + "originalText": "A" + }, + { + "type": "rectangle", + "version": 300, + "versionNonce": 1241735094, + "isDeleted": false, + "id": "EliJKYDq1GH48V4c34Lk1", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1379.2104165589833, + "y": -30.86433975043724, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 39, + "height": 43.90625, + "seed": 784717354, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "HgWFwyAcCrdOtSBQ3AMZC", + "type": "text" + }, + { + "id": "Y38QXVN9QX9BPQY-2ARsJ", + "type": "arrow" + }, + { + "type": "text", + "id": "HgWFwyAcCrdOtSBQ3AMZC" + } + ], + "updated": 1666058888072, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 158, + "versionNonce": 1378623146, + "isDeleted": false, + "id": "HgWFwyAcCrdOtSBQ3AMZC", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1384.2104165589833, + "y": -20.41121475043724, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 29, + "height": 23, + "seed": 213745898, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [], + "updated": 1666058888072, + "link": null, + "locked": false, + "fontSize": 20, + "fontFamily": 1, + "text": "B", + "baseline": 17, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "EliJKYDq1GH48V4c34Lk1", + "originalText": "B" + }, + { + "type": "arrow", + "version": 657, + "versionNonce": 1294596342, + "isDeleted": false, + "id": "E2mIJ2Sl4cVsW3XGconh3", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1326.6992629818317, + "y": 143.9103142656606, + "strokeColor": "#d9480f", + "backgroundColor": "transparent", + "width": 123.98105485129145, + "height": 193.77103893976562, + "seed": 200397738, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888072, + "link": null, + "locked": false, + "startBinding": { + "focus": -0.36231278326472255, + "gap": 5.17578125, + "elementId": "kKu5Ydyyyeh2KCFqMwJS4" + }, + "endBinding": { + "focus": 0.4326012077832137, + "gap": 7.628341077151447, + "elementId": "VjZnajfB7LKe5DMAoo0Do" + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 123.98105485129145, + 193.77103893976562 + ] + ] + }, + { + "type": "rectangle", + "version": 276, + "versionNonce": 1833351530, + "isDeleted": false, + "id": "kKu5Ydyyyeh2KCFqMwJS4", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1281.2656692318317, + "y": 101.25675399956276, + "strokeColor": "#d9480f", + "backgroundColor": "transparent", + "width": 40.2578125, + "height": 45.4921875, + "seed": 781045354, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "Gm1LhBSST9OO99UkA6y8X", + "type": "arrow" + }, + { + "id": "E2mIJ2Sl4cVsW3XGconh3", + "type": "arrow" + }, + { + "id": "ScvEbRxtwuaacH97zK60u", + "type": "arrow" + } + ], + "updated": 1666058888072, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 848, + "versionNonce": 2058054198, + "isDeleted": false, + "id": "Gm1LhBSST9OO99UkA6y8X", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1301.5745444918389, + "y": 152.10888167241131, + "strokeColor": "#d9480f", + "backgroundColor": "transparent", + "width": 33.471215097623144, + "height": 84.01952770009473, + "seed": 2036101418, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888072, + "link": null, + "locked": false, + "startBinding": { + "focus": -0.3812012413933706, + "gap": 5.359940172848553, + "elementId": "kKu5Ydyyyeh2KCFqMwJS4" + }, + "endBinding": { + "focus": 0.05283911803202242, + "gap": 3.667403577151447, + "elementId": "RTqCReH94hrfFOfKvX3Xb" + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -33.471215097623144, + 84.01952770009473 + ] + ] + }, + { + "type": "rectangle", + "version": 144, + "versionNonce": 1740635178, + "isDeleted": false, + "id": "61dxkeeBxIqh92cwqJ1lM", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1411.0736978089833, + "y": 74.74503524956276, + "strokeColor": "#1864ab", + "backgroundColor": "transparent", + "width": 40.76953125, + "height": 44.6484375, + "seed": 533249002, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "Y38QXVN9QX9BPQY-2ARsJ", + "type": "arrow" + }, + { + "id": "RPqJsAAGLOdXHUSB03TtV", + "type": "arrow" + }, + { + "id": "XWSRH57gG_wU8RgnjMLPE", + "type": "arrow" + } + ], + "updated": 1666058888072, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 116, + "versionNonce": 1902949238, + "isDeleted": false, + "id": "dLAlZSp2rgCOTpB7iMpyy", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1527.7690103089833, + "y": 138.34659774956276, + "strokeColor": "#1864ab", + "backgroundColor": "transparent", + "width": 50.74609375, + "height": 53.9140625, + "seed": 803057322, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "RPqJsAAGLOdXHUSB03TtV", + "type": "arrow" + }, + { + "id": "TggNAReZxluYM_0WrlwOr", + "type": "arrow" + } + ], + "updated": 1666058888072, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 517, + "versionNonce": 797107946, + "isDeleted": false, + "id": "TggNAReZxluYM_0WrlwOr", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1544.1200151918972, + "y": 204.47533658893383, + "strokeColor": "#364fc7", + "backgroundColor": "transparent", + "width": 69.09497933411285, + "height": 137.03534395324658, + "seed": 1044793706, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888072, + "link": null, + "locked": false, + "startBinding": { + "focus": -0.27534455299140825, + "gap": 12.214676339371067, + "elementId": "dLAlZSp2rgCOTpB7iMpyy" + }, + "endBinding": { + "focus": 0.26615030765156217, + "gap": 3.7990137403972426, + "elementId": "VjZnajfB7LKe5DMAoo0Do" + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -69.09497933411285, + 137.03534395324658 + ] + ] + }, + { + "type": "arrow", + "version": 889, + "versionNonce": 733140150, + "isDeleted": false, + "id": "XWSRH57gG_wU8RgnjMLPE", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1427.4290168688203, + "y": 128.0884665163548, + "strokeColor": "#364fc7", + "backgroundColor": "transparent", + "width": 36.397392064383894, + "height": 161.61331552909485, + "seed": 307072042, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888072, + "link": null, + "locked": false, + "startBinding": { + "focus": -0.12673028883550302, + "gap": 8.692142102217417, + "elementId": "61dxkeeBxIqh92cwqJ1lM" + }, + "endBinding": { + "focus": 0.31432596950919534, + "gap": 8.532211308753404, + "elementId": "ar7IzDq-viaTUF4A87b_3" + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -36.397392064383894, + 161.61331552909485 + ] + ] + }, + { + "type": "arrow", + "version": 302, + "versionNonce": 1025145258, + "isDeleted": false, + "id": "Y38QXVN9QX9BPQY-2ARsJ", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1398.6789464539738, + "y": 19.830407576714318, + "strokeColor": "#364fc7", + "backgroundColor": "transparent", + "width": 27.97996373105775, + "height": 42.725507914574635, + "seed": 1373533930, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888072, + "link": null, + "locked": false, + "startBinding": { + "focus": 0.5580789543163621, + "gap": 6.7884973271515605, + "elementId": "EliJKYDq1GH48V4c34Lk1" + }, + "endBinding": { + "focus": 0.5164516591849347, + "gap": 12.19197142284844, + "elementId": "61dxkeeBxIqh92cwqJ1lM" + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 27.97996373105775, + 42.725507914574635 + ] + ] + }, + { + "type": "arrow", + "version": 292, + "versionNonce": 321048054, + "isDeleted": false, + "id": "RPqJsAAGLOdXHUSB03TtV", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1461.0276308555176, + "y": 102.16607065982501, + "strokeColor": "#364fc7", + "backgroundColor": "transparent", + "width": 88.74199015486556, + "height": 26.39089941688931, + "seed": 763627946, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888072, + "link": null, + "locked": false, + "startBinding": { + "focus": -0.1330668246148966, + "gap": 9.458161595696765, + "elementId": "61dxkeeBxIqh92cwqJ1lM" + }, + "endBinding": { + "focus": 1.0359692074076057, + "gap": 9.78962767284844, + "elementId": "dLAlZSp2rgCOTpB7iMpyy" + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 88.74199015486556, + 26.39089941688931 + ] + ] + }, + { + "type": "arrow", + "version": 474, + "versionNonce": 1068610666, + "isDeleted": false, + "id": "ScvEbRxtwuaacH97zK60u", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1297.6782558166597, + "y": 22.562006672411314, + "strokeColor": "#d9480f", + "backgroundColor": "transparent", + "width": 4.274858880633019, + "height": 65.71875, + "seed": 208225386, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888072, + "link": null, + "locked": false, + "startBinding": { + "focus": 0.28876032726731193, + "gap": 8.965408922848553, + "elementId": "v3ufybsWQdzjQGQnJYauJ" + }, + "endBinding": { + "focus": 0.13338192970579654, + "gap": 12.975997327151447, + "elementId": "kKu5Ydyyyeh2KCFqMwJS4" + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 4.274858880633019, + 65.71875 + ] + ] + }, + { + "type": "rectangle", + "version": 172, + "versionNonce": 348233526, + "isDeleted": false, + "id": "PDhm6OLJ3n51SVKIISduG", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1350.6704030660512, + "y": 344.5622776479856, + "strokeColor": "#e67700", + "backgroundColor": "transparent", + "width": 36.1762450326662, + "height": 48.752847554257414, + "seed": 945366826, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "PIZ_GY_wTp8sNyDtJZcnP", + "type": "arrow" + }, + { + "id": "0vxegaqxPquMPgd_r72TA", + "type": "arrow" + } + ], + "updated": 1666058888072, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 575, + "versionNonce": 1226020650, + "isDeleted": false, + "id": "PIZ_GY_wTp8sNyDtJZcnP", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1272.6633717593447, + "y": 291.80621417637605, + "strokeColor": "#d9480f", + "backgroundColor": "transparent", + "width": 70.51754950478039, + "height": 53.81745925866585, + "seed": 163492330, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888072, + "link": null, + "locked": false, + "startBinding": { + "focus": 0.5414720442395691, + "gap": 10.186182476718557, + "elementId": "RTqCReH94hrfFOfKvX3Xb" + }, + "endBinding": { + "focus": 0.09938892036401328, + "gap": 7.48948180192599, + "elementId": "PDhm6OLJ3n51SVKIISduG" + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 70.51754950478039, + 53.81745925866585 + ] + ] + }, + { + "type": "arrow", + "version": 278, + "versionNonce": 888424566, + "isDeleted": false, + "id": "p4uOM1CtA_v2Pgf6ijK9W", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 2, + "opacity": 100, + "angle": 0, + "x": 1816.8655741450823, + "y": 804.5549918935991, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 4.425399877032078, + "height": 863.6896840333961, + "seed": 639832037, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888073, + "link": null, + "locked": false, + "startBinding": null, + "endBinding": null, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 4.425399877032078, + -863.6896840333961 + ] + ] + }, + { + "type": "text", + "version": 155, + "versionNonce": 2041258474, + "isDeleted": false, + "id": "j8MvrQTIbxG8XlEdrUsHO", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 2, + "opacity": 100, + "angle": 0, + "x": 1710.063519604042, + "y": -31.350809576339884, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 80, + "height": 25, + "seed": 786904971, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [], + "updated": 1666058888073, + "link": null, + "locked": false, + "fontSize": 20, + "fontFamily": 1, + "text": "Lamport", + "baseline": 18, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "Lamport" + }, + { + "type": "rectangle", + "version": 243, + "versionNonce": 595577270, + "isDeleted": false, + "id": "PNo3TYUfJ10IJi6267aAh", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1350.7224316028319, + "y": 295.70182854499143, + "strokeColor": "#1864ab", + "backgroundColor": "transparent", + "width": 37.41238356134112, + "height": 46.67318451720979, + "seed": 329573546, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "Y38QXVN9QX9BPQY-2ARsJ", + "type": "arrow" + }, + { + "id": "RPqJsAAGLOdXHUSB03TtV", + "type": "arrow" + }, + { + "id": "XWSRH57gG_wU8RgnjMLPE", + "type": "arrow" + } + ], + "updated": 1666058888073, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 163, + "versionNonce": 746804394, + "isDeleted": false, + "id": "2ZLvwZY7jTj3Y7A867PUw", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 2, + "opacity": 100, + "angle": 0, + "x": 907.8436350772699, + "y": 2.945826594468457, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 200, + "height": 25, + "seed": 1748790533, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [], + "updated": 1666058888073, + "link": null, + "locked": false, + "fontSize": 20, + "fontFamily": 1, + "text": "Need to find all the", + "baseline": 18, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "Need to find all the" + }, + { + "type": "rectangle", + "version": 620, + "versionNonce": 1334387446, + "isDeleted": false, + "id": "awuQ8K6kOHnRv5HDrvJWm", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 926.507571138557, + "y": 50.93006817318019, + "strokeColor": "#d9480f", + "backgroundColor": "transparent", + "width": 41.6796875, + "height": 41.82421874999999, + "seed": 489627941, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "7o4qiZeXIC5XlAhjbYObN", + "type": "arrow" + }, + { + "id": "D7U5Ron34tY9c82vBqDaB", + "type": "arrow" + } + ], + "updated": 1666058888073, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 226, + "versionNonce": 591867754, + "isDeleted": false, + "id": "E3RI5q1yc3GeaorRS1w0w", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1046.9959782392652, + "y": 46.34815941558787, + "strokeColor": "#1864ab", + "backgroundColor": "transparent", + "width": 40.76953125, + "height": 44.6484375, + "seed": 1561043499, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "strWHmShIdrbcdVwMei4L", + "type": "arrow" + }, + { + "id": "gouvuPNNZV_QH-D9GO0TA", + "type": "arrow" + }, + { + "id": "JZQIc_KcE8CBpiAjy1v9j", + "type": "arrow" + } + ], + "updated": 1666058888073, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 39, + "versionNonce": 2073383990, + "isDeleted": false, + "id": "jhZwZVYHq9DVCoTsVfoNA", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 2, + "opacity": 100, + "angle": 0, + "x": 920.1084297636246, + "y": 103.86982908128641, + "strokeColor": "#d9480f", + "backgroundColor": "transparent", + "width": 55, + "height": 20, + "seed": 1102244805, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [], + "updated": 1666058888073, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "From A", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "From A" + }, + { + "type": "text", + "version": 74, + "versionNonce": 843166250, + "isDeleted": false, + "id": "nwkFN4UJbc5Jg4yZ0eNgs", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 2, + "opacity": 100, + "angle": 0, + "x": 1041.1473326329826, + "y": 101.57709076329934, + "strokeColor": "#1864ab", + "backgroundColor": "transparent", + "width": 56, + "height": 20, + "seed": 1781306661, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [], + "updated": 1666058888073, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "From B", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "From B" + }, + { + "type": "text", + "version": 81, + "versionNonce": 1367956854, + "isDeleted": false, + "id": "kIrTAwchInsTnq6LCV63N", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 2, + "opacity": 100, + "angle": 0, + "x": 895.0964797797128, + "y": 397.1606789154389, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 214, + "height": 20, + "seed": 167668933, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "KLyXLrCImVTQP_jU2EX9O", + "type": "arrow" + } + ], + "updated": 1666058888073, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "This case is a little tricky", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "This case is a little tricky" + }, + { + "type": "arrow", + "version": 142, + "versionNonce": 1818090730, + "isDeleted": false, + "id": "KLyXLrCImVTQP_jU2EX9O", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "dashed", + "roughness": 2, + "opacity": 100, + "angle": 0, + "x": 1119.079507605672, + "y": 405.26356192336186, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 205.11324147046594, + "height": 45.80272521494817, + "seed": 459973771, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888073, + "link": null, + "locked": false, + "startBinding": { + "elementId": "kIrTAwchInsTnq6LCV63N", + "focus": 0.714758414751021, + "gap": 9.983027825959198 + }, + "endBinding": null, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 205.11324147046594, + -45.80272521494817 + ] + ] + }, + { + "type": "text", + "version": 188, + "versionNonce": 1990868662, + "isDeleted": false, + "id": "SztschmDzOAjFuvTM-OuV", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "dashed", + "roughness": 2, + "opacity": 100, + "angle": 0, + "x": 893.4248140652206, + "y": 439.769275660658, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 276, + "height": 40, + "seed": 332043691, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [], + "updated": 1666058888073, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "we need to slice it after we found\nthe nearest common ancestors", + "baseline": 34, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "we need to slice it after we found\nthe nearest common ancestors" + }, + { + "type": "rectangle", + "version": 108, + "versionNonce": 1327935402, + "isDeleted": false, + "id": "Y9vqkX1na6vJDz7l4PspG", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 207.46439441645236, + "y": 845.6472361162546, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 62.180771530968855, + "height": 64.0268369575906, + "seed": 1456447841, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "type": "text", + "id": "EJMwY_7pCh7jyDuzdJg6V" + }, + { + "id": "iBGHqUq7evqeBkhBGjpIY", + "type": "arrow" + }, + { + "id": "1FA4XZdv1alN1lHogHTl9", + "type": "arrow" + } + ], + "updated": 1666058888073, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 144, + "versionNonce": 530380790, + "isDeleted": false, + "id": "t51UcEh3ESlAROMlQggkd", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 150.87847590478498, + "y": 750.1418377874459, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 609.5544837384108, + "height": 29.249255457697245, + "seed": 699369007, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [], + "updated": 1666058888073, + "link": null, + "locked": false, + "fontSize": 23.399404366157817, + "fontFamily": 1, + "text": "What is the nearest common ancestors in this case?", + "baseline": 21.249255457697245, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "What is the nearest common ancestors in this case?" + }, + { + "type": "rectangle", + "version": 178, + "versionNonce": 1858546282, + "isDeleted": false, + "id": "OcZsyOg63jPy2vGYoS8dH", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 197.62028593945422, + "y": 994.0831720435232, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 55, + "height": 67.44040550860723, + "seed": 1580756303, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "type": "text", + "id": "BW_IPp1nyObkgC7Vu3OYp" + }, + { + "id": "iBGHqUq7evqeBkhBGjpIY", + "type": "arrow" + }, + { + "id": "WLEzhxAeQRhzi52ESFg-q", + "type": "arrow" + } + ], + "updated": 1666058888073, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 14, + "versionNonce": 610699574, + "isDeleted": false, + "id": "EJMwY_7pCh7jyDuzdJg6V", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 212.46439441645236, + "y": 865.1606545950499, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 52, + "height": 23, + "seed": 546698977, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [], + "updated": 1666058888073, + "link": null, + "locked": false, + "fontSize": 20, + "fontFamily": 1, + "text": "A", + "baseline": 17, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "Y9vqkX1na6vJDz7l4PspG", + "originalText": "A" + }, + { + "type": "text", + "version": 49, + "versionNonce": 571100458, + "isDeleted": false, + "id": "BW_IPp1nyObkgC7Vu3OYp", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 202.62028593945422, + "y": 1015.3033747978269, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 45, + "height": 23, + "seed": 1644625505, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [], + "updated": 1666058888073, + "link": null, + "locked": false, + "fontSize": 20, + "fontFamily": 1, + "text": "B", + "baseline": 17, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "OcZsyOg63jPy2vGYoS8dH", + "originalText": "B" + }, + { + "type": "rectangle", + "version": 136, + "versionNonce": 342618742, + "isDeleted": false, + "id": "gCZ4PjyxtYNIC3EGFAKxA", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 356.58493261190574, + "y": 974.517711358304, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 54, + "height": 58, + "seed": 1573171983, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "1FA4XZdv1alN1lHogHTl9", + "type": "arrow" + }, + { + "id": "fOuTMdWLfJmuB2ov11q7v", + "type": "arrow" + }, + { + "type": "text", + "id": "86HkQ09dZUZF0MFiJVz63" + } + ], + "updated": 1666058888073, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 185, + "versionNonce": 777721834, + "isDeleted": false, + "id": "hLRIE0OQJr_P647NEGYUb", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 339.35656242842015, + "y": 1187.155175856424, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 63.38000584903011, + "height": 74.49416956838468, + "seed": 1700659777, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "WLEzhxAeQRhzi52ESFg-q", + "type": "arrow" + }, + { + "id": "fOuTMdWLfJmuB2ov11q7v", + "type": "arrow" + }, + { + "type": "text", + "id": "uc0dpPTx3z-Oih__yVi7r" + } + ], + "updated": 1666058888073, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 52, + "versionNonce": 1248998326, + "isDeleted": false, + "id": "iBGHqUq7evqeBkhBGjpIY", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 238.39425275353517, + "y": 916.6475734194214, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 3.4230113409226988, + "height": 64.07877230207362, + "seed": 355303279, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888073, + "link": null, + "locked": false, + "startBinding": { + "elementId": "Y9vqkX1na6vJDz7l4PspG", + "focus": -0.05859992602942486, + "gap": 6.973500345576213 + }, + "endBinding": { + "elementId": "OcZsyOg63jPy2vGYoS8dH", + "focus": 0.2503697342260085, + "gap": 13.356826322028098 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -3.4230113409226988, + 64.07877230207362 + ] + ] + }, + { + "type": "arrow", + "version": 221, + "versionNonce": 1942260394, + "isDeleted": false, + "id": "1FA4XZdv1alN1lHogHTl9", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 280.2730259866174, + "y": 874.3827042892433, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 108.9112565402437, + "height": 90.60251065896, + "seed": 958569537, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888073, + "link": null, + "locked": false, + "startBinding": { + "elementId": "Y9vqkX1na6vJDz7l4PspG", + "gap": 10.627860039196209, + "focus": -0.6558398277536983 + }, + "endBinding": { + "elementId": "gCZ4PjyxtYNIC3EGFAKxA", + "gap": 9.53249641010075, + "focus": 0.8392853510122925 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 108.9112565402437, + 90.60251065896 + ] + ] + }, + { + "type": "arrow", + "version": 352, + "versionNonce": 10374390, + "isDeleted": false, + "id": "WLEzhxAeQRhzi52ESFg-q", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 225.846817638945, + "y": 1073.1712589011736, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 130.36469464909064, + "height": 104.34282846123074, + "seed": 974853505, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888073, + "link": null, + "locked": false, + "startBinding": { + "elementId": "OcZsyOg63jPy2vGYoS8dH", + "focus": 0.8039769929584992, + "gap": 11.647681349043125 + }, + "endBinding": { + "elementId": "hLRIE0OQJr_P647NEGYUb", + "focus": 0.5592314137305141, + "gap": 9.641088494019641 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 130.36469464909064, + 104.34282846123074 + ] + ] + }, + { + "type": "arrow", + "version": 447, + "versionNonce": 418489706, + "isDeleted": false, + "id": "fOuTMdWLfJmuB2ov11q7v", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 383.2859330385243, + "y": 1039.3495698552904, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 0.8019442573741458, + "height": 139.71785644664988, + "seed": 1093453345, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888073, + "link": null, + "locked": false, + "startBinding": { + "elementId": "gCZ4PjyxtYNIC3EGFAKxA", + "focus": 0.0034356344242915454, + "gap": 6.831858496986342 + }, + "endBinding": { + "elementId": "hLRIE0OQJr_P647NEGYUb", + "focus": 0.35034121969360427, + "gap": 8.087749554483707 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -0.8019442573741458, + 139.71785644664988 + ] + ] + }, + { + "type": "text", + "version": 86, + "versionNonce": 1685405238, + "isDeleted": false, + "id": "86HkQ09dZUZF0MFiJVz63", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 361.58493261190574, + "y": 992.017711358304, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 44, + "height": 23, + "seed": 1827546465, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [], + "updated": 1666058888073, + "link": null, + "locked": false, + "fontSize": 20, + "fontFamily": 1, + "text": "1", + "baseline": 17, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "gCZ4PjyxtYNIC3EGFAKxA", + "originalText": "1" + }, + { + "type": "text", + "version": 97, + "versionNonce": 925722666, + "isDeleted": false, + "id": "uc0dpPTx3z-Oih__yVi7r", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 344.35656242842015, + "y": 1211.9022606406163, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 53, + "height": 23, + "seed": 1503038895, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [], + "updated": 1666058888073, + "link": null, + "locked": false, + "fontSize": 20, + "fontFamily": 1, + "text": "2", + "baseline": 17, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "hLRIE0OQJr_P647NEGYUb", + "originalText": "2" + }, + { + "type": "text", + "version": 253, + "versionNonce": 719062902, + "isDeleted": false, + "id": "kI_tZyDto7LyRVi7fZo1b", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 458.5859491764503, + "y": 836.3733952089806, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 268, + "height": 25, + "seed": 381074991, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [], + "updated": 1666058888073, + "link": null, + "locked": false, + "fontSize": 20, + "fontFamily": 1, + "text": "Is it just B? Or B and 2?", + "baseline": 18, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "Is it just B? Or B and 2?" + }, + { + "type": "text", + "version": 172, + "versionNonce": 107139818, + "isDeleted": false, + "id": "Ucbb53X_Dwn5QAY3EB0D8", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 459.9740392926316, + "y": 901.9771780808721, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 254, + "height": 25, + "seed": 343812495, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [], + "updated": 1666058888073, + "link": null, + "locked": false, + "fontSize": 20, + "fontFamily": 1, + "text": "Should be both B and 2. ", + "baseline": 18, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "Should be both B and 2. " + }, + { + "type": "text", + "version": 59, + "versionNonce": 605530294, + "isDeleted": false, + "id": "cOQa-Eb9wSAJWzgq9aI7x", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 159.49974308895753, + "y": 1311.600682807928, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 424, + "height": 25, + "seed": 1981785473, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [], + "updated": 1666058888073, + "link": null, + "locked": false, + "fontSize": 20, + "fontFamily": 1, + "text": "How to tackle this case when finding path?", + "baseline": 18, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "How to tackle this case when finding path?" + }, + { + "type": "rectangle", + "version": 52, + "versionNonce": 152888746, + "isDeleted": false, + "id": "hwOHW2_5-VmaE0Y21RH6I", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 174.03219675428193, + "y": 1394.1222450159953, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 49.409398183112216, + "height": 51.5151403321488, + "seed": 578415457, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "type": "text", + "id": "o2Bs8-53MFFZjUFarRyRN" + }, + { + "id": "qCbjrqfXeEu-zEq2Ffb3T", + "type": "arrow" + }, + { + "id": "R6d9DcWsOcD8YWu9N7JxL", + "type": "arrow" + } + ], + "updated": 1666058888074, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 14, + "versionNonce": 1338843638, + "isDeleted": false, + "id": "o2Bs8-53MFFZjUFarRyRN", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 179.03219675428193, + "y": 1407.3798151820697, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 39, + "height": 23, + "seed": 1451199681, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [], + "updated": 1666058888074, + "link": null, + "locked": false, + "fontSize": 20, + "fontFamily": 1, + "text": "A", + "baseline": 17, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "hwOHW2_5-VmaE0Y21RH6I", + "originalText": "A" + }, + { + "type": "rectangle", + "version": 48, + "versionNonce": 948346986, + "isDeleted": false, + "id": "D36mTqaq_mm5bD6sonefu", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 341.06570740140523, + "y": 1402.8757112588514, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 52, + "height": 54.08830058153217, + "seed": 1613404719, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "type": "text", + "id": "PnMoL-isuYyODBEw1k2wf" + }, + { + "id": "NyiyA39uyFO2fv9N82bF4", + "type": "arrow" + } + ], + "updated": 1666058888074, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 14, + "versionNonce": 1895920438, + "isDeleted": false, + "id": "PnMoL-isuYyODBEw1k2wf", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 346.06570740140523, + "y": 1417.4198615496175, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 42, + "height": 23, + "seed": 467983471, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [], + "updated": 1666058888074, + "link": null, + "locked": false, + "fontSize": 20, + "fontFamily": 1, + "text": "B", + "baseline": 17, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "D36mTqaq_mm5bD6sonefu", + "originalText": "B" + }, + { + "type": "arrow", + "version": 83, + "versionNonce": 812319530, + "isDeleted": false, + "id": "qCbjrqfXeEu-zEq2Ffb3T", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 199.85822714717483, + "y": 1452.9508261303365, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 40.22628499953339, + "height": 184.9087119391695, + "seed": 105061935, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888074, + "link": null, + "locked": false, + "startBinding": { + "elementId": "hwOHW2_5-VmaE0Y21RH6I", + "focus": 0.20038022884771262, + "gap": 7.3134407821924015 + }, + "endBinding": null, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 40.22628499953339, + 184.9087119391695 + ] + ] + }, + { + "type": "arrow", + "version": 50, + "versionNonce": 572103798, + "isDeleted": false, + "id": "NyiyA39uyFO2fv9N82bF4", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 370.7066249163197, + "y": 1466.4917868555315, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 113.29459329211295, + "height": 165.0694103466767, + "seed": 366695329, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888074, + "link": null, + "locked": false, + "startBinding": { + "elementId": "D36mTqaq_mm5bD6sonefu", + "focus": -0.6449918259980243, + "gap": 9.527775015147881 + }, + "endBinding": null, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -113.29459329211295, + 165.0694103466767 + ] + ] + }, + { + "type": "rectangle", + "version": 74, + "versionNonce": 1530203626, + "isDeleted": false, + "id": "s54CZ5KG2aYjaNsp6G4VZ", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 217.9317270272469, + "y": 1640.4751908734663, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 63.21475702567511, + "height": 68.62547564180954, + "seed": 638536961, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "aw37qj_R25zNHnnWOQGPF", + "type": "arrow" + } + ], + "updated": 1666058888074, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 59, + "versionNonce": 1759157686, + "isDeleted": false, + "id": "R6d9DcWsOcD8YWu9N7JxL", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 179.45235816032232, + "y": 1451.817691341617, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 111.97732410022672, + "height": 94.94253110981413, + "seed": 1780700449, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888074, + "link": null, + "locked": false, + "startBinding": { + "elementId": "hwOHW2_5-VmaE0Y21RH6I", + "focus": -0.3337409231098588, + "gap": 6.1803059934729845 + }, + "endBinding": { + "elementId": "Uu6icBmJsdOeGkGZzL7sU", + "focus": -0.6678530131421979, + "gap": 9.065078309754199 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -111.97732410022672, + 94.94253110981413 + ] + ] + }, + { + "type": "rectangle", + "version": 72, + "versionNonce": 502251690, + "isDeleted": false, + "id": "Uu6icBmJsdOeGkGZzL7sU", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 36.14857854696129, + "y": 1555.8253007611854, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 54.08830058153217, + "height": 59.489576407760524, + "seed": 1867485153, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "2-6tyGZ_ZmFG9B-mW7C21", + "type": "arrow" + }, + { + "id": "R6d9DcWsOcD8YWu9N7JxL", + "type": "arrow" + } + ], + "updated": 1666058888074, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 299, + "versionNonce": 1248425718, + "isDeleted": false, + "id": "2-6tyGZ_ZmFG9B-mW7C21", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 69.80268177192295, + "y": 1626.4762548378305, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 40.55206125129018, + "height": 95.98595939442657, + "seed": 803727777, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888074, + "link": null, + "locked": false, + "startBinding": { + "elementId": "Uu6icBmJsdOeGkGZzL7sU", + "focus": -0.6031691429467954, + "gap": 11.161377668884597 + }, + "endBinding": { + "elementId": "kMzfHdkQJvza3Y7oH-Stk", + "focus": -0.780062894083163, + "gap": 7.828072832068642 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -40.55206125129018, + 95.98595939442657 + ] + ] + }, + { + "type": "rectangle", + "version": 117, + "versionNonce": 71703402, + "isDeleted": false, + "id": "kMzfHdkQJvza3Y7oH-Stk", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 5.5633820414475395, + "y": 1730.2902870643259, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 49.86265209859994, + "height": 320.63937404792387, + "seed": 1045724623, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "2-6tyGZ_ZmFG9B-mW7C21", + "type": "arrow" + }, + { + "id": "StiyyzlA2m4GVtTLr2uzG", + "type": "arrow" + }, + { + "id": "3HbM7P7WEUHkMZ9blj5wa", + "type": "arrow" + } + ], + "updated": 1666058888074, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 125, + "versionNonce": 1598761014, + "isDeleted": false, + "id": "xG6Ryld1g4RsmlYfeyRyl", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 4.930715117745535, + "y": 1894.3540402860144, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 49.206378200133486, + "height": 156.04210319654658, + "seed": 441826383, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "29BHND6z19Wd0fVvFoxsH", + "type": "arrow" + } + ], + "updated": 1666058888074, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 40, + "versionNonce": 1820532266, + "isDeleted": false, + "id": "7avwsrGLRQ9zXDSv0w1np", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 166.8651192156325, + "y": 1781.2294172122108, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 24.806209083045587, + "height": 32.90812282238835, + "seed": 1685597185, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "aw37qj_R25zNHnnWOQGPF", + "type": "arrow" + }, + { + "id": "9h36VDCVJrRu8C1CMjJtB", + "type": "arrow" + } + ], + "updated": 1666058888074, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 52, + "versionNonce": 1734652278, + "isDeleted": false, + "id": "0uds6kbQkH4ArKucMh0Vg", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 115.45857096740258, + "y": 1831.7105720496534, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 33.93738692214151, + "height": 30.297191413380915, + "seed": 1820056143, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "9h36VDCVJrRu8C1CMjJtB", + "type": "arrow" + }, + { + "id": "29BHND6z19Wd0fVvFoxsH", + "type": "arrow" + } + ], + "updated": 1666058888074, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 47, + "versionNonce": 865192170, + "isDeleted": false, + "id": "aw37qj_R25zNHnnWOQGPF", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 241.67090085091502, + "y": 1720.7483478643207, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 40.703145889786015, + "height": 49.39051260330052, + "seed": 1142455201, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888074, + "link": null, + "locked": false, + "startBinding": { + "elementId": "s54CZ5KG2aYjaNsp6G4VZ", + "focus": -0.5010982988927215, + "gap": 11.647681349044774 + }, + "endBinding": { + "elementId": "7avwsrGLRQ9zXDSv0w1np", + "focus": -0.03852497283277824, + "gap": 11.090556744589435 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -40.703145889786015, + 49.39051260330052 + ] + ] + }, + { + "type": "arrow", + "version": 36, + "versionNonce": 1576735414, + "isDeleted": false, + "id": "9h36VDCVJrRu8C1CMjJtB", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 164.44776499969817, + "y": 1807.2348606133176, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 17.138663679378624, + "height": 16.893151141822955, + "seed": 285622561, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888074, + "link": null, + "locked": false, + "startBinding": { + "elementId": "7avwsrGLRQ9zXDSv0w1np", + "focus": 0.17632017043480894, + "gap": 2.417354215934324 + }, + "endBinding": { + "elementId": "0uds6kbQkH4ArKucMh0Vg", + "focus": -0.2529475107653309, + "gap": 7.582560294512859 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -17.138663679378624, + 16.893151141822955 + ] + ] + }, + { + "type": "arrow", + "version": 53, + "versionNonce": 1627200426, + "isDeleted": false, + "id": "29BHND6z19Wd0fVvFoxsH", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 111.12433040055146, + "y": 1855.3930891338857, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 51.59068265139683, + "height": 34.503954316500995, + "seed": 668671791, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888074, + "link": null, + "locked": false, + "startBinding": { + "elementId": "0uds6kbQkH4ArKucMh0Vg", + "focus": 0.2156267298677806, + "gap": 4.334240566851122 + }, + "endBinding": { + "elementId": "xG6Ryld1g4RsmlYfeyRyl", + "focus": -0.6606370592162203, + "gap": 5.396554431275604 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -51.59068265139683, + 34.503954316500995 + ] + ] + } + ], + "appState": { + "gridSize": null, + "viewBackgroundColor": "#ffffff" + }, + "files": {} +} \ No newline at end of file diff --git a/3rdparty/loro/crates/loro-internal/src/dag/test.rs b/3rdparty/loro/crates/loro-internal/src/dag/test.rs new file mode 100644 index 0000000000..dc3afba8eb --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/dag/test.rs @@ -0,0 +1,1467 @@ +#![allow(unused)] +use arbitrary::Arbitrary; +use im::HashSet; +use loro_common::HasCounter; +use std::{cmp::Ordering, sync::Arc}; + +use super::*; +use crate::{ + array_mut_ref, + change::Lamport, + id::{Counter, PeerID, ID}, + span::HasIdSpan, +}; + +#[derive(Debug, PartialEq, Eq, Clone)] +struct TestNode { + id: ID, + lamport: Lamport, + len: usize, + deps: Arc, +} + +impl TestNode { + fn new(id: ID, lamport: Lamport, deps: Frontiers, len: usize) -> Self { + Self { + id, + lamport, + deps: Arc::new(deps), + len, + } + } +} + +impl DagNode for TestNode { + fn deps(&self) -> &Frontiers { + &self.deps + } +} + +impl Sliceable for TestNode { + fn slice(&self, from: usize, to: usize) -> Self { + Self { + id: self.id.inc(from as Counter), + lamport: self.lamport + from as Lamport, + len: to - from, + deps: if from > 0 { + Arc::new(self.id.inc(from as Counter - 1).into()) + } else { + self.deps.clone() + }, + } + } +} + +impl HasLamport for TestNode { + fn lamport(&self) -> Lamport { + self.lamport + } +} + +impl HasId for TestNode { + fn id_start(&self) -> ID { + self.id + } +} + +impl HasCounter for TestNode { + fn ctr_start(&self) -> Counter { + self.id.counter + } +} + +impl HasLength for TestNode { + fn content_len(&self) -> usize { + self.len + } +} + +#[derive(Debug, PartialEq, Eq)] +struct TestDag { + nodes: FxHashMap>, + frontier: Frontiers, + version_vec: VersionVector, + next_lamport: Lamport, + client_id: PeerID, +} + +impl Dag for TestDag { + type Node = TestNode; + + fn get(&self, id: ID) -> Option { + let arr = self.nodes.get(&id.peer)?; + arr.binary_search_by(|node| { + if node.id.counter > id.counter { + Ordering::Greater + } else if node.id.counter + node.len as i32 <= id.counter { + Ordering::Less + } else { + Ordering::Equal + } + }) + .map_or(None, |x| Some(arr[x].clone())) + } + + fn frontier(&self) -> &Frontiers { + &self.frontier + } + + fn vv(&self) -> &VersionVector { + &self.version_vec + } + + fn contains(&self, id: ID) -> bool { + self.version_vec.includes_id(id) + } +} + +impl TestDag { + pub fn new(client_id: PeerID) -> Self { + Self { + nodes: FxHashMap::default(), + frontier: Frontiers::new(), + version_vec: VersionVector::new(), + next_lamport: 0, + client_id, + } + } + + fn get_last_node(&mut self) -> &mut TestNode { + self.nodes + .get_mut(&self.client_id) + .unwrap() + .last_mut() + .unwrap() + } + + fn push(&mut self, len: usize) { + let client_id = self.client_id; + let counter = self.version_vec.entry(client_id).or_insert(0); + let id = ID::new(client_id, *counter); + *counter += len as Counter; + let deps = std::mem::replace(&mut self.frontier, id.inc(len as Counter - 1).into()); + self.nodes.entry(client_id).or_default().push(TestNode::new( + id, + self.next_lamport, + deps, + len, + )); + self.next_lamport += len as u32; + } + + fn merge(&mut self, other: &TestDag) { + let mut pending = Vec::new(); + for (_, nodes) in other.nodes.iter() { + for (i, node) in nodes.iter().enumerate() { + if self._try_push_node(node, &mut pending, i) { + break; + } + } + } + + let mut current = pending; + let mut pending = Vec::new(); + while !pending.is_empty() || !current.is_empty() { + if current.is_empty() { + std::mem::swap(&mut pending, &mut current); + } + + let (client_id, index) = current.pop().unwrap(); + let node_vec = other.nodes.get(&client_id).unwrap(); + #[allow(clippy::needless_range_loop)] + for i in index..node_vec.len() { + let node = &node_vec[i]; + if self._try_push_node(node, &mut pending, i) { + break; + } + } + } + } + + fn update_frontier(frontier: &mut Frontiers, new_node_id: ID, new_node_deps: &Frontiers) { + frontier.retain(|x| { + if x.peer == new_node_id.peer && x.counter <= new_node_id.counter { + return false; + } + + !new_node_deps + .iter() + .any(|y| y.peer == x.peer && y.counter >= x.counter) + }); + + // nodes from the same client with `counter < new_node_id.counter` + // are filtered out from frontier. + if frontier.iter().all(|x| x.peer != new_node_id.peer) { + frontier.push(new_node_id); + } + } + + fn _try_push_node( + &mut self, + node: &TestNode, + pending: &mut Vec<(PeerID, usize)>, + i: usize, + ) -> bool { + let client_id = node.id.peer; + if self.contains(node.id_last()) { + return false; + } + if node.deps.iter().any(|dep| !self.contains(dep)) { + pending.push((client_id, i)); + return true; + } + Self::update_frontier(&mut self.frontier, node.id_last(), &node.deps); + let contains_start = self.contains(node.id_start()); + let arr = self.nodes.entry(client_id).or_default(); + if contains_start { + arr.pop(); + arr.push(node.clone()); + } else { + arr.push(node.clone()); + } + self.version_vec.set_end(node.id_end()); + self.next_lamport = self.next_lamport.max(node.lamport + node.len as u32); + false + } +} + +/// ```mermaid /// flowchart RL +/// subgraph client0 +/// 0-1("c0: [1, 3)") --> 0-0("c0: [0, 1)") +/// end +/// +/// subgraph client1 +/// 1-0("c1: [0, 2)") +/// end +/// +/// 0-1 --> 1-0 +/// ``` +#[test] +fn test_dag() { + let mut a = TestDag::new(0); + let mut b = TestDag::new(1); + a.push(1); + assert_eq!(a.frontier().len(), 1); + assert_eq!(a.frontier().as_single().unwrap().counter, 0); + b.push(1); + a.merge(&b); + assert_eq!(a.frontier().len(), 2); + a.push(1); + assert_eq!(a.frontier().len(), 1); + // a: 0 --(merge)--- 1 + // ↑ + // | + // b: 0 ---- + assert_eq!( + a.frontier().iter().next().unwrap(), + ID { + peer: 0, + counter: 1 + } + ); + + // a: 0 --(merge)--- 1 --- 2 ------- + // ↑ | + // | ↓ + // b: 0 ------------1----------(merge) + a.push(1); + b.push(1); + b.merge(&a); + assert_eq!(b.next_lamport, 3); + assert_eq!(b.frontier().len(), 2); + // println!("{}", b.mermaid()); + assert_eq!( + b.find_common_ancestor(&[ID::new(0, 2)].into(), &[ID::new(1, 1)].into()) + .0 + .len(), + 0 + ); +} + +#[derive(Debug, Clone, Copy, Arbitrary)] +pub struct Interaction { + dag_idx: usize, + merge_with: Option, + len: usize, +} + +impl Interaction { + fn generate(rng: &mut impl rand::Rng, num: usize) -> Self { + if rng.gen_bool(0.5) { + let dag_idx = rng.gen_range(0..num); + let merge_with = (rng.gen_range(1..num - 1) + dag_idx) % num; + Self { + dag_idx, + merge_with: Some(merge_with), + len: rng.gen_range(1..10), + } + } else { + Self { + dag_idx: rng.gen_range(0..num), + merge_with: None, + len: rng.gen_range(1..10), + } + } + } + + fn apply(&self, dags: &mut [TestDag]) { + if let Some(merge_with) = self.merge_with { + if merge_with != self.dag_idx { + let (from, to) = array_mut_ref!(dags, [self.dag_idx, merge_with]); + from.merge(to); + } + } + + dags[self.dag_idx].push(self.len); + } +} + +// prop_compose! { +// fn gen_interaction(num: usize) ( +// dag_idx in 0..num, +// merge_with in 0..num, +// length in 1..10, +// should_merge in 0..2 +// ) -> Interaction { +// Interaction { +// dag_idx, +// merge_with: if should_merge == 1 && merge_with != dag_idx { Some(merge_with) } else { None }, +// len: length as usize, +// } +// } +// } + +fn preprocess(interactions: &mut [Interaction], num: i32) { + for interaction in interactions.iter_mut() { + interaction.dag_idx %= num as usize; + if let Some(ref mut merge_with) = interaction.merge_with { + *merge_with %= num as usize; + if *merge_with == interaction.dag_idx { + *merge_with = (*merge_with + 1) % num as usize; + } + } + + interaction.len = (interaction.len % 128).max(1); + } +} + +mod iter { + use super::*; + + #[test] + fn test() { + let mut a = TestDag::new(0); + let mut b = TestDag::new(1); + // 0-0 + a.push(1); + // 1-0 + b.push(1); + a.merge(&b); + // 0-1 + a.push(1); + b.merge(&a); + // 1-1 + b.push(1); + a.merge(&b); + // 0-2 + a.push(1); + + let mut count = 0; + for (node, vv) in a.iter_with_vv() { + count += 1; + if node.id == ID::new(0, 0) { + assert_eq!(vv, vec![ID::new(0, 0)].into()); + } else if node.id == ID::new(0, 2) { + assert_eq!(vv, vec![ID::new(0, 2), ID::new(1, 1)].into()); + } + } + + assert_eq!(count, 5); + } +} + +mod dfs { + #[allow(unused)] + #[allow(dead_code)] + use super::*; + use crate::{allocation::calc_critical_version_dfs, delta::DeltaValue}; + use rand::{rngs::StdRng, SeedableRng}; + + #[test] + fn test_dfs_small() { + let mut a = TestDag::new(0); + let mut b = TestDag::new(1); + let mut c = TestDag::new(2); + a.push(10); + b.merge(&a); + b.push(3); + c.merge(&b); + c.push(4); + a.push(3); + a.merge(&c); + a.push(2); + b.merge(&a); + assert_eq!( + calc_critical_version_dfs::( + &b, + &[ID { + peer: 0, + counter: 13, + }] + .into(), + &[ID { + peer: 0, + counter: 9, + }] + .into(), + ) + .length(), + 0 + ); + } + + #[test] + fn test_dfs_big() { + let num = 5; + let mut rng = StdRng::seed_from_u64(100); + let mut dags = (0..num).map(TestDag::new).collect::>(); + for _ in 0..100 { + Interaction::generate(&mut rng, num as usize).apply(&mut dags); + } + for i in 1..num { + let (a, other) = array_mut_ref!(&mut dags, [0, i as usize]); + a.merge(other); + } + let start = dags[0].frontier(); + let ends = [ + ID { + peer: 3, + counter: 7, + }, + ID { + peer: 4, + counter: 6, + }, + ] + .into(); + assert_eq!( + calc_critical_version_dfs(&dags[0], start, &ends).length(), + 0 + ); + } +} + +mod bfs { + use arref::array_mut_ref; + use loro_common::ID; + use rand::{rngs::StdRng, SeedableRng}; + + use crate::{ + allocation::calc_critical_version, + dag::{ + test::{TestDag, TestNode}, + Dag, Interaction, + }, + delta::DeltaValue as _, + }; + + #[test] + fn test_bfs_small() { + let mut a = TestDag::new(0); + let mut b = TestDag::new(1); + let mut c = TestDag::new(2); + a.push(10); + b.merge(&a); + b.push(3); + c.merge(&b); + c.push(4); + a.push(3); + a.merge(&c); + a.push(2); + b.merge(&a); + assert_eq!( + calc_critical_version::( + &b, + &[ID { + peer: 0, + counter: 13, + }] + .into(), + ) + .length(), + 0 + ); + } + + #[test] + fn test_bfs_big() { + let num = 5; + let mut rng = StdRng::seed_from_u64(100); + let mut dags = (0..num).map(TestDag::new).collect::>(); + for _ in 0..100 { + Interaction::generate(&mut rng, num as usize).apply(&mut dags); + } + for i in 1..num { + let (a, other) = array_mut_ref!(&mut dags, [0, i as usize]); + a.merge(other); + } + let start = dags[0].frontier(); + let ends = [ID { + peer: 3, + counter: 7, + }]; + assert_eq!(calc_critical_version(&dags[0], start).length(), 0); + } +} + +mod mermaid { + use super::*; + use rand::{rngs::StdRng, SeedableRng}; + + #[test] + fn simple() { + let mut a = TestDag::new(0); + let mut b = TestDag::new(1); + // 0-0 + a.push(1); + // 1-0 + b.push(1); + a.merge(&b); + // 0-1 + a.push(1); + b.merge(&a); + // 1-1 + b.push(1); + a.merge(&b); + // 0-2 + a.push(1); + + println!("{}", a.mermaid()); + } + + #[test] + fn three() { + let mut a = TestDag::new(0); + let mut b = TestDag::new(1); + let mut c = TestDag::new(2); + a.push(10); + b.merge(&a); + b.push(3); + c.merge(&b); + c.push(4); + a.push(3); + a.merge(&c); + a.push(2); + b.merge(&a); + println!("{}", a.mermaid()); + } + + #[test] + fn gen_graph() { + let num = 5; + let mut rng = StdRng::seed_from_u64(100); + let mut dags = (0..num).map(TestDag::new).collect::>(); + for _ in 0..100 { + Interaction::generate(&mut rng, num as usize).apply(&mut dags); + } + for i in 1..num { + let (a, other) = array_mut_ref!(&mut dags, [0, i as usize]); + a.merge(other); + } + println!("{}", dags[0].mermaid()); + } +} + +mod get_version_vector { + #[allow(unused)] + #[allow(dead_code)] + use super::*; + + #[test] + fn vv() { + let mut a = TestDag::new(0); + let mut b = TestDag::new(1); + a.push(1); + b.push(1); + a.merge(&b); + a.push(1); + let actual = a.get_vv(ID::new(0, 0)); + assert_eq!(actual, vec![ID::new(0, 0)].into()); + let actual = a.get_vv(ID::new(0, 1)); + assert_eq!(actual, vec![ID::new(0, 1), ID::new(1, 0)].into()); + + let mut c = TestDag::new(2); + c.merge(&a); + b.push(1); + c.merge(&b); + c.push(1); + let actual = c.get_vv(ID::new(2, 0)); + assert_eq!( + actual, + vec![ID::new(0, 1), ID::new(1, 1), ID::new(2, 0)].into() + ); + } +} + +#[test] +fn test_alloc() { + use rand::thread_rng; + for _ in 1..10000 { + let num = 4; + let mut rng = thread_rng(); + let mut dags = (0..num).map(TestDag::new).collect::>(); + for _ in 0..20 { + Interaction::generate(&mut rng, num as usize).apply(&mut dags); + } + for i in 1..num { + let (a, other) = array_mut_ref!(&mut dags, [0, i as usize]); + a.merge(other); + } + let a = &dags[0]; + let start = a.frontier(); + let end = crate::allocation::get_end_list(a, start); + let critical_version_actual: HashSet = + crate::allocation::calc_critical_version(a, start) + .into_iter() + .collect(); + let critical_version: HashSet = + crate::allocation::calc_critical_version_dfs(a, start, &end) + .into_iter() + .collect(); + assert!(critical_version_actual.eq(&critical_version)); + } +} + +pub fn fuzz_alloc_tree(num: usize, mut actions: Vec) { + preprocess(&mut actions, num as i32); + let mut dags = (0..num as u64).map(TestDag::new).collect::>(); + for action in actions { + action.apply(&mut dags); + } + + for i in 1..num { + let (a, other) = array_mut_ref!(&mut dags, [0, i]); + a.merge(other); + } + + let a = &dags[0]; + let start = a.frontier(); + let end = crate::allocation::get_end_list(a, start); + let actual: HashSet = crate::allocation::calc_critical_version(a, start) + .into_iter() + .collect(); + let expected: HashSet = crate::allocation::calc_critical_version_dfs(a, start, &end) + .into_iter() + .collect(); + assert!(actual.eq(&expected)); +} + +#[test] +fn failed_fuzz() { + fuzz_alloc_tree( + 10, + vec![Interaction { + dag_idx: 14685055086129564619, + merge_with: Some(10), + len: 777939436809428736, + }], + ) +} + +// #[cfg(test)] +// mod find_path { +// use crate::{fx_map, span::CounterSpan}; + +// use super::*; + +// #[test] +// fn retreat_to_beginning() { +// let mut a = TestDag::new(0); +// let mut b = TestDag::new(1); +// a.push(1); +// b.push(1); +// a.merge(&b); +// let actual = a.find_path(&[ID::new(0, 0)], &[ID::new(1, 0)]); +// assert_eq!( +// actual, +// VersionVectorDiff { +// left: fx_map!(0 => CounterSpan { start: 0, end: 1 }), +// right: fx_map!(1 => CounterSpan { start: 0, end: 1 }) +// } +// ); +// } + +// #[test] +// fn one_path() { +// let mut a = TestDag::new(0); +// let mut b = TestDag::new(1); +// // 0 - 0 +// a.push(1); +// b.merge(&a); +// // 1 - 0 +// b.push(1); +// // 0 - 1 +// a.push(1); +// a.merge(&b); +// let actual = a.find_path(&[ID::new(0, 1)], &[ID::new(1, 0)]); +// assert_eq!( +// actual, +// VersionVectorDiff { +// left: fx_map!(0 => CounterSpan { start: 1, end: 2 }), +// right: fx_map!(1 => CounterSpan { start: 0, end: 1 }) +// } +// ); +// } + +// #[test] +// fn middle() { +// let mut a = TestDag::new(0); +// let mut b = TestDag::new(1); +// a.push(2); +// b.push(1); +// b.merge(&a); +// b.push(2); +// a.push(2); +// b.merge(&a); +// // println!("{}", b.mermaid()); +// let actual = b.find_path(&[ID::new(0, 3)], &[ID::new(1, 2)]); +// assert_eq!( +// actual, +// VersionVectorDiff { +// left: fx_map!(0 => CounterSpan { start: 2, end: 4 }), +// right: fx_map!(1 => CounterSpan { start: 0, end: 3 }) +// } +// ); +// let actual = b.find_path(&[ID::new(1, 1), ID::new(1, 2)], &[ID::new(0, 3)]); +// assert_eq!( +// actual, +// VersionVectorDiff { +// left: fx_map!(1 => CounterSpan { start: 0, end: 3 }), +// right: fx_map!(0 => CounterSpan { start: 2, end: 4 }) +// } +// ); +// } + +// fn test_find_path( +// dag_num: i32, +// mut interactions: Vec, +// ) -> Result<(), TestCaseError> { +// preprocess(&mut interactions, dag_num); +// let mut dags = Vec::new(); +// for i in 0..dag_num { +// dags.push(TestDag::new(i as PeerID)); +// } + +// for interaction in interactions.iter_mut() { +// interaction.apply(&mut dags); +// } + +// for i in 1..dag_num { +// let (a, b) = array_mut_ref!(&mut dags, [0, i as usize]); +// a.merge(b); +// } + +// let a = &dags[0]; +// let mut nodes = Vec::new(); +// for (node, vv) in a.iter_with_vv() { +// nodes.push((node, vv)); +// } + +// // println!("{}", a.mermaid()); +// let vec: Vec<_> = nodes.iter().enumerate().collect(); +// for &(i, (node, vv)) in vec.iter() { +// if i > 3 { +// break; +// } + +// for &(j, (other_node, other_vv)) in vec.iter() { +// if i >= j { +// continue; +// } + +// let actual = a.find_path(&[node.id], &[other_node.id]); +// let expected = vv.diff(other_vv); +// prop_assert_eq!( +// actual, +// expected, +// "\ni={} j={} node={} other={}", +// i, +// j, +// node.id, +// other_node.id +// ); + +// for iter in nodes[j + 1..].iter() { +// let mut vv = vv.clone(); +// vv.merge(other_vv); +// let actual = a.find_path(&[node.id, other_node.id], &[iter.0.id]); +// let expected = vv.diff(&iter.1); +// prop_assert_eq!(actual, expected); +// } +// } +// } + +// Ok(()) +// } + +// #[test] +// fn issue() { +// if let Err(err) = test_find_path( +// 5, +// vec![ +// Interaction { +// dag_idx: 1, +// merge_with: None, +// len: 3, +// }, +// Interaction { +// dag_idx: 1, +// merge_with: None, +// len: 3, +// }, +// Interaction { +// dag_idx: 4, +// merge_with: Some(1), +// len: 1, +// }, +// Interaction { +// dag_idx: 0, +// merge_with: None, +// len: 1, +// }, +// Interaction { +// dag_idx: 2, +// merge_with: Some(0), +// len: 1, +// }, +// Interaction { +// dag_idx: 0, +// merge_with: None, +// len: 1, +// }, +// Interaction { +// dag_idx: 2, +// merge_with: Some(0), +// len: 1, +// }, +// Interaction { +// dag_idx: 3, +// merge_with: Some(0), +// len: 1, +// }, +// ], +// ) { +// panic!("{}", err); +// } +// } + +// proptest! { +// #[test] +// fn proptest_path( +// interactions in prop::collection::vec(gen_interaction(5), 0..10 * PROPTEST_FACTOR_10), +// ) { +// test_find_path(5, interactions)?; +// } + +// #[test] +// fn proptest_path_large( +// interactions in prop::collection::vec(gen_interaction(10), 0..PROPTEST_FACTOR_10 * PROPTEST_FACTOR_10 + 10), +// ) { +// test_find_path(10, interactions)?; +// } +// } +// } + +// mod find_common_ancestors { + +// use super::*; + +// #[test] +// fn siblings() { +// let mut a = TestDag::new(0); +// a.push(5); +// let actual = a +// .find_common_ancestor(&[ID::new(0, 2)], &[ID::new(0, 4)]) +// .0 +// .first() +// .copied(); +// assert_eq!(actual, Some(ID::new(0, 2))); +// assert_eq!( +// a.find_common_ancestor(&[ID::new(0, 2)], &[ID::new(0, 4)]).1, +// DiffMode::Linear +// ); +// } + +// #[test] +// fn no_common_ancestors() { +// let mut a = TestDag::new(0); +// let mut b = TestDag::new(1); +// a.push(1); +// b.push(1); +// a.merge(&b); +// let actual = a +// .find_common_ancestor(&[ID::new(0, 0)], &[ID::new(1, 0)]) +// .0 +// .first() +// .copied(); +// assert_eq!(actual, None); + +// // interactions between b and c +// let mut c = TestDag::new(2); +// c.merge(&b); +// c.push(2); +// b.merge(&c); +// b.push(3); + +// // should no exist any common ancestor between a and b +// let actual = a +// .find_common_ancestor(&[ID::new(0, 0)], &[ID::new(1, 0)]) +// .0 +// .first() +// .copied(); +// assert_eq!(actual, None); +// assert_eq!( +// a.find_common_ancestor(&[ID::new(0, 0)], &[ID::new(1, 0)]).1, +// DiffMode::Checkout +// ) +// } + +// #[test] +// fn no_common_ancestors_when_there_is_an_redundant_node() { +// let mut a = TestDag::new(0); +// let mut b = TestDag::new(1); +// a.push(1); +// b.push(1); +// b.merge(&a); +// b.push(1); +// a.push(4); +// a.merge(&b); +// println!("{}", a.mermaid()); +// let actual = a +// .find_common_ancestor(&[ID::new(0, 4)], &[ID::new(0, 1), ID::new(1, 1)]) +// .0 +// .first() +// .copied(); +// assert_eq!(actual, None); +// let actual = a +// .find_common_ancestor(&[ID::new(0, 4)], &[ID::new(1, 1)]) +// .0 +// .first() +// .copied(); +// assert_eq!(actual, None); +// } + +// #[test] +// fn dep_in_middle() { +// let mut a = TestDag::new(0); +// let mut b = TestDag::new(1); +// a.push(3); +// b.merge(&a); +// b.push(9); +// a.push(2); +// b.merge(&a); +// println!("{}", b.mermaid()); +// assert_eq!( +// b.find_common_ancestor(&[ID::new(0, 3)], &[ID::new(1, 8)]) +// .0 +// .first() +// .copied(), +// Some(ID::new(0, 2)) +// ); +// } + +// /// ![](https://mermaid.ink/img/pako:eNqNkTFPwzAQhf_K6SYqOZJ9CYsHJroxwYgXY7skInEq1xFCVf87jg5XVQQSnk6fz_feO5_RzT6gxsM4f7repgzPTyZCOafl7T3ZYw9uHELMkqls2juDTmp4bQV0O4M7aJqHwqlyEtDecFW5EkA3XFYuBaiVs0CInotfXSimqunW16q87gTcX6cqdqe27hSrKVZr_6tGTImn0nYqcWbaZiZWI1ajP9WK2zqnClFd5jVn3SIKnEKa7ODLb53Xa4O5D1MwqEvpbfowaOKl9C1Hb3PY-yHPCfXBjqcg0C55fvmKDnVOS6hNj4Mtgaefrss3dp6HFg) +// #[test] +// fn large_lamport_with_longer_path() { +// let mut a0 = TestDag::new(0); +// let mut a1 = TestDag::new(1); +// let mut a2 = TestDag::new(2); + +// a0.push(3); +// a1.merge(&a0); +// a2.merge(&a0); +// a1.push(3); +// a2.push(2); +// a2.push(1); +// a1.merge(&a2); +// a2.push(1); +// a1.push(1); +// a1.merge(&a2); +// a1.push(1); +// a0.push(1); +// a1.merge(&a2); +// a1.merge(&a0); +// println!("{}", a1.mermaid()); +// assert_eq!( +// a1.find_common_ancestor(&[ID::new(0, 3)], &[ID::new(1, 4)]) +// .0 +// .first() +// .copied(), +// Some(ID::new(0, 2)) +// ); +// assert_eq!( +// a1.find_common_ancestor(&[ID::new(2, 3)], &[ID::new(1, 3)]) +// .0 +// .iter() +// .copied() +// .collect::>(), +// vec![ID::new(0, 2)] +// ); +// } +// } + +// mod find_common_ancestors_proptest { + +// use crate::{ +// array_mut_ref, +// span::HasIdSpan, +// tests::{PROPTEST_FACTOR_1, PROPTEST_FACTOR_10}, +// }; + +// use super::*; + +// proptest! { +// #[test] +// fn test_2dags( +// before_merged_insertions in prop::collection::vec(gen_interaction(2), 0..100 * PROPTEST_FACTOR_10), +// after_merged_insertions in prop::collection::vec(gen_interaction(2), 0..100 * PROPTEST_FACTOR_10) +// ) { +// test_single_common_ancestor(2, before_merged_insertions, after_merged_insertions)?; +// } + +// #[test] +// fn test_4dags( +// before_merged_insertions in prop::collection::vec(gen_interaction(4), 0..50 * PROPTEST_FACTOR_10), +// after_merged_insertions in prop::collection::vec(gen_interaction(4), 0..50 * PROPTEST_FACTOR_10) +// ) { +// test_single_common_ancestor(4, before_merged_insertions, after_merged_insertions)?; +// } + +// #[test] +// fn test_10dags( +// before_merged_insertions in prop::collection::vec(gen_interaction(10), 0..10 * PROPTEST_FACTOR_10 * PROPTEST_FACTOR_10), +// after_merged_insertions in prop::collection::vec(gen_interaction(10), 0..10 * PROPTEST_FACTOR_10 * PROPTEST_FACTOR_10) +// ) { +// test_single_common_ancestor(10, before_merged_insertions, after_merged_insertions)?; +// } + +// #[test] +// fn test_mul_ancestors_5dags( +// before_merged_insertions in prop::collection::vec(gen_interaction(10), 0..30 + PROPTEST_FACTOR_1 * 500), +// after_merged_insertions in prop::collection::vec(gen_interaction(10), 0..30 + PROPTEST_FACTOR_1 * 500) +// ) { +// test_mul_ancestors::<2>(5, before_merged_insertions, after_merged_insertions)?; +// } + +// #[test] +// fn test_mul_ancestors_10dags( +// before_merged_insertions in prop::collection::vec(gen_interaction(10), 0..10 * PROPTEST_FACTOR_10 * PROPTEST_FACTOR_10), +// after_merged_insertions in prop::collection::vec(gen_interaction(10), 0..10 * PROPTEST_FACTOR_10 * PROPTEST_FACTOR_10) +// ) { +// test_mul_ancestors::<3>(10, before_merged_insertions, after_merged_insertions)?; +// } + +// #[test] +// fn test_mul_ancestors_15dags_2( +// before_merged_insertions in prop::collection::vec(gen_interaction(15), 0..10 * PROPTEST_FACTOR_10 * PROPTEST_FACTOR_10), +// after_merged_insertions in prop::collection::vec(gen_interaction(15), 0..10 * PROPTEST_FACTOR_10 * PROPTEST_FACTOR_10) +// ) { +// test_mul_ancestors::<5>(15, before_merged_insertions, after_merged_insertions)?; +// } +// } + +// #[test] +// fn issue_0() { +// if let Err(err) = test_mul_ancestors::<3>( +// 10, +// vec![ +// Interaction { +// dag_idx: 1, +// merge_with: None, +// len: 1, +// }, +// Interaction { +// dag_idx: 8, +// merge_with: None, +// len: 2, +// }, +// ], +// vec![Interaction { +// dag_idx: 1, +// merge_with: None, +// len: 1, +// }], +// ) { +// println!("{}", err); +// panic!(); +// } +// } + +// #[test] +// fn issue_1() { +// test_single_common_ancestor( +// 2, +// vec![], +// vec![ +// Interaction { +// dag_idx: 0, +// merge_with: None, +// len: 1, +// }, +// Interaction { +// dag_idx: 0, +// merge_with: None, +// len: 1, +// }, +// ], +// ) +// .unwrap(); +// } + +// fn test_single_common_ancestor( +// dag_num: i32, +// mut before_merge_insertion: Vec, +// mut after_merge_insertion: Vec, +// ) -> Result<(), TestCaseError> { +// preprocess(&mut before_merge_insertion, dag_num); +// preprocess(&mut after_merge_insertion, dag_num); +// let mut dags = Vec::new(); +// for i in 0..dag_num { +// dags.push(TestDag::new(i as PeerID)); +// } + +// for interaction in before_merge_insertion { +// apply(interaction, &mut dags); +// } + +// for dag_idx in 1..dags.len() { +// let (dag0, dag) = arref::array_mut_ref!(&mut dags, [0, dag_idx]); +// dag0.merge(dag); +// } + +// dags[0].push(1); +// let expected = dags[0].frontier()[0]; +// for dag_idx in 1..dags.len() { +// let (dag0, dag) = arref::array_mut_ref!(&mut dags, [0, dag_idx]); +// dag.merge(dag0); +// } +// for interaction in after_merge_insertion.iter_mut() { +// if let Some(merge) = interaction.merge_with { +// // odd dag merges with the odd +// // even dag merges with the even +// if merge % 2 != interaction.dag_idx % 2 { +// interaction.merge_with = None; +// } +// } + +// apply(*interaction, &mut dags); +// } + +// let (dag0, dag1) = array_mut_ref!(&mut dags, [0, 1]); +// dag1.push(1); +// dag0.merge(dag1); +// // println!("{}", dag0.mermaid()); +// let a = dags[0].nodes.get(&0).unwrap().last().unwrap().id_last(); +// let b = dags[1].nodes.get(&1).unwrap().last().unwrap().id_last(); +// let actual = dags[0].find_common_ancestor(&[a], &[b]); +// prop_assert_eq!(&**actual.0, &[expected]); +// Ok(()) +// } + +// fn apply(interaction: Interaction, dags: &mut [TestDag]) { +// let Interaction { +// dag_idx, +// len, +// merge_with, +// } = interaction; +// if let Some(merge_with) = merge_with { +// let (dag, merge_target): (&mut TestDag, &mut TestDag) = +// array_mut_ref!(dags, [dag_idx, merge_with]); +// dag.push(len); +// dag.merge(merge_target); +// } else { +// dags[dag_idx].push(len); +// } +// } + +// fn test_mul_ancestors( +// dag_num: i32, +// mut before_merge_insertion: Vec, +// mut after_merge_insertion: Vec, +// ) -> Result<(), TestCaseError> { +// assert!(dag_num - 2 >= N as i32); +// preprocess(&mut before_merge_insertion, dag_num); +// preprocess(&mut after_merge_insertion, dag_num); +// let mut dags = Vec::new(); +// for i in 0..dag_num { +// dags.push(TestDag::new(i as PeerID)); +// } + +// for mut interaction in before_merge_insertion { +// if interaction.dag_idx < N { +// // cannot act on first N nodes +// interaction.dag_idx = interaction.dag_idx % (dags.len() - N) + N; +// } +// if let Some(merge) = interaction.merge_with { +// if interaction.dag_idx == merge { +// let next_merge = (merge + 1) % dags.len(); +// interaction.merge_with = Some(next_merge); +// } +// } + +// apply(interaction, &mut dags); +// } + +// for target in 0..N { +// for i in N..dags.len() { +// let (target, dag): (&mut TestDag, &mut TestDag) = +// arref::array_mut_ref!(&mut dags, [target, i]); +// target.merge(dag); +// } +// } + +// let mut expected = Vec::with_capacity(N); +// for dag in dags[0..N].iter_mut() { +// dag.push(1); +// expected.push(dag.frontier[0]); +// } + +// for target in 0..N { +// for i in N..dags.len() { +// let (target, dag): (&mut TestDag, &mut TestDag) = +// arref::array_mut_ref!(&mut dags, [target, i]); +// dag.merge(target); +// } +// } + +// let mut merged_to_even = [false; N]; +// let mut merged_to_odd = [false; N]; + +// for interaction in after_merge_insertion.iter_mut() { +// if interaction.dag_idx < N { +// // cannot act on first N nodes +// interaction.dag_idx = interaction.dag_idx % (dags.len() - N) + N; +// } +// if let Some(mut merge) = interaction.merge_with { +// if interaction.dag_idx == merge { +// let next_merge = (merge + 1) % dags.len(); +// interaction.merge_with = Some(next_merge); +// merge = next_merge; +// } + +// // odd dag merges with the odd +// // even dag merges with the even +// if merge >= N && merge % 2 != interaction.dag_idx % 2 { +// interaction.merge_with = None; +// } +// if merge < N { +// if interaction.dag_idx % 2 == 0 { +// merged_to_even[merge] = true; +// } else { +// merged_to_odd[merge] = true; +// } +// } +// } + +// if dags[interaction.dag_idx].is_first() { +// // need to merge to one of the common ancestors first +// let target = interaction.dag_idx % N; +// let (dag, target) = arref::array_mut_ref!(&mut dags, [interaction.dag_idx, target]); +// dag.merge(target); +// } + +// apply(*interaction, &mut dags); +// } + +// // make common ancestor dags be merged to opposite side (even/odd) +// for i in 0..N { +// let (odd, even) = if N % 2 == 0 { (N + 1, N) } else { (N, N + 1) }; +// if !merged_to_even[i] && i % 2 != 0 { +// let (dag_a, dag_b) = array_mut_ref!(&mut dags, [even, i]); +// dag_a.merge(dag_b); +// dag_a.push(1); +// } +// if !merged_to_odd[i] && i % 2 == 0 { +// let (dag_a, dag_b) = array_mut_ref!(&mut dags, [odd, i]); +// dag_a.merge(dag_b); +// dag_a.push(1); +// } +// } + +// // merge with all odds or evens +// for i in dags.len() - 2..dags.len() { +// if i % 2 == 0 { +// for target in (0..dags.len() - 2).step_by(2) { +// let (dag_a, dag_b) = array_mut_ref!(&mut dags, [i, target]); +// dag_a.merge(dag_b); +// dag_a.push(1); +// } +// } else { +// for target in (1..dags.len() - 2).step_by(2) { +// let (dag_a, dag_b) = array_mut_ref!(&mut dags, [i, target]); +// dag_a.merge(dag_b); +// dag_a.push(1); +// } +// } +// } + +// let len = dags.len(); +// let (dag_a, dag_b) = array_mut_ref!(&mut dags, [len - 2, len - 1]); +// dag_a.push(1); +// dag_b.push(1); +// dag_a.merge(dag_b); +// let a = dag_a.get_last_node().id; +// let b = dag_b.get_last_node().id; +// let mut actual = dag_a.find_common_ancestor(&[a], &[b]).0; +// actual.sort(); +// let actual = actual.iter().copied().collect::>(); +// if actual != expected { +// println!("{}", dag_a.mermaid()); +// } + +// prop_assert_eq!(actual, expected); +// Ok(()) +// } +// } + +// #[cfg(test)] +// mod dag_partial_iter { +// use loro_common::HasCounterSpan; + +// use crate::{dag::iter::IterReturn, tests::PROPTEST_FACTOR_10}; + +// use super::*; + +// fn test_partial_iter( +// dag_num: i32, +// mut interactions: Vec, +// ) -> Result<(), TestCaseError> { +// preprocess(&mut interactions, dag_num); +// let mut dags = Vec::new(); +// for i in 0..dag_num { +// dags.push(TestDag::new(i as PeerID)); +// } + +// for interaction in interactions.iter_mut() { +// interaction.apply(&mut dags); +// } + +// for i in 1..dag_num { +// let (a, b) = array_mut_ref!(&mut dags, [0, i as usize]); +// a.merge(b); +// } + +// let a = &dags[0]; +// let mut nodes = Vec::new(); +// for (node, vv) in a.iter_with_vv() { +// nodes.push((node, vv)); +// } + +// let mut map = FxHashMap::default(); +// for (node, vv) in nodes.iter() { +// map.insert(node.id, vv.clone()); +// } +// let vec: Vec<_> = nodes.iter().enumerate().collect(); +// { +// // println!("{}", a.mermaid()); +// } +// for &(i, (node, vv)) in vec.iter() { +// if i > 3 { +// break; +// } + +// for &(j, (_other_node, other_vv)) in vec.iter() { +// if i >= j { +// continue; +// } + +// let diff_spans = other_vv.diff(vv).left; +// { +// // println!("TARGET IS TO GO FROM {} TO {}", node.id, other_node.id); +// // dbg!(&other_vv, &vv, &diff_spans); +// } +// let mut target_vv = vv.clone(); +// target_vv.forward(&diff_spans); + +// for IterReturn { data, slice } in a.iter_causal(&[node.id], diff_spans.clone()) { +// let sliced = data.slice(slice.start as usize, slice.end as usize); +// { +// // println!("-----------------------------------"); +// // dbg!(&sliced, &forward, &retreat, slice); +// } +// assert!(diff_spans +// .get(&data.id.peer) +// .unwrap() +// .contains(sliced.id.counter)); +// let mut data_vv = map.get(&data.id).unwrap().clone(); +// data_vv.extend_to_include(IdSpan::new( +// sliced.id.peer, +// sliced.id.counter, +// sliced.id.counter + 1, +// )); +// data_vv.shrink_to_exclude(IdSpan::new( +// sliced.id.peer, +// sliced.id.counter, +// sliced.ctr_end(), +// )); +// } +// } +// } + +// Ok(()) +// } + +// #[test] +// #[ignore] +// fn issue() { +// if let Err(err) = test_partial_iter( +// 5, +// vec![ +// Interaction { +// dag_idx: 0, +// merge_with: None, +// len: 2, +// }, +// Interaction { +// dag_idx: 1, +// merge_with: Some(0), +// len: 1, +// }, +// Interaction { +// dag_idx: 0, +// merge_with: None, +// len: 1, +// }, +// Interaction { +// dag_idx: 0, +// merge_with: Some(1), +// len: 1, +// }, +// ], +// ) { +// panic!("{}", err); +// } +// } + +// proptest! { +// #[test] +// #[ignore] +// fn proptest_iter_2( +// interactions in prop::collection::vec(gen_interaction(2), 0..40 * PROPTEST_FACTOR_10), +// ) { +// test_partial_iter(2, interactions)?; +// } + +// fn proptest_iter_3( +// interactions in prop::collection::vec(gen_interaction(3), 0..40 * PROPTEST_FACTOR_10), +// ) { +// test_partial_iter(3, interactions)?; +// } + +// fn proptest_iter_5( +// interactions in prop::collection::vec(gen_interaction(5), 0..40 * PROPTEST_FACTOR_10), +// ) { +// test_partial_iter(5, interactions)?; +// } +// } +// } diff --git a/3rdparty/loro/crates/loro-internal/src/delta.rs b/3rdparty/loro/crates/loro-internal/src/delta.rs new file mode 100644 index 0000000000..efc72a0e00 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/delta.rs @@ -0,0 +1,14 @@ +mod movable_list; +pub(crate) use movable_list::{ElementDelta, MovableListInnerDelta}; +mod seq; +pub use seq::{Delta, DeltaItem, DeltaType, DeltaValue, Meta}; +mod map; +pub use map::{MapDiff, ValuePair}; +mod map_delta; +pub use map_delta::{MapDelta, MapValue, ResolvedMapDelta, ResolvedMapValue}; +mod text; +pub use text::{StyleMeta, StyleMetaItem}; +mod tree; +pub use tree::{ + TreeDelta, TreeDeltaItem, TreeDiff, TreeDiffItem, TreeExternalDiff, TreeInternalDiff, +}; diff --git a/3rdparty/loro/crates/loro-internal/src/delta/map.rs b/3rdparty/loro/crates/loro-internal/src/delta/map.rs new file mode 100644 index 0000000000..9b650f7218 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/delta/map.rs @@ -0,0 +1,203 @@ +use rustc_hash::{FxHashMap, FxHashSet}; +use serde::Serialize; +use std::fmt::Debug; + +use crate::{InternalString, LoroValue}; + +#[derive(Clone, Debug, Serialize)] +pub struct ValuePair { + pub old: T, + pub new: T, +} + +impl From<(LoroValue, LoroValue)> for ValuePair { + fn from((old, new): (LoroValue, LoroValue)) -> Self { + ValuePair { old, new } + } +} + +#[derive(Clone, Debug, Serialize)] +pub struct MapDiff { + pub added: FxHashMap, + pub updated: FxHashMap>, + pub deleted: FxHashMap, +} + +impl MapDiff { + pub fn new() -> Self { + Self::default() + } + + pub fn insert(mut self, key: InternalString, value: T) -> Self { + self.deleted.remove(&key); + self.added.insert(key, value); + self + } + + pub fn delete(mut self, key: &InternalString, value: T) -> Self { + self.added.remove(key); + self.deleted.insert(key.clone(), value); + self + } + + pub fn compose(mut self, other: Self) -> Self { + for (k, v) in other.added.into_iter() { + if let Some(dv) = self.deleted.remove(&k) { + self.updated.insert(k, ValuePair { old: dv, new: v }); + } else if let Some(ValuePair { old: _, new }) = self.updated.get_mut(&k) { + // should not happen in transaction + *new = v; + } else { + self.added.insert(k, v); + } + } + + for (k, v) in other.deleted.into_iter() { + if let Some(av) = self.added.remove(&k) { + self.deleted.insert(k, av); + } else if let Some(ValuePair { old, .. }) = self.updated.remove(&k) { + self.deleted.insert(k, old); + } else { + // delete old value + self.deleted.insert(k, v); + } + } + + for (k, ValuePair { old, new }) in other.updated.into_iter() { + if let Some(av) = self.added.get_mut(&k) { + *av = new; + } else if let Some(dv) = self.deleted.remove(&k) { + self.updated.insert(k, ValuePair { old: dv, new }); + } else if let Some(ValuePair { old: _, new: n }) = self.updated.get_mut(&k) { + *n = new + } else { + self.updated.insert(k, ValuePair { old, new }); + } + } + self + } +} + +#[derive(Clone, Debug, Serialize)] +#[allow(dead_code)] +pub struct MapDiffRaw { + pub(crate) added: FxHashMap, + pub(crate) deleted: FxHashSet, +} + +impl Default for MapDiff { + fn default() -> Self { + Self { + added: Default::default(), + updated: Default::default(), + deleted: Default::default(), + } + } +} + +#[cfg(test)] +mod tests { + use super::{MapDiff, ValuePair}; + use crate::InternalString; + + fn key(value: &str) -> InternalString { + value.into() + } + + fn map_diff( + added: &[(&str, i32)], + updated: &[(&str, (i32, i32))], + deleted: &[(&str, i32)], + ) -> MapDiff { + let mut ans = MapDiff::new(); + for (k, v) in added { + ans.added.insert(key(k), *v); + } + for (k, (old, new)) in updated { + ans.updated.insert( + key(k), + ValuePair { + old: *old, + new: *new, + }, + ); + } + for (k, v) in deleted { + ans.deleted.insert(key(k), *v); + } + ans + } + + fn assert_map_diff_eq(actual: MapDiff, expected: MapDiff) { + assert_eq!(actual.added, expected.added); + assert_eq!(actual.deleted, expected.deleted); + + assert_eq!(actual.updated.len(), expected.updated.len()); + for (key, expected_pair) in expected.updated { + let actual_pair = actual.updated.get(&key).unwrap_or_else(|| { + panic!("missing updated entry for key {:?}", key); + }); + assert_eq!(actual_pair.old, expected_pair.old); + assert_eq!(actual_pair.new, expected_pair.new); + } + } + + #[test] + fn insert_replaces_previous_delete_marker() { + let diff = map_diff(&[], &[], &[("key", 1)]).insert(key("key"), 2); + assert_map_diff_eq(diff, map_diff(&[("key", 2)], &[], &[])); + } + + #[test] + fn delete_removes_pending_add_and_records_deleted_value() { + let diff = map_diff(&[("key", 1)], &[], &[]).delete(&key("key"), 1); + assert_map_diff_eq(diff, map_diff(&[], &[], &[("key", 1)])); + } + + #[test] + fn compose_add_then_update_keeps_latest_value_in_added() { + let lhs = map_diff(&[("key", 1)], &[], &[]); + let rhs = map_diff(&[], &[("key", (1, 2))], &[]); + + assert_map_diff_eq(lhs.compose(rhs), map_diff(&[("key", 2)], &[], &[])); + } + + #[test] + fn compose_delete_then_add_becomes_update_with_original_old_value() { + let lhs = map_diff(&[], &[], &[("key", 1)]); + let rhs = map_diff(&[("key", 2)], &[], &[]); + + assert_map_diff_eq(lhs.compose(rhs), map_diff(&[], &[("key", (1, 2))], &[])); + } + + #[test] + fn compose_update_then_delete_keeps_initial_old_value() { + let lhs = map_diff(&[], &[("key", (1, 2))], &[]); + let rhs = map_diff(&[], &[], &[("key", 2)]); + + assert_map_diff_eq(lhs.compose(rhs), map_diff(&[], &[], &[("key", 1)])); + } + + #[test] + fn compose_multiple_updates_preserves_first_old_and_latest_new() { + let lhs = map_diff(&[], &[("key", (1, 2))], &[]); + let rhs = map_diff(&[], &[("key", (2, 3))], &[]); + let third = map_diff(&[], &[("key", (3, 4))], &[]); + + assert_map_diff_eq( + lhs.compose(rhs).compose(third), + map_diff(&[], &[("key", (1, 4))], &[]), + ); + } + + #[test] + fn compose_is_key_local_and_preserves_other_entries() { + let lhs = map_diff(&[("added", 10)], &[("updated", (1, 2))], &[("deleted", 3)]); + let rhs = map_diff(&[("added", 11)], &[("updated", (2, 3))], &[("deleted", 4)]); + + assert_map_diff_eq( + lhs.compose(rhs), + map_diff(&[("added", 11)], &[("updated", (1, 3))], &[("deleted", 4)]), + ); + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/delta/map_delta.rs b/3rdparty/loro/crates/loro-internal/src/delta/map_delta.rs new file mode 100644 index 0000000000..966ebeb384 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/delta/map_delta.rs @@ -0,0 +1,315 @@ +use std::hash::Hash; + +use loro_common::IdLp; +use rustc_hash::FxHashMap; +use serde::{ser::SerializeStruct, Serialize}; + +use crate::{ + change::Lamport, handler::ValueOrHandler, id::PeerID, span::HasLamport, InternalString, + LoroValue, +}; + +#[derive(Default, Debug, Clone, Serialize)] +pub struct MapDelta { + /// If the value is none, it's a uncreate op that should remove the entry from the + /// map container. + pub updated: FxHashMap>, +} + +#[derive(Debug, Clone)] +pub struct MapValue { + pub value: Option, + pub lamp: Lamport, + pub peer: PeerID, +} + +impl Ord for MapValue { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.lamp + .cmp(&other.lamp) + .then_with(|| self.peer.cmp(&other.peer)) + } +} + +impl PartialOrd for MapValue { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl PartialEq for MapValue { + fn eq(&self, other: &Self) -> bool { + self.lamp == other.lamp && self.peer == other.peer + } +} + +impl Eq for MapValue {} + +impl MapValue { + pub fn idlp(&self) -> IdLp { + IdLp::new(self.peer, self.lamp) + } +} + +#[derive(Default, Debug, Clone)] +pub struct ResolvedMapDelta { + pub updated: FxHashMap, +} + +#[derive(Debug, Clone)] +pub struct ResolvedMapValue { + pub value: Option, + pub idlp: IdLp, +} + +impl ResolvedMapValue { + /// This is used to indicate that the entry is unset. (caused by checkout to before the entry is created) + pub fn new_unset() -> Self { + ResolvedMapValue { + idlp: IdLp::new(PeerID::default(), Lamport::MAX), + value: None, + } + } +} + +impl MapDelta { + pub(crate) fn compose(mut self, x: MapDelta) -> MapDelta { + for (k, v) in x.updated.into_iter() { + if let Some(old) = self.updated.get_mut(&k) { + if &v > old { + *old = v; + } + } else { + self.updated.insert(k, v); + } + } + self + } + + #[inline] + pub fn new() -> Self { + MapDelta { + updated: FxHashMap::default(), + } + } + + #[inline] + pub fn with_entry(mut self, key: InternalString, map_value: MapValue) -> Self { + self.updated.insert(key, Some(map_value)); + self + } +} + +impl ResolvedMapDelta { + pub(crate) fn compose(mut self, x: ResolvedMapDelta) -> ResolvedMapDelta { + // Compose into `self` in place; cloning `self.updated` here made + // composing N fragments into one map event O(N^2). + for (k, v) in x.updated.into_iter() { + if let Some(old) = self.updated.get_mut(&k) { + if v.idlp > old.idlp { + *old = v; + } + } else { + self.updated.insert(k, v); + } + } + self + } + + #[inline] + pub fn new() -> Self { + ResolvedMapDelta { + updated: FxHashMap::default(), + } + } + + #[inline] + pub fn with_entry(mut self, key: InternalString, map_value: ResolvedMapValue) -> Self { + self.updated.insert(key, map_value); + self + } + + pub(crate) fn transform(&mut self, b: &ResolvedMapDelta, left_prior: bool) { + for (k, _) in b.updated.iter() { + if !left_prior { + self.updated.remove(k); + } + } + } +} + +impl Hash for MapValue { + fn hash(&self, state: &mut H) { + // value is not being hashed + self.peer.hash(state); + self.lamp.hash(state); + } +} + +impl HasLamport for MapValue { + fn lamport(&self) -> Lamport { + self.lamp + } +} + +impl Serialize for MapValue { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + let mut s = serializer.serialize_struct("MapValue", 2)?; + s.serialize_field("value", &self.value)?; + s.serialize_field("lamport", &self.lamp)?; + s.serialize_field("id", &self.idlp())?; + s.end() + } +} + +#[cfg(test)] +mod tests { + use std::hash::{Hash, Hasher}; + + use rustc_hash::FxHasher; + + use crate::{handler::ValueOrHandler, LoroValue}; + + use super::*; + + fn key(value: &str) -> InternalString { + value.into() + } + + fn map_value(lamp: Lamport, peer: PeerID, value: i64) -> MapValue { + MapValue { + value: Some(LoroValue::I64(value)), + lamp, + peer, + } + } + + fn resolved_value(lamp: Lamport, peer: PeerID, value: i64) -> ResolvedMapValue { + ResolvedMapValue { + value: Some(ValueOrHandler::Value(LoroValue::I64(value))), + idlp: IdLp::new(peer, lamp), + } + } + + fn hash_map_value(value: &MapValue) -> u64 { + let mut hasher = FxHasher::default(); + value.hash(&mut hasher); + hasher.finish() + } + + #[test] + fn map_value_ordering_identity_and_hash_are_timestamp_based() { + let lower_lamport = map_value(1, 9, 10); + let higher_lamport = map_value(2, 1, 20); + let lower_peer = map_value(2, 1, 30); + let higher_peer = map_value(2, 2, 40); + + assert!(higher_lamport > lower_lamport); + assert!(higher_peer > lower_peer); + assert_eq!(higher_peer.idlp(), IdLp::new(2, 2)); + assert_eq!(higher_peer.lamport(), 2); + + let same_id_different_value = MapValue { + value: Some(LoroValue::I64(999)), + ..higher_peer.clone() + }; + assert_eq!(higher_peer, same_id_different_value); + assert_eq!( + hash_map_value(&higher_peer), + hash_map_value(&same_id_different_value) + ); + } + + #[test] + fn map_delta_compose_keeps_latest_timestamp_per_key() { + let initial = MapDelta::new() + .with_entry(key("same"), map_value(2, 1, 10)) + .with_entry(key("left-only"), map_value(1, 1, 11)); + + let mut incoming = MapDelta::new() + .with_entry(key("same"), map_value(2, 2, 20)) + .with_entry(key("right-only"), map_value(1, 1, 21)); + incoming.updated.insert(key("unset"), None); + + let composed = initial.compose(incoming); + assert_eq!( + composed + .updated + .get(&key("same")) + .and_then(|value| value.as_ref()) + .and_then(|value| value.value.as_ref()), + Some(&LoroValue::I64(20)) + ); + assert!(composed.updated.contains_key(&key("left-only"))); + assert!(composed.updated.contains_key(&key("right-only"))); + assert_eq!(composed.updated.get(&key("unset")), Some(&None)); + } + + #[test] + fn map_delta_compose_does_not_let_older_entries_overwrite_newer_ones() { + let initial = MapDelta::new().with_entry(key("same"), map_value(5, 1, 50)); + let incoming = MapDelta::new().with_entry(key("same"), map_value(4, 9, 40)); + + let composed = initial.compose(incoming); + let value = composed + .updated + .get(&key("same")) + .and_then(|value| value.as_ref()) + .and_then(|value| value.value.as_ref()); + assert_eq!(value, Some(&LoroValue::I64(50))); + } + + #[test] + fn resolved_map_delta_compose_and_transform_apply_the_same_conflict_contract() { + let left = ResolvedMapDelta::new() + .with_entry(key("same"), resolved_value(1, 1, 10)) + .with_entry(key("left-only"), resolved_value(1, 1, 11)); + let right = ResolvedMapDelta::new() + .with_entry(key("same"), resolved_value(1, 2, 20)) + .with_entry(key("right-only"), resolved_value(1, 1, 21)); + + let composed = left.compose(right.clone()); + let value = composed.updated.get(&key("same")).unwrap(); + assert_eq!(value.idlp, IdLp::new(2, 1)); + assert_eq!(value.value.as_ref().unwrap().to_value(), LoroValue::I64(20)); + assert!(composed.updated.contains_key(&key("left-only"))); + assert!(composed.updated.contains_key(&key("right-only"))); + + let mut left_prior = composed.clone(); + left_prior.transform(&right, true); + assert!(left_prior.updated.contains_key(&key("same"))); + + let mut right_prior = composed; + right_prior.transform(&right, false); + assert!(!right_prior.updated.contains_key(&key("same"))); + assert!(!right_prior.updated.contains_key(&key("right-only"))); + assert!(right_prior.updated.contains_key(&key("left-only"))); + } + + #[test] + fn resolved_unset_uses_max_lamport_to_win_against_material_values() { + let unset = ResolvedMapValue::new_unset(); + let value = resolved_value(Lamport::MAX - 1, PeerID::MAX, 1); + + let composed = ResolvedMapDelta::new() + .with_entry(key("field"), value) + .compose(ResolvedMapDelta::new().with_entry(key("field"), unset)); + let entry = composed.updated.get(&key("field")).unwrap(); + assert_eq!(entry.idlp, IdLp::new(PeerID::default(), Lamport::MAX)); + assert!(entry.value.is_none()); + } + + #[test] + fn map_value_serialization_exposes_value_lamport_and_id() { + let serialized = serde_json::to_value(map_value(7, 8, 42)).unwrap(); + assert_eq!(serialized["value"], serde_json::json!(42)); + assert_eq!(serialized["lamport"], serde_json::json!(7)); + assert_eq!( + serialized["id"], + serde_json::json!({ "peer": 8, "lamport": 7 }) + ); + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/delta/movable_list.rs b/3rdparty/loro/crates/loro-internal/src/delta/movable_list.rs new file mode 100644 index 0000000000..6c2a1e42eb --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/delta/movable_list.rs @@ -0,0 +1,56 @@ +use loro_common::{CompactIdLp, IdFull, IdLp, LoroValue}; +use rustc_hash::FxHashMap; +use smallvec::SmallVec; + +use super::{Delta, DeltaValue}; + +#[derive(Clone, Debug)] +pub(crate) struct MovableListInnerDelta { + pub list: Delta, ()>, + pub elements: FxHashMap, +} + +impl DeltaValue for SmallVec<[IdFull; 1]> { + fn value_extend(&mut self, other: Self) -> Result<(), Self> { + for v in other { + self.push(v) + } + + Ok(()) + } + + fn take(&mut self, length: usize) -> Self { + self.drain(..length).collect() + } + + fn length(&self) -> usize { + self.len() + } +} + +impl MovableListInnerDelta { + pub(crate) fn is_empty(&self) -> bool { + self.list.is_empty() && self.elements.is_empty() + } +} + +#[derive(Clone, Debug)] +pub struct ElementDelta { + /// This must be Some if it's in checkout mode (not fast-forward mode) + pub pos: Option, + pub value: LoroValue, + pub value_updated: bool, + /// This must be Some if it's in checkout mode (not fast-forward mode) + pub value_id: Option, +} + +impl ElementDelta { + pub fn placeholder() -> Self { + Self { + pos: None, + value: LoroValue::Null, + value_updated: false, + value_id: None, + } + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/delta/seq.rs b/3rdparty/loro/crates/loro-internal/src/delta/seq.rs new file mode 100644 index 0000000000..eed69a55b3 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/delta/seq.rs @@ -0,0 +1,950 @@ +use enum_as_inner::EnumAsInner; +use serde::Serialize; +use std::fmt::Debug; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Delta { + pub(crate) vec: Vec>, +} + +impl Serialize for Delta { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.collect_seq(self.vec.iter()) + } +} + +#[derive(Debug, EnumAsInner, Clone, PartialEq, Eq, Serialize)] +pub enum DeltaItem { + Retain { retain: usize, attributes: Meta }, + Insert { insert: Value, attributes: Meta }, + Delete { delete: usize, attributes: Meta }, +} + +#[derive(PartialEq, Debug, Eq)] +pub enum DeltaType { + Retain, + Insert, + Delete, +} + +/// The metadata of a DeltaItem +/// If empty metadata is used to override the older one, we will remove the metadata into None +pub trait Meta: Debug + Clone + PartialEq + Default { + fn empty() -> Self { + Self::default() + } + fn is_empty(&self) -> bool; + + /// this is used when composing two [DeltaItem]s with the same length + fn compose(&mut self, other: &Self, type_pair: (DeltaType, DeltaType)); + #[allow(unused)] + fn take(&mut self, other: &Self) -> Self { + self.clone() + } + + fn is_mergeable(&self, other: &Self) -> bool; + /// This is used when we merge two [DeltaItem]s. + /// And it's guaranteed that [Meta::is_mergeable] is true + fn merge(&mut self, other: &Self); +} + +impl Meta for () { + fn empty() -> Self {} + fn is_empty(&self) -> bool { + true + } + + fn compose(&mut self, _other: &Self, _type_pair: (DeltaType, DeltaType)) {} + + fn is_mergeable(&self, _other: &Self) -> bool { + true + } + + fn merge(&mut self, _other: &Self) {} +} + +/// The value of [DeltaItem::Insert] +pub trait DeltaValue: Debug + Sized { + /// the other will be merged into self + fn value_extend(&mut self, other: Self) -> Result<(), Self>; + /// takes the first number of `length` elements + fn take(&mut self, length: usize) -> Self; + /// the length of the value + fn length(&self) -> usize; +} + +impl DeltaValue for DeltaItem { + fn value_extend(&mut self, _other: Self) -> Result<(), Self> { + unreachable!() + } + + fn take(&mut self, _length: usize) -> Self { + unreachable!() + } + + fn length(&self) -> usize { + match self { + DeltaItem::Retain { + retain: len, + attributes: _, + } => *len, + DeltaItem::Insert { + insert: value, + attributes: _, + } => value.length(), + DeltaItem::Delete { + delete: len, + attributes: _, + } => *len, + } + } +} + +impl DeltaItem { + pub fn meta(&self) -> &M { + match self { + DeltaItem::Insert { + attributes: meta, .. + } => meta, + DeltaItem::Retain { + attributes: meta, .. + } => meta, + DeltaItem::Delete { + delete: _, + attributes: meta, + } => meta, + } + } + + pub fn meta_mut(&mut self) -> &mut M { + match self { + DeltaItem::Insert { + attributes: meta, .. + } => meta, + DeltaItem::Retain { + attributes: meta, .. + } => meta, + DeltaItem::Delete { + delete: _, + attributes: meta, + } => meta, + } + } + + pub fn set_meta(&mut self, meta: M) { + match self { + DeltaItem::Insert { attributes: m, .. } => *m = meta, + DeltaItem::Retain { attributes: m, .. } => *m = meta, + DeltaItem::Delete { + delete: _, + attributes: m, + } => *m = meta, + } + } + + fn type_(&self) -> DeltaType { + match self { + DeltaItem::Insert { .. } => DeltaType::Insert, + DeltaItem::Retain { .. } => DeltaType::Retain, + DeltaItem::Delete { .. } => DeltaType::Delete, + } + } + + pub fn compose_meta(&mut self, other: &Self) { + let type_pair = (self.type_(), other.type_()); + let meta = self.meta_mut(); + let other_meta = other.meta(); + Meta::compose(meta, other_meta, type_pair); + } + + // change self-length to self.len()-length + // and return the taken one. + pub(crate) fn take(&mut self, length: usize) -> Self { + match self { + DeltaItem::Insert { + insert: value, + attributes: meta, + } => { + let v = value.take(length); + Self::Insert { + insert: v, + attributes: meta.clone(), + } + } + DeltaItem::Retain { + retain: len, + attributes: meta, + } => { + *len -= length; + Self::Retain { + retain: length, + attributes: meta.clone(), + } + } + DeltaItem::Delete { + delete: len, + attributes: _, + } => { + *len -= length; + Self::Delete { + delete: length, + // meta may store utf16 length, this take will invalidate it + attributes: M::empty(), + } + } + } + } + + pub(crate) fn take_with_meta_ref(&mut self, length: usize, other_meta: &Self) -> Self { + match self { + DeltaItem::Insert { + insert: value, + attributes: meta, + } => { + let v = value.take(length); + Self::Insert { + insert: v, + attributes: meta.take(other_meta.meta()), + } + } + DeltaItem::Retain { + retain: len, + attributes: meta, + } => { + *len -= length; + Self::Retain { + retain: length, + attributes: meta.take(other_meta.meta()), + } + } + DeltaItem::Delete { + delete: len, + attributes: meta, + } => { + *len -= length; + Self::Delete { + delete: length, + attributes: meta.take(other_meta.meta()), + } + } + } + } + + fn insert_inner(self) -> Value { + match self { + DeltaItem::Insert { insert: value, .. } => value, + _ => unreachable!(), + } + } +} + +#[derive(Debug)] +pub struct DeltaIterator { + // The reversed Vec uses pop() to simulate getting the first element each time + ops: Vec>, +} + +impl DeltaIterator { + fn new(ops: Vec>) -> Self { + Self { ops } + } + + #[inline(always)] + fn next>>(&mut self, len: L) -> DeltaItem { + self.next_impl(len.into()) + } + + fn next_impl(&mut self, len: Option) -> DeltaItem { + let length = len.unwrap_or(usize::MAX); + let next_op = self.peek_mut(); + if next_op.is_none() { + return DeltaItem::Retain { + retain: usize::MAX, + attributes: M::empty(), + }; + } + let op = next_op.unwrap(); + let op_length = op.length(); + if length < op_length { + // a part of the peek op + op.take(length) + } else { + self.take_peek().unwrap() + } + } + + fn next_with_ref(&mut self, len: usize, other: &DeltaItem) -> DeltaItem { + let next_op = self.peek_mut(); + if next_op.is_none() { + return DeltaItem::Retain { + retain: other.length(), + attributes: M::empty(), + }; + } + let op = next_op.unwrap(); + let op_length = op.length(); + if len < op_length { + // a part of the peek op + op.take_with_meta_ref(len, other) + } else { + self.take_peek().unwrap() + } + } + + fn next_pair(&mut self, other: &mut Self) -> (DeltaItem, DeltaItem) { + let self_len = self.peek_length(); + let other_len = other.peek_length(); + if self_len > other_len { + let length = other_len; + let other_op = other.next(None); + debug_assert_eq!(other_op.length(), length); + let this_op = self.next_with_ref(length, &other_op); + (this_op, other_op) + } else { + let length = self_len; + let this_op = self.next(None); + debug_assert_eq!(this_op.length(), length); + let other_op = other.next_with_ref(length, &this_op); + (this_op, other_op) + } + } + + fn peek_mut(&mut self) -> Option<&mut DeltaItem> { + self.ops.last_mut() + } + + fn take_peek(&mut self) -> Option> { + self.ops.pop() + } + + fn rest(mut self) -> Vec> { + self.ops.reverse(); + self.ops + } + + fn has_next(&self) -> bool { + !self.ops.is_empty() + } + + fn peek(&self) -> Option<&DeltaItem> { + self.ops.last() + } + + fn peek_length(&self) -> usize { + if let Some(op) = self.peek() { + op.length() + } else { + usize::MAX + } + } + + fn peek_is_insert(&self) -> bool { + if let Some(op) = self.peek() { + op.is_insert() + } else { + false + } + } + + fn peek_is_delete(&self) -> bool { + if let Some(op) = self.peek() { + op.is_delete() + } else { + false + } + } +} + +impl Delta { + pub fn new() -> Self { + Self { vec: Vec::new() } + } + + pub fn items(&self) -> &[DeltaItem] { + &self.vec + } + + pub fn inner(self) -> Vec> { + self.vec + } + + pub fn retain_with_meta(mut self, len: usize, meta: M) -> Self { + if len == 0 { + // currently no meta for retain if len == 0 + return self; + } + + self.push(DeltaItem::Retain { + retain: len, + attributes: meta, + }); + self + } + + pub fn insert_with_meta>(mut self, value: V, meta: M) -> Self { + self.push(DeltaItem::Insert { + insert: value.into(), + attributes: meta, + }); + self + } + + pub fn delete_with_meta(mut self, len: usize, meta: M) -> Self { + self.push(DeltaItem::Delete { + delete: len, + attributes: meta, + }); + self + } + + pub fn delete(mut self, len: usize) -> Self { + if len == 0 { + return self; + } + self.push(DeltaItem::Delete { + delete: len, + attributes: M::empty(), + }); + self + } + + pub fn retain(mut self, len: usize) -> Self { + if len == 0 { + return self; + } + + self.push(DeltaItem::Retain { + retain: len, + attributes: M::empty(), + }); + self + } + + pub fn insert>(mut self, value: V) -> Self { + self.push(DeltaItem::Insert { + insert: value.into(), + attributes: M::empty(), + }); + self + } + + // If the new_op is merged, return true + pub fn push(&mut self, new_op: DeltaItem) -> bool { + let mut index = self.vec.len(); + let last_op = self.vec.pop(); + if let Some(mut last_op) = last_op { + if new_op.is_delete() && last_op.is_delete() { + *last_op.as_delete_mut().unwrap().0 += new_op.length(); + self.vec.push(last_op); + return true; + } + // Since it does not matter if we insert before or after deleting at the same index, + // always prefer to insert first + if last_op.is_delete() && new_op.is_insert() { + index -= 1; + let _last_op = self.vec.pop(); + self.vec.push(last_op); + if let Some(_last_op_inner) = _last_op { + last_op = _last_op_inner; + } else { + self.vec.insert(0, new_op); + return true; + }; + } + if last_op.meta().is_mergeable(new_op.meta()) { + if new_op.is_insert() && last_op.is_insert() { + let value = last_op.as_insert_mut().unwrap().0; + let meta = new_op.meta().clone(); + match value.value_extend(new_op.insert_inner()) { + Ok(_) => { + last_op.meta_mut().merge(&meta); + self.vec.insert(index - 1, last_op); + return true; + } + Err(inner) => { + self.vec.insert(index - 1, last_op); + self.vec.insert( + index, + DeltaItem::Insert { + insert: inner, + attributes: meta, + }, + ); + return false; + } + } + } else if new_op.is_retain() && last_op.is_retain() { + *last_op.as_retain_mut().unwrap().0 += new_op.length(); + last_op.meta_mut().merge(new_op.meta()); + // self.vec.push(last_op); + self.vec.insert(index - 1, last_op); + return true; + } + } + + // self.vec.push(last_op); + self.vec.insert(index - 1, last_op); + } + if index == self.vec.len() { + self.vec.push(new_op); + false + } else { + self.vec.insert(index, new_op); + true + } + } + + pub fn iter(&self) -> impl Iterator> { + self.vec.iter() + } + + pub fn iter_mut(&mut self) -> impl Iterator> { + self.vec.iter_mut() + } + + pub fn into_op_iter(self) -> DeltaIterator { + DeltaIterator::new(self.vec.into_iter().rev().collect()) + } + + pub fn len(&self) -> usize { + self.vec.len() + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Reference: [Quill Delta](https://github.com/quilljs/delta) + // TODO: PERF use &mut self and &other + pub fn compose(self, other: Delta) -> Delta { + let mut this_iter = self.into_op_iter(); + let mut other_iter = other.into_op_iter(); + let mut ops = vec![]; + let first_other = other_iter.peek(); + if let Some(first_other) = first_other { + // if other.delta starts with retain, we insert corresponding number of inserts from self.delta + if first_other.is_retain() && first_other.meta().is_empty() { + let mut first_left = first_other.length(); + let mut first_this = this_iter.peek(); + while let Some(first_this_inner) = first_this { + if first_this_inner.is_insert() && first_this_inner.length() <= first_left { + first_left -= first_this_inner.length(); + let mut op = this_iter.next(None); + op.compose_meta(first_other); + ops.push(op); + first_this = this_iter.peek(); + } else { + break; + } + } + if first_other.length() - first_left > 0 { + other_iter.next(first_other.length() - first_left); + } + } + } + let mut delta = Delta { vec: ops }; + while this_iter.has_next() || other_iter.has_next() { + if other_iter.peek_is_insert() { + // nothing to compose here + delta.push(other_iter.next(None)); + } else if this_iter.peek_is_delete() { + // nothing to compose here + delta.push(this_iter.next(None)); + } else { + // possible cases: + // 1. this: insert, other: retain + // 2. this: retain, other: retain + // 3. this: retain, other: delete + // 4. this: insert, other: delete + + let (mut this_op, mut other_op) = this_iter.next_pair(&mut other_iter); + if other_op.is_retain() { + // 1. this: insert, other: retain + // 2. this: retain, other: retain + this_op.compose_meta(&other_op); + let merged = delta.push(this_op); + let concat_rest = !other_iter.has_next() && !merged; + if concat_rest { + let vec = this_iter.rest(); + if vec.is_empty() { + break; + } + + let rest = Delta { vec }; + delta = delta.concat(rest); + break; + } + } else if other_op.is_delete() && this_op.is_retain() { + // 3. this: retain, other: delete + other_op.compose_meta(&this_op); + // other deletes the retained text + delta.push(other_op); + } else { + // 4. this: insert, other: delete + // nothing to do here, because insert and delete have the same length + } + } + } + + delta.chop() + } + + pub(crate) fn concat(mut self, mut other: Self) -> Self { + if !other.vec.is_empty() { + let other_first = other.vec.remove(0); + self.push(other_first); + self.vec.extend(other.vec); + } + self + } + + pub fn chop(mut self) -> Self { + loop { + match self.vec.last() { + Some(last_op) if last_op.is_retain() && last_op.meta().is_empty() => { + self.vec.pop(); + } + _ => break, + } + } + + self + } + + pub fn insert_len(&self) -> usize { + self.vec + .iter() + .map(|x| match x { + DeltaItem::Insert { insert, .. } => insert.length(), + _ => 0, + }) + .sum() + } +} + +impl IntoIterator for Delta { + type Item = DeltaItem; + + type IntoIter = std::vec::IntoIter>; + fn into_iter(self) -> Self::IntoIter { + self.vec.into_iter() + } +} + +impl Default for Delta { + fn default() -> Self { + Self::new() + } +} + +impl DeltaValue for Vec { + fn value_extend(&mut self, other: Self) -> Result<(), Self> { + self.extend(other); + Ok(()) + } + + fn take(&mut self, length: usize) -> Self { + let mut new = self.split_off(length); + std::mem::swap(self, &mut new); + new + } + + fn length(&self) -> usize { + self.len() + } +} + +impl DeltaValue for String { + fn value_extend(&mut self, other: Self) -> Result<(), Self> { + self.push_str(&other); + Ok(()) + } + + fn take(&mut self, length: usize) -> Self { + let mut new = self.split_off(length); + std::mem::swap(self, &mut new); + new + } + fn length(&self) -> usize { + self.len() + } +} + +#[cfg(test)] +mod test { + use super::{Delta, DeltaItem, DeltaType, Meta}; + + #[derive(Debug, PartialEq, Clone, Default)] + struct TestMeta { + bold: Option>, + color: Option>, + } + + impl Meta for TestMeta { + fn empty() -> Self { + TestMeta { + bold: None, + color: None, + } + } + fn is_empty(&self) -> bool { + self.bold.is_none() && self.color.is_none() + } + + fn compose( + &mut self, + other: &Self, + (this_type, other_type): (super::DeltaType, super::DeltaType), + ) { + if this_type != DeltaType::Delete && other_type != DeltaType::Delete { + if let Some(other_bold) = other.bold { + self.bold = Some(other_bold); + } + if let Some(other_color) = other.color { + self.color = Some(other_color); + } + } + } + + fn is_mergeable(&self, other: &Self) -> bool { + self == other + } + + fn merge(&mut self, _other: &Self) {} + } + + type TestDelta = Delta; + const BOLD_META: TestMeta = TestMeta { + bold: Some(Some(true)), + color: None, + }; + const COLOR_META: TestMeta = TestMeta { + bold: None, + color: Some(Some(())), + }; + const EMPTY_META: TestMeta = TestMeta { + bold: Some(None), + color: Some(None), + }; + const BOTH_META: TestMeta = TestMeta { + bold: Some(Some(true)), + color: Some(Some(())), + }; + + #[test] + fn delta_push() { + let mut a: Delta = Delta::new().insert("a".to_string()); + a.push(DeltaItem::Insert { + insert: "b".to_string(), + attributes: (), + }); + assert_eq!(a, Delta::new().insert("ab".to_string())); + } + + #[test] + fn delta_compose() { + let a: Delta = Delta::new().retain(3).insert("abcde".to_string()); + let b = Delta::new().retain(5).delete(6); + assert_eq!( + a.compose(b), + Delta::new().retain(3).insert("ab".to_string()).delete(3) + ); + + let a: Delta = Delta::new().insert("123".to_string()); + let b = Delta::new().retain(1).insert("123".to_string()); + assert_eq!(a.compose(b), Delta::new().insert("112323".to_string())); + } + + #[test] + fn delete_failed() { + let a: Delta = Delta::new() + .retain(2) + .insert("[31354]") + .retain(1) + .insert("[31354]") + .retain(12) + .insert("[31354]"); + let b: Delta = Delta::new().retain(27).delete(3); + assert_eq!( + a.compose(b), + Delta::new() + .retain(2) + .insert("[31354]") + .retain(1) + .insert("[31354]") + .retain(10) + .insert("31354]") + .delete(2) + ); + } + + #[test] + fn insert_insert() { + let a = TestDelta::new().insert("a"); + let b = TestDelta::new().insert("b"); + assert_eq!(a.compose(b), TestDelta::new().insert("b").insert("a")); + } + + #[test] + fn insert_retain() { + let a = TestDelta::new().insert("a"); + let b = TestDelta::new().retain_with_meta(1, BOLD_META); + assert_eq!( + a.compose(b), + TestDelta::new().insert_with_meta("a", BOLD_META) + ); + } + + #[test] + fn insert_delete() { + let a = TestDelta::new().insert("a"); + let b = TestDelta::new().delete(1); + assert_eq!(a.compose(b), TestDelta::new()); + } + + #[test] + fn delete_insert() { + let a = TestDelta::new().delete(1); + let b = TestDelta::new().insert("b"); + assert_eq!(a.compose(b), TestDelta::new().insert("b").delete(1)); + } + + #[test] + fn delete_retain() { + let a = TestDelta::new().delete(1); + let b = TestDelta::new().retain_with_meta(1, BOLD_META); + assert_eq!( + a.compose(b), + TestDelta::new().delete(1).retain_with_meta(1, BOLD_META) + ); + } + + #[test] + fn delete_delete() { + let a = TestDelta::new().delete(1); + let b = TestDelta::new().delete(1); + assert_eq!(a.compose(b), TestDelta::new().delete(2)); + } + + #[test] + fn retain_insert() { + let a = TestDelta::new().retain_with_meta(1, BOLD_META); + let b = TestDelta::new().insert("b"); + assert_eq!( + a.compose(b), + TestDelta::new().insert("b").retain_with_meta(1, BOLD_META) + ); + } + + #[test] + fn retain_retain() { + let a = TestDelta::new().retain_with_meta(1, BOLD_META); + let b = TestDelta::new().retain_with_meta(1, COLOR_META); + assert_eq!( + a.compose(b), + TestDelta::new().retain_with_meta(1, BOTH_META) + ); + } + + #[test] + fn retain_delete() { + let a = TestDelta::new().retain_with_meta(1, BOLD_META); + let b = TestDelta::new().delete(1); + assert_eq!(a.compose(b), TestDelta::new().delete(1)); + } + + #[test] + fn delete_entire() { + let a = TestDelta::new().retain(4).insert("abcde"); + let b = TestDelta::new().delete(9); + assert_eq!(a.compose(b), TestDelta::new().delete(4)); + } + + #[test] + fn retain_more() { + let a = TestDelta::new().insert("abcde"); + let b = TestDelta::new().retain(10); + assert_eq!(a.compose(b), TestDelta::new().insert("abcde")); + } + + #[test] + fn remove_meta() { + let a = TestDelta::new().insert_with_meta("a", BOLD_META); + let b = TestDelta::new().retain_with_meta(1, EMPTY_META); + assert_eq!( + a.compose(b), + TestDelta::new().insert_with_meta("a", EMPTY_META) + ); + } + + #[test] + fn retain_start_opt() { + let a = TestDelta::new() + .insert_with_meta("a", BOLD_META) + .insert("b") + .insert_with_meta("c", BOLD_META) + .delete(1); + let b = TestDelta::new().retain(3).insert("d"); + let expect = TestDelta::new() + .insert_with_meta("a", BOLD_META) + .insert("b") + .insert_with_meta("c", BOLD_META) + .insert("d") + .delete(1); + assert_eq!(a.compose(b), expect); + } + + #[test] + fn retain_start_opt_split() { + let a = TestDelta::new() + .insert_with_meta("a", BOLD_META) + .insert("b") + .insert_with_meta("c", BOLD_META) + .retain(5) + .delete(1); + let b = TestDelta::new().retain(4).insert("d"); + let expect = TestDelta::new() + .insert_with_meta("a", BOLD_META) + .insert("b") + .insert_with_meta("c", BOLD_META) + .retain(1) + .insert("d") + .retain(4) + .delete(1); + assert_eq!(a.compose(b), expect); + } + + #[test] + fn retain_end_opt() { + let a = TestDelta::new() + .insert_with_meta("a", BOLD_META) + .insert("b") + .insert_with_meta("c", BOLD_META); + let b = TestDelta::new().delete(1); + let expect = TestDelta::new() + .insert("b") + .insert_with_meta("c", BOLD_META); + assert_eq!(a.compose(b), expect); + } + + #[test] + fn retain_end_opt_join() { + let a = TestDelta::new() + .insert_with_meta("a", BOLD_META) + .insert("b") + .insert_with_meta("c", BOLD_META) + .insert("d") + .insert_with_meta("e", BOLD_META) + .insert("f"); + let b = TestDelta::new().retain(1).delete(1); + let expect = TestDelta::new() + .insert_with_meta("ac", BOLD_META) + .insert("d") + .insert_with_meta("e", BOLD_META) + .insert("f"); + assert_eq!(a.compose(b), expect); + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/delta/text.rs b/3rdparty/loro/crates/loro-internal/src/delta/text.rs new file mode 100644 index 0000000000..f9739017c9 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/delta/text.rs @@ -0,0 +1,298 @@ +use loro_common::{InternalString, LoroValue, PeerID}; +use rustc_hash::FxHashMap; +use serde::{Deserialize, Serialize}; + +use crate::change::Lamport; +use crate::container::richtext::{Style, Styles}; +use crate::event::TextMeta; +use crate::ToJson; + +use super::Meta; + +#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct StyleMeta { + map: FxHashMap, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct StyleMetaItem { + // We need lamport and peer to compose the event + pub lamport: Lamport, + pub peer: PeerID, + pub value: LoroValue, +} + +impl StyleMetaItem { + pub fn try_replace(&mut self, other: &StyleMetaItem) { + if (self.lamport, self.peer) < (other.lamport, other.peer) { + self.lamport = other.lamport; + self.peer = other.peer; + self.value = other.value.clone(); + } + } +} + +impl From<&Styles> for StyleMeta { + fn from(styles: &Styles) -> Self { + let mut map = FxHashMap::with_capacity_and_hasher(styles.len(), Default::default()); + for (key, value) in styles.iter() { + if let Some(value) = value.get() { + map.insert( + key.key().clone(), + StyleMetaItem { + value: value.to_value(), + lamport: value.lamport, + peer: value.peer, + }, + ); + } + } + Self { map } + } +} + +impl From for StyleMeta { + fn from(styles: Styles) -> Self { + let temp = &styles; + temp.into() + } +} + +impl Meta for StyleMeta { + fn is_empty(&self) -> bool { + self.map.is_empty() + } + + fn compose(&mut self, other: &Self, _type_pair: (super::DeltaType, super::DeltaType)) { + for (key, value) in other.map.iter() { + match self.map.get_mut(key) { + Some(old_value) => { + old_value.try_replace(value); + } + None => { + self.map.insert(key.clone(), value.clone()); + } + } + } + } + + fn is_mergeable(&self, other: &Self) -> bool { + self.map == other.map + } + + fn merge(&mut self, _: &Self) {} +} + +impl Meta for TextMeta { + fn is_empty(&self) -> bool { + self.0.is_empty() + } + + fn compose(&mut self, other: &Self, _: (super::DeltaType, super::DeltaType)) { + for (key, value) in other.0.iter() { + self.0.insert(key.clone(), value.clone()); + } + } + + fn is_mergeable(&self, other: &Self) -> bool { + self.0 == other.0 + } + + fn merge(&mut self, _: &Self) {} +} + +impl StyleMeta { + pub(crate) fn iter(&self) -> impl Iterator + '_ { + self.map.iter().map(|(key, style)| { + ( + key.clone(), + Style { + key: key.clone(), + data: style.value.clone(), + }, + ) + }) + } + + pub(crate) fn insert(&mut self, key: InternalString, value: StyleMetaItem) { + self.map.insert(key, value); + } + + pub(crate) fn contains_key(&self, key: &InternalString) -> bool { + self.map.contains_key(key) + } + + pub(crate) fn to_value(&self) -> LoroValue { + LoroValue::Map(self.to_map_without_null_value().into()) + } + + fn to_map_without_null_value(&self) -> FxHashMap { + self.map + .iter() + .filter_map(|(key, value)| { + if value.value.is_null() { + None + } else { + Some((key.to_string(), value.value.clone())) + } + }) + .collect() + } + + pub(crate) fn to_map(&self) -> FxHashMap { + self.map + .iter() + .map(|(key, value)| (key.to_string(), value.value.clone())) + .collect() + } + + pub(crate) fn to_option_map(&self) -> Option> { + if self.is_empty() { + return None; + } + + Some(self.to_map()) + } + + pub(crate) fn to_option_map_without_null_value(&self) -> Option> { + let map = self.to_map_without_null_value(); + if map.is_empty() { + None + } else { + Some(map) + } + } +} + +impl ToJson for TextMeta { + fn to_json_value(&self) -> serde_json::Value { + let mut map = serde_json::Map::new(); + for (key, value) in self.0.iter() { + let value = serde_json::to_value(value).unwrap(); + map.insert(key.to_string(), value); + } + + serde_json::Value::Object(map) + } + + fn from_json(s: &str) -> Self { + let map: FxHashMap = serde_json::from_str(s).unwrap(); + TextMeta(map) + } +} + +#[cfg(test)] +mod tests { + use rustc_hash::FxHashMap; + + use super::*; + use crate::delta::{DeltaType, Meta}; + + fn key(value: &str) -> InternalString { + value.into() + } + + fn style_item(lamport: Lamport, peer: PeerID, value: LoroValue) -> StyleMetaItem { + StyleMetaItem { + lamport, + peer, + value, + } + } + + #[test] + fn style_meta_item_replacement_is_ordered_by_lamport_then_peer() { + let mut item = style_item(1, 9, LoroValue::String("old".into())); + item.try_replace(&style_item(1, 8, LoroValue::String("ignored".into()))); + assert_eq!(item.value, LoroValue::String("old".into())); + + item.try_replace(&style_item(1, 10, LoroValue::String("peer-wins".into()))); + assert_eq!(item.value, LoroValue::String("peer-wins".into())); + + item.try_replace(&style_item(2, 0, LoroValue::String("lamport-wins".into()))); + assert_eq!(item.value, LoroValue::String("lamport-wins".into())); + } + + #[test] + fn style_meta_compose_keeps_latest_value_per_key_and_preserves_new_keys() { + let mut left = StyleMeta::default(); + left.insert(key("bold"), style_item(2, 1, LoroValue::Bool(true))); + left.insert( + key("color"), + style_item(1, 1, LoroValue::String("red".into())), + ); + + let mut right = StyleMeta::default(); + right.insert(key("bold"), style_item(1, 99, LoroValue::Bool(false))); + right.insert( + key("color"), + style_item(3, 0, LoroValue::String("blue".into())), + ); + right.insert( + key("link"), + style_item(1, 1, LoroValue::String("docs".into())), + ); + + left.compose(&right, (DeltaType::Retain, DeltaType::Retain)); + let values = left.to_map(); + assert_eq!(values.get("bold"), Some(&LoroValue::Bool(true))); + assert_eq!(values.get("color"), Some(&LoroValue::String("blue".into()))); + assert_eq!(values.get("link"), Some(&LoroValue::String("docs".into()))); + } + + #[test] + fn style_meta_value_views_distinguish_null_from_absent() { + let mut meta = StyleMeta::default(); + assert!(meta.is_empty()); + assert_eq!(meta.to_option_map(), None); + + meta.insert(key("bold"), style_item(1, 1, LoroValue::Bool(true))); + meta.insert(key("deleted"), style_item(1, 2, LoroValue::Null)); + + assert!(meta.contains_key(&key("deleted"))); + assert_eq!( + meta.to_map().get("deleted"), + Some(&LoroValue::Null), + "to_map keeps explicit null style values" + ); + assert_eq!( + meta.to_map_without_null_value().get("deleted"), + None, + "serialized style values omit null entries" + ); + assert_eq!( + meta.to_value(), + LoroValue::Map( + FxHashMap::from_iter([(String::from("bold"), LoroValue::Bool(true))]).into() + ) + ); + + let styles: FxHashMap<_, _> = meta.iter().map(|style| (style.0, style.1.data)).collect(); + assert_eq!(styles.get(&key("bold")), Some(&LoroValue::Bool(true))); + assert_eq!(styles.get(&key("deleted")), Some(&LoroValue::Null)); + } + + #[test] + fn text_meta_compose_and_json_roundtrip_are_map_like() { + let mut left = TextMeta(FxHashMap::from_iter([( + String::from("lang"), + LoroValue::String("en".into()), + )])); + let right = TextMeta(FxHashMap::from_iter([ + (String::from("lang"), LoroValue::String("fr".into())), + (String::from("author"), LoroValue::String("loro".into())), + ])); + + left.compose(&right, (DeltaType::Retain, DeltaType::Retain)); + assert_eq!(left.0.get("lang"), Some(&LoroValue::String("fr".into()))); + assert_eq!( + left.0.get("author"), + Some(&LoroValue::String("loro".into())) + ); + assert!(left.is_mergeable(&left.clone())); + + let json = left.to_json_value(); + let decoded = TextMeta::from_json(&json.to_string()); + assert_eq!(decoded.0, left.0); + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/delta/tree.rs b/3rdparty/loro/crates/loro-internal/src/delta/tree.rs new file mode 100644 index 0000000000..8f368e590e --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/delta/tree.rs @@ -0,0 +1,209 @@ +use fractional_index::FractionalIndex; +use itertools::Itertools; +use loro_common::{IdFull, TreeID}; +use rustc_hash::{FxHashMap, FxHashSet}; +use std::fmt::Debug; +use std::ops::{Deref, DerefMut}; + +use crate::state::TreeParentId; + +#[derive(Debug, Clone, Default)] +pub struct TreeDiff { + pub diff: Vec, +} + +#[derive(Debug, Clone)] +pub struct TreeDiffItem { + pub target: TreeID, + pub action: TreeExternalDiff, +} + +#[derive(Debug, Clone, PartialEq)] +pub enum TreeExternalDiff { + Create { + parent: TreeParentId, + index: usize, + position: FractionalIndex, + }, + Move { + parent: TreeParentId, + index: usize, + position: FractionalIndex, + old_parent: TreeParentId, + old_index: usize, + }, + Delete { + old_parent: TreeParentId, + old_index: usize, + }, +} + +impl TreeDiff { + pub(crate) fn compose(mut self, other: Self) -> Self { + self.diff.extend(other.diff); + // self = compose_tree_diff(&self); + self + } + + pub(crate) fn extend>(mut self, other: I) -> Self { + self.diff.extend(other); + self + } + + fn to_hash_map_mut(&mut self) -> FxHashMap { + let mut ans = FxHashSet::default(); + for index in (0..self.diff.len()).rev() { + let target = self.diff[index].target; + if ans.contains(&target) { + self.diff.remove(index); + continue; + } + ans.insert(target); + } + self.iter() + .map(|x| x.target) + .enumerate() + .map(|(i, x)| (x, i)) + .collect() + } + + pub(crate) fn transform(&mut self, b: &TreeDiff, left_prior: bool) { + // println!("\ntransform prior {:?} {:?} \nb {:?}", left_prior, self, b); + if b.is_empty() || self.is_empty() { + return; + } + if !left_prior { + let mut self_update = self.to_hash_map_mut(); + for i in b + .iter() + .map(|x| x.target) + .filter_map(|x| self_update.remove(&x)) + .sorted() + .rev() + { + self.remove(i); + } + } + } +} + +/// Representation of differences in movable tree. It's an ordered list of [`TreeDiff`]. +#[derive(Clone, Default)] +pub struct TreeDelta { + pub(crate) diff: Vec, +} + +impl Debug for TreeDelta { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("TreeDelta{ diff: [\n")?; + for item in self.diff.iter() { + f.write_fmt(format_args!("\t{:?}, \n", item))?; + } + f.write_str("]}") + } +} + +/// The semantic action in movable tree. +#[derive(Debug, Clone)] +pub struct TreeDeltaItem { + pub target: TreeID, + pub action: TreeInternalDiff, + pub last_effective_move_op_id: IdFull, +} + +/// The action of [`TreeDiff`]. It's the same as [`crate::container::tree::tree_op::TreeOp`], but semantic. +#[derive(Debug, Clone)] +pub enum TreeInternalDiff { + /// First create the node, have not seen it before + Create { + parent: TreeParentId, + position: FractionalIndex, + }, + /// For retreating, if the node is only created, not move it to `DELETED_ROOT` but delete it directly + UnCreate, + /// Move the node to the parent, the node exists + Move { + parent: TreeParentId, + position: FractionalIndex, + }, + /// move under a parent that is deleted + Delete { + parent: TreeParentId, + position: Option, + }, + /// old parent is deleted, new parent is deleted too + MoveInDelete { + parent: TreeParentId, + position: Option, + }, +} + +impl TreeDeltaItem { + /// * `is_new_parent_deleted` and `is_old_parent_deleted`: we need to infer whether it's a `creation`. + /// It's a creation if the old_parent is deleted but the new parent isn't. + /// If it is a creation, we need to emit the `Create` event so that downstream event handler can + /// handle the new containers easier. + pub(crate) fn new( + target: TreeID, + parent: TreeParentId, + old_parent: TreeParentId, + op_id: IdFull, + is_new_parent_deleted: bool, + is_old_parent_deleted: bool, + position: Option, + ) -> Self { + let action = if matches!(parent, TreeParentId::Unexist) { + TreeInternalDiff::UnCreate + } else { + match ( + is_new_parent_deleted, + is_old_parent_deleted || old_parent == TreeParentId::Unexist, + ) { + (true, true) => TreeInternalDiff::MoveInDelete { parent, position }, + (true, false) => TreeInternalDiff::Delete { parent, position }, + (false, true) => TreeInternalDiff::Create { + parent, + position: position.unwrap(), + }, + (false, false) => TreeInternalDiff::Move { + parent, + position: position.unwrap(), + }, + } + }; + + TreeDeltaItem { + target, + action, + last_effective_move_op_id: op_id, + } + } +} + +impl Deref for TreeDelta { + type Target = Vec; + fn deref(&self) -> &Self::Target { + &self.diff + } +} + +impl TreeDelta { + // TODO: cannot handle this for now + pub(crate) fn compose(mut self, x: TreeDelta) -> TreeDelta { + self.diff.extend(x.diff); + self + } +} + +impl Deref for TreeDiff { + type Target = Vec; + fn deref(&self) -> &Self::Target { + &self.diff + } +} + +impl DerefMut for TreeDiff { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.diff + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/diff/LICENSE b/3rdparty/loro/crates/loro-internal/src/diff/LICENSE new file mode 100644 index 0000000000..f49a4e16e6 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/diff/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/3rdparty/loro/crates/loro-internal/src/diff/diff_impl.rs b/3rdparty/loro/crates/loro-internal/src/diff/diff_impl.rs new file mode 100644 index 0000000000..aef44cd5bf --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/diff/diff_impl.rs @@ -0,0 +1,857 @@ +//! This file was copied from the similar crate at +//! https://github.com/mitsuhiko/similar/blob/2b31f65445df9093ba007ca5a5ae6a71b899d491/src/algorithms/myers.rs +//! The original license is in the LICENSE file in the same directory as this file +//! +//! This file has been optimized for constant performance, removed some unnecessary cases +//! and simplified the use of DiffHandler. +//! Myers' diff algorithm. +//! +//! * time: `O((N+M)D)` +//! * space `O(N+M)` +//! +//! See [the original article by Eugene W. Myers](http://www.xmailserver.org/diff2.pdf) +//! describing it. +//! +//! The implementation of this algorithm is based on the implementation by +//! Brandon Williams. +use crate::change::get_sys_timestamp; +use rustc_hash::FxHashMap; +use std::cmp::Ordering; +use std::collections::BinaryHeap; +use std::iter::zip; +use std::ops::{Index, IndexMut}; + +/// Options for controlling the text update behavior. +/// +/// - `timeout_ms`: Optional timeout in milliseconds for the diff computation +/// - `use_refined_diff`: Whether to use a more refined but slower diff algorithm. Defaults to true. +#[derive(Clone, Debug)] +pub struct UpdateOptions { + pub timeout_ms: Option, + pub use_refined_diff: bool, +} + +impl Default for UpdateOptions { + fn default() -> Self { + Self { + timeout_ms: None, + use_refined_diff: true, + } + } +} + +#[derive(thiserror::Error, Debug, PartialEq)] +pub enum UpdateTimeoutError { + #[error("Timeout")] + Timeout, +} + +/// Utility function to check if a range is empty that works on older rust versions +#[inline(always)] +fn is_empty_range(start: usize, end: usize) -> bool { + start >= end +} + +#[inline(always)] +fn is_not_empty_range(start: usize, end: usize) -> bool { + start < end +} + +fn common_prefix(xs: &[u32], ys: &[u32]) -> usize { + let chunk_size = 4; + let off = zip(xs.chunks_exact(chunk_size), ys.chunks_exact(chunk_size)) + .take_while(|(xs_chunk, ys_chunk)| xs_chunk == ys_chunk) + .count() + * chunk_size; + off + zip(&xs[off..], &ys[off..]) + .take_while(|(x, y)| x == y) + .count() +} + +fn common_suffix_len(old: &[u32], new: &[u32]) -> usize { + let chunk_size = 4; + let old_len = old.len(); + let new_len = new.len(); + + let off = zip(old.rchunks_exact(chunk_size), new.rchunks_exact(chunk_size)) + .take_while(|(old_chunk, new_chunk)| old_chunk == new_chunk) + .count() + * chunk_size; + + off + zip( + old[..old_len - off].iter().rev(), + new[..new_len - off].iter().rev(), + ) + .take_while(|(o, n)| o == n) + .count() +} + +pub(crate) trait DiffHandler { + fn insert(&mut self, old_index: usize, new_index: usize, new_len: usize); + fn delete(&mut self, old_index: usize, old_len: usize); +} + +#[derive(Debug)] +pub(crate) struct OperateProxy { + handler: D, +} + +impl OperateProxy { + pub fn new(handler: D) -> Self { + Self { handler } + } + + pub fn delete(&mut self, old_index: usize, old_len: usize) { + self.handler.delete(old_index, old_len); + } + + pub fn insert(&mut self, old_index: usize, new_index: usize, new_len: usize) { + self.handler.insert(old_index, new_index, new_len); + } + + #[allow(unused)] + fn unwrap(self) -> D { + self.handler + } +} + +pub(crate) fn diff( + proxy: &mut OperateProxy, + options: UpdateOptions, + old: &[u32], + new: &[u32], +) -> Result<(), UpdateTimeoutError> { + let max_d = (old.len() + new.len()).div_ceil(2) + 1; + let mut vb = OffsetVec::new(max_d); + let mut vf = OffsetVec::new(max_d); + let start_time = if options.timeout_ms.is_some() { + get_sys_timestamp() + } else { + 0. + }; + + conquer( + proxy, + options.use_refined_diff, + options.timeout_ms, + start_time, + old, + 0, + old.len(), + new, + 0, + new.len(), + &mut vf, + &mut vb, + ) +} + +struct OffsetVec(isize, Vec); + +impl OffsetVec { + fn new(max_d: usize) -> Self { + Self(max_d as isize, vec![0; max_d << 1]) + } + + fn len(&self) -> usize { + self.1.len() + } +} + +impl Index for OffsetVec { + type Output = usize; + fn index(&self, index: isize) -> &Self::Output { + &self.1[(index + self.0) as usize] + } +} + +impl IndexMut for OffsetVec { + fn index_mut(&mut self, index: isize) -> &mut Self::Output { + &mut self.1[(index + self.0) as usize] + } +} + +struct MiddleSnakeResult { + #[allow(unused)] + d: usize, + x_start: usize, + y_start: usize, +} + +#[allow(clippy::too_many_arguments)] +fn find_middle_snake( + timeout_ms: Option, + start_time: f64, + old: &[u32], + old_start: usize, + old_end: usize, + new: &[u32], + new_start: usize, + new_end: usize, + vf: &mut OffsetVec, + vb: &mut OffsetVec, +) -> Result, UpdateTimeoutError> { + let n = old_end - old_start; + let m = new_end - new_start; + let delta = n as isize - m as isize; + let odd = delta & 1 != 0; + vf[1] = 0; + vb[1] = 0; + let d_max = (n + m).div_ceil(2) + 1; + assert!(vf.len() >= d_max); + assert!(vb.len() >= d_max); + + for d in 0..d_max as isize { + for k in (-d..=d).rev().step_by(2) { + let mut x = if k == -d || (k != d && vf[k - 1] < vf[k + 1]) { + vf[k + 1] + } else { + vf[k - 1] + 1 + }; + let y = (x as isize - k) as usize; + let (x0, y0) = (x, y); + if x < n && y < m { + let advance = + common_prefix(&old[old_start + x..old_end], &new[new_start + y..new_end]); + x += advance; + } + vf[k] = x; + if odd && (k - delta).abs() <= (d - 1) && vf[k] + vb[delta - k] >= n { + return Ok(Some(MiddleSnakeResult { + d: d as usize, + x_start: x0 + old_start, + y_start: y0 + new_start, + })); + } + + if let Some(timeout_ms) = timeout_ms { + if get_sys_timestamp() - start_time > timeout_ms { + return Err(UpdateTimeoutError::Timeout); + } + } + } + + for k in (-d..=d).rev().step_by(2) { + let mut x = if k == -d || (k != d && vb[k - 1] < vb[k + 1]) { + vb[k + 1] + } else { + vb[k - 1] + 1 + }; + let mut y = (x as isize - k) as usize; + if x < n && y < m { + let advance = common_suffix_len( + &old[old_start..old_start + n - x], + &new[new_start..new_start + m - y], + ); + x += advance; + y += advance; + } + vb[k] = x; + if !odd && (k - delta).abs() <= d && vb[k] + vf[delta - k] >= n { + return Ok(Some(MiddleSnakeResult { + d: d as usize, + x_start: n - x + old_start, + y_start: m - y + new_start, + })); + } + + if let Some(timeout_ms) = timeout_ms { + if get_sys_timestamp() - start_time > timeout_ms { + return Err(UpdateTimeoutError::Timeout); + } + } + } + } + + Ok(None) +} + +#[allow(clippy::too_many_arguments)] +fn conquer( + proxy: &mut OperateProxy, + should_use_dj: bool, + timeout_ms: Option, + start_time: f64, + old: &[u32], + mut old_start: usize, + mut old_end: usize, + new: &[u32], + mut new_start: usize, + mut new_end: usize, + vf: &mut OffsetVec, + vb: &mut OffsetVec, +) -> Result<(), UpdateTimeoutError> { + let common_prefix_len = common_prefix(&old[old_start..old_end], &new[new_start..new_end]); + if common_prefix_len > 0 { + old_start += common_prefix_len; + new_start += common_prefix_len; + } + + let common_suffix_len = common_suffix_len(&old[old_start..old_end], &new[new_start..new_end]); + old_end -= common_suffix_len; + new_end -= common_suffix_len; + + if is_not_empty_range(old_start, old_end) || is_not_empty_range(new_start, new_end) { + let len_old = old_end - old_start; + let len_new = new_end - new_start; + if should_use_dj && (len_old.max(1) * len_new.max(1) < 128 * 128) { + let ok = dj_diff( + proxy, + &old[old_start..old_end], + &new[new_start..new_end], + old_start, + new_start, + 10_000, + ); + if ok { + return Ok(()); + } + } + + if is_empty_range(new_start, new_end) { + proxy.delete(old_start, old_end - old_start); + } else if is_empty_range(old_start, old_end) { + proxy.insert(old_start, new_start, new_end - new_start); + } else if let Some(MiddleSnakeResult { + d: _, + x_start, + y_start, + }) = find_middle_snake( + timeout_ms, start_time, old, old_start, old_end, new, new_start, new_end, vf, vb, + )? { + conquer( + proxy, + should_use_dj, + timeout_ms, + start_time, + old, + old_start, + x_start, + new, + new_start, + y_start, + vf, + vb, + )?; + conquer( + proxy, + should_use_dj, + timeout_ms, + start_time, + old, + x_start, + old_end, + new, + y_start, + new_end, + vf, + vb, + )?; + } else { + proxy.delete(old_start, old_end - old_start); + proxy.insert(old_start, new_start, new_end - new_start); + } + } + + Ok(()) +} + +/// Return false if this method gives up early +#[must_use] +pub(crate) fn dj_diff( + proxy: &mut OperateProxy, + old: &[u32], + new: &[u32], + old_offset: usize, + new_offset: usize, + max_try_count: usize, +) -> bool { + let common_prefix_len = common_prefix(old, new); + let common_suffix_len = common_suffix_len(&old[common_prefix_len..], &new[common_prefix_len..]); + let old = &old[common_prefix_len..old.len() - common_suffix_len]; + let new = &new[common_prefix_len..new.len() - common_suffix_len]; + if old.len() >= u16::MAX as usize || new.len() >= u16::MAX as usize { + return false; + } + + if old.is_empty() { + if new.is_empty() { + return true; + } + + proxy.insert( + old_offset + common_prefix_len, + new_offset + common_prefix_len, + new.len(), + ); + return true; + } + + if new.is_empty() { + proxy.delete(old_offset + common_prefix_len, old.len()); + return true; + } + + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] + struct Point { + x: u16, + y: u16, + } + + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + enum Direction { + Up, + Left, + UpLeft, + } + + struct QueueItem { + point: Point, + cost: u32, + from: Direction, + } + + impl PartialEq for QueueItem { + fn eq(&self, other: &Self) -> bool { + self.cost == other.cost + } + } + + impl PartialOrd for QueueItem { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } + } + + impl Ord for QueueItem { + fn cmp(&self, other: &Self) -> Ordering { + other.cost.cmp(&self.cost) + } + } + + impl Eq for QueueItem {} + + let mut visited: FxHashMap = FxHashMap::default(); + let mut q: BinaryHeap = BinaryHeap::new(); + q.push(QueueItem { + point: Point { x: 0, y: 0 }, + cost: 0, + from: Direction::UpLeft, + }); + + while let Some(QueueItem { point, cost, from }) = q.pop() { + if visited.contains_key(&point) { + continue; + } + + visited.insert(point, from); + if point.x == old.len() as u16 && point.y == new.len() as u16 { + break; + } + + if visited.len() + q.len() > max_try_count { + // println!("give up on: visited len: {}", visited.len()); + // println!("queue len: {}", q.len()); + return false; + } + + if point.x < old.len() as u16 { + let next_point = Point { + x: point.x + 1, + y: point.y, + }; + + if !visited.contains_key(&next_point) { + let mut next_cost = cost + 1; + if from != Direction::Left { + next_cost += 8; + } + + q.push(QueueItem { + point: next_point, + cost: next_cost, + from: Direction::Left, + }); + } + } + + if point.y < new.len() as u16 { + let next_point = Point { + x: point.x, + y: point.y + 1, + }; + + if !visited.contains_key(&next_point) { + let direction = Direction::Up; + let mut next_cost = cost + 1; + if from != direction { + next_cost += 8; + } + + q.push(QueueItem { + point: next_point, + cost: next_cost, + from: Direction::Up, + }); + } + } + + if point.x < old.len() as u16 + && point.y < new.len() as u16 + && old[point.x as usize] == new[point.y as usize] + { + let next_point = Point { + x: point.x + 1, + y: point.y + 1, + }; + + if !visited.contains_key(&next_point) { + let next_cost = if from == Direction::UpLeft { + cost + } else { + cost + 1 + }; + + q.push(QueueItem { + point: next_point, + cost: next_cost, + from: Direction::UpLeft, + }); + } + } + } + + // println!("visited len: {}", visited.len()); + // println!("queue len: {}", q.len()); + + // Backtrack from end point to construct diff operations + let mut current = Point { + x: old.len() as u16, + y: new.len() as u16, + }; + + let mut path: Vec<(Direction, usize)> = Vec::new(); + while current.x > 0 || current.y > 0 { + let direction = visited.get(¤t).unwrap(); + if let Some((last_dir, count)) = path.last_mut() { + if last_dir == direction { + *count += 1; + } else { + path.push((*direction, 1)); + } + } else { + path.push((*direction, 1)); + } + match direction { + Direction::Left => { + current.x -= 1; + } + Direction::Up => { + current.y -= 1; + } + Direction::UpLeft => { + current.x -= 1; + current.y -= 1; + } + } + } + + path.reverse(); + + let mut old_index = common_prefix_len; + let mut new_index = common_prefix_len; + + for (direction, count) in path { + match direction { + Direction::Left => { + proxy.delete(old_offset + old_index, count); + old_index += count; + } + Direction::Up => { + proxy.insert(old_offset + old_index, new_index + new_offset, count); + new_index += count; + } + Direction::UpLeft => { + old_index += count; + new_index += count; + } + } + } + + true +} + +#[cfg(test)] +mod tests { + use super::*; + + #[derive(Debug, Default)] + struct RecordingDiffHandler { + ops: Vec, + } + + #[derive(Debug, PartialEq)] + enum DiffOperation { + Insert { + old_index: usize, + new_index: usize, + length: usize, + }, + Delete { + old_index: usize, + length: usize, + }, + } + + impl DiffHandler for RecordingDiffHandler { + fn insert(&mut self, pos: usize, start: usize, len: usize) { + self.ops.push(DiffOperation::Insert { + old_index: pos, + new_index: start, + length: len, + }); + } + + fn delete(&mut self, pos: usize, len: usize) { + self.ops.push(DiffOperation::Delete { + old_index: pos, + length: len, + }); + } + } + + #[test] + fn test_recording_diff_handler() { + let handler = RecordingDiffHandler::default(); + let mut proxy = OperateProxy::new(handler); + let old = vec![1, 2, 3]; + let new = vec![1, 4, 3]; + + let _ = dj_diff(&mut proxy, &old, &new, 0, 0, 100_000); + let handler = proxy.unwrap(); + assert_eq!( + handler.ops, + vec![ + DiffOperation::Delete { + old_index: 1, + length: 1 + }, + DiffOperation::Insert { + old_index: 2, + new_index: 1, + length: 1 + }, + ] + ); + } + + #[test] + fn test_dj_diff_same() { + let handler = RecordingDiffHandler::default(); + let mut proxy = OperateProxy::new(handler); + let old = vec![1, 2, 3]; + let new = vec![1, 2, 3]; + let _ = dj_diff(&mut proxy, &old, &new, 0, 0, 100_000); + let handler = proxy.unwrap(); + assert_eq!(handler.ops, vec![]); + } + + #[test] + fn test_dj_diff_1() { + let handler = RecordingDiffHandler::default(); + let mut proxy = OperateProxy::new(handler); + let old = vec![1]; + let new = vec![0, 1, 2]; + let _ = dj_diff(&mut proxy, &old, &new, 0, 0, 100_000); + let handler = proxy.unwrap(); + assert_eq!( + handler.ops, + vec![ + DiffOperation::Insert { + old_index: 0, + new_index: 0, + length: 1 + }, + DiffOperation::Insert { + old_index: 1, + new_index: 2, + length: 1 + }, + ] + ); + } + + #[test] + fn test_diff_may_scatter() { + let handler = RecordingDiffHandler::default(); + let mut proxy = OperateProxy::new(handler); + let old = vec![1, 2, 3, 4, 5]; + let new = vec![99, 1, 2, 3, 4, 5, 98, 97, 96, 3, 95, 4, 93, 92, 5, 91]; + diff(&mut proxy, UpdateOptions::default(), &old, &new).unwrap(); + let handler = proxy.unwrap(); + assert_eq!( + handler.ops, + vec![ + DiffOperation::Insert { + old_index: 0, + new_index: 0, + length: 1 + }, + DiffOperation::Insert { + old_index: 5, + new_index: 6, + length: 10 + }, + ] + ); + } + + #[test] + fn test_diff_may_scatter_1() { + let handler = RecordingDiffHandler::default(); + let mut proxy = OperateProxy::new(handler); + let old = vec![1, 2, 3, 4, 5]; + let new = vec![99, 1, 2, 98, 97, 96, 3, 95, 4, 93, 92, 5, 1, 2, 3, 4, 5, 91]; + diff(&mut proxy, UpdateOptions::default(), &old, &new).unwrap(); + let handler = proxy.unwrap(); + assert_eq!( + handler.ops, + vec![ + DiffOperation::Insert { + old_index: 0, + new_index: 0, + length: 12 + }, + DiffOperation::Insert { + old_index: 5, + new_index: 17, + length: 1 + }, + ] + ); + } + + #[test] + fn test_dj_diff_may_scatter() { + let handler = RecordingDiffHandler::default(); + let mut proxy = OperateProxy::new(handler); + let old = vec![1, 2, 3, 4, 5]; + let new = vec![99, 1, 2, 3, 4, 5, 98, 97, 96, 3, 95, 4, 93, 92, 5, 91]; + let _ = dj_diff(&mut proxy, &old, &new, 0, 0, 100_000); + let handler = proxy.unwrap(); + assert_eq!( + handler.ops, + vec![ + DiffOperation::Insert { + old_index: 0, + new_index: 0, + length: 1 + }, + DiffOperation::Insert { + old_index: 5, + new_index: 6, + length: 10 + }, + ] + ); + } + + #[test] + fn test_dj_diff_may_scatter_1() { + let handler = RecordingDiffHandler::default(); + let mut proxy = OperateProxy::new(handler); + let old = vec![1, 2, 3, 4, 5]; + let new = vec![99, 1, 2, 98, 97, 96, 3, 95, 4, 93, 92, 5, 1, 2, 3, 4, 5, 91]; + let _ = dj_diff(&mut proxy, &old, &new, 0, 0, 100_000); + let handler = proxy.unwrap(); + assert_eq!( + handler.ops, + vec![ + DiffOperation::Insert { + old_index: 0, + new_index: 0, + length: 12 + }, + DiffOperation::Insert { + old_index: 5, + new_index: 17, + length: 1 + }, + ] + ); + } + + #[test] + fn test_dj_diff_100_differences() { + let handler = RecordingDiffHandler::default(); + let mut proxy = OperateProxy::new(handler); + let old = vec![1; 100]; + let new = vec![2; 100]; + let _ = dj_diff(&mut proxy, &old, &new, 0, 0, 100_000); + let handler = proxy.unwrap(); + assert_eq!( + handler.ops, + vec![ + DiffOperation::Delete { + old_index: 0, + length: 100 + }, + DiffOperation::Insert { + old_index: 100, + new_index: 0, + length: 100 + }, + ] + ); + } + + #[test] + fn test_dj_diff_insert() { + let handler = RecordingDiffHandler::default(); + let mut proxy = OperateProxy::new(handler); + let old = vec![1; 100]; + let mut new = old.clone(); + new.splice(50..50, [1, 2, 3, 4, 1, 2, 3, 4]); + new.splice(0..0, [0, 1, 2, 3, 4, 5, 6, 7]); + let _ = dj_diff(&mut proxy, &old, &new, 0, 0, 100_000); + let handler = proxy.unwrap(); + assert_eq!( + handler.ops, + vec![ + DiffOperation::Insert { + old_index: 0, + new_index: 0, + length: 9 + }, + DiffOperation::Insert { + old_index: 50, + new_index: 59, + length: 7 + } + ] + ); + } + + #[test] + fn test_diff() { + let handler = RecordingDiffHandler::default(); + let mut proxy = OperateProxy::new(handler); + let old = vec![1; 100]; + let new = vec![2; 100]; + diff(&mut proxy, Default::default(), &old, &new).unwrap(); + } + + #[test] + fn test_timeout() { + let handler = RecordingDiffHandler::default(); + let mut proxy = OperateProxy::new(handler); + let old = vec![1; 10000]; + let new = vec![2; 10000]; + let options = UpdateOptions { + timeout_ms: Some(0.1), + ..Default::default() + }; + let result = diff(&mut proxy, options, &old, &new); + assert!(result.is_err()); + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/diff/mod.rs b/3rdparty/loro/crates/loro-internal/src/diff/mod.rs new file mode 100644 index 0000000000..5d9d5cb794 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/diff/mod.rs @@ -0,0 +1,4 @@ +pub mod diff_impl; +pub(crate) use diff_impl::diff; +pub(crate) use diff_impl::DiffHandler; +pub(crate) use diff_impl::OperateProxy; diff --git a/3rdparty/loro/crates/loro-internal/src/diff_calc.rs b/3rdparty/loro/crates/loro-internal/src/diff_calc.rs new file mode 100644 index 0000000000..1af8ccd606 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/diff_calc.rs @@ -0,0 +1,2036 @@ +use std::{num::NonZeroU16, sync::Arc}; + +#[cfg(feature = "counter")] +mod counter; +#[cfg(feature = "counter")] +pub(crate) use counter::CounterDiffCalculator; +pub(super) mod tree; +mod unknown; +use either::Either; +use generic_btree::rle::{HasLength as _, Sliceable as _}; +use itertools::Itertools; + +use enum_dispatch::enum_dispatch; +use loro_common::{ + CompactIdLp, ContainerID, Counter, HasCounterSpan, IdFull, IdLp, IdSpan, LoroValue, PeerID, ID, +}; +use loro_delta::DeltaRope; +use rustc_hash::{FxHashMap, FxHashSet}; +use smallvec::SmallVec; +use tracing::{info_span, instrument}; + +use crate::{ + change::Lamport, + container::{ + idx::ContainerIdx, + list::list_op::InnerListOp, + richtext::{ + richtext_state::{RichtextStateChunk, TextChunk}, + AnchorType, CrdtRopeDelta, RichtextChunk, RichtextChunkValue, RichtextTracker, StyleOp, + }, + }, + cursor::AbsolutePosition, + delta::{ + Delta, DeltaItem, DeltaValue, ElementDelta, MapDelta, MapValue, MovableListInnerDelta, + }, + event::{DiffVariant, InternalDiff}, + op::{InnerContent, RichOp, SliceRange, SliceWithId}, + span::{HasId, HasLamport}, + version::Frontiers, + InternalString, VersionVector, +}; + +use self::tree::TreeDiffCalculator; + +use self::unknown::UnknownDiffCalculator; + +use super::{event::InternalContainerDiff, oplog::OpLog}; + +/// Calculate the diff between two versions. given [OpLog][super::oplog::OpLog] +/// and [AppState][super::state::AppState]. +/// +/// TODO: persist diffCalculator and skip processed version +#[derive(Debug)] +pub struct DiffCalculator { + /// ContainerIdx -> (depth, calculator) + /// + /// if depth is None, we need to calculate it again + calculators: FxHashMap, ContainerDiffCalculator)>, + retain_mode: DiffCalculatorRetainMode, +} + +#[derive(Debug)] +enum DiffCalculatorRetainMode { + /// The diff calculator can only be used once. + Once { used: bool }, + /// The diff calculator will be persisted and can be reused after the diff calc is done. + Persist, +} + +/// This mode defines how the diff is calculated and how it should be applied on the state. +#[derive(Debug, Clone, PartialEq, Eq, Copy)] +pub(crate) enum DiffMode { + /// This is the most general mode of diff calculation. + /// + /// When applying `Checkout` diff, we already know the current state of the affected registers. + /// So there is no need to compare the lamport values. + /// + /// It can be used whenever a user want to switch to a different version. + /// But it is also the slowest mode. It relies on the `ContainerHistoryCache`, which is expensive to build and maintain in memory. + Checkout, + /// This mode is used when the user imports new updates. + /// + /// When applying `Import` diff, we may need to know the the current state. + /// For example, we may need to compare the current register's lamport with the update's lamport to decide + /// what's the new value. + /// + /// It has stricter requirements than `Checkout`: + /// + /// - The target version vector must be greater than the current version vector. + Import, + /// This mode is used when the user imports new updates and all the updates are guaranteed to greater than the current version. + /// + /// It has stricter requirements than `Import`. + /// - All the updates are greater than the current version. No update is concurrent to the current version. + /// - So LCA is always the `from` version + ImportGreaterUpdates, + /// This mode is used when we don't need to build CRDTs to calculate the difference. It is the fastest mode. + /// + /// It has stricter requirements than `ImportGreaterUpdates`. + /// - In `ImportGreaterUpdates`, all the updates are guaranteed to be greater than the current version. + /// - In `Linear`, all the updates are ordered, no concurrent update exists. + Linear, +} + +#[derive(Debug, Clone, Copy)] +pub(crate) struct DiffCalcVersionInfo<'a> { + from_vv: &'a VersionVector, + to_vv: &'a VersionVector, + from_frontiers: &'a Frontiers, + to_frontiers: &'a Frontiers, + lca_vv: &'a VersionVector, +} + +impl DiffCalculator { + /// Create a new diff calculator. + /// + /// If `persist` is true, the diff calculator will be persisted after the diff calc is done. + /// This is useful when we need to cache the diff calculator for future use. But it is slower + /// for importing updates and requires more memory. + pub fn new(persist: bool) -> Self { + Self { + calculators: Default::default(), + retain_mode: if persist { + DiffCalculatorRetainMode::Persist + } else { + DiffCalculatorRetainMode::Once { used: false } + }, + } + } + + #[allow(unused)] + pub(crate) fn get_calc(&self, container: ContainerIdx) -> Option<&ContainerDiffCalculator> { + self.calculators.get(&container).map(|(_, c)| c) + } + + /// Calculate the diff between two versions. + /// + /// Return the diff and the origin diff mode (it's not the diff mode used by the diff calculator. + /// It's the expected diff mode inferred from the two version, which can reflect the direction of the + /// change). + pub(crate) fn calc_diff_internal( + &mut self, + oplog: &super::oplog::OpLog, + before: &crate::VersionVector, + before_frontiers: &Frontiers, + after: &crate::VersionVector, + after_frontiers: &Frontiers, + container_filter: Option<&dyn Fn(ContainerIdx) -> bool>, + ) -> (Vec, DiffMode) { + if before == after { + return (Vec::new(), DiffMode::Linear); + } + + let s = tracing::span!(tracing::Level::INFO, "DiffCalc", ?before, ?after,); + let _e = s.enter(); + + let mut merged = before.clone(); + merged.merge(after); + let (lca, origin_diff_mode, iter) = + oplog.iter_from_lca_causally(before, before_frontiers, after, after_frontiers); + let mut diff_mode = origin_diff_mode; + match &mut self.retain_mode { + DiffCalculatorRetainMode::Once { used } => { + if *used { + panic!("DiffCalculator with retain_mode Once can only be used once"); + } + } + DiffCalculatorRetainMode::Persist => { + diff_mode = DiffMode::Checkout; + } + } + + let affected_set = { + loro_common::debug!("LCA: {:?} mode={:?}", &lca, diff_mode); + let mut started_set = FxHashSet::default(); + for (change, (start_counter, end_counter), vv) in iter { + let iter_start = change + .ops + .binary_search_by(|op| op.ctr_last().cmp(&start_counter)) + .unwrap_or_else(|e| e); + let mut visited = FxHashSet::default(); + for mut op in &change.ops.vec()[iter_start..] { + if op.counter >= end_counter { + break; + } + + let idx = op.container; + if let Some(filter) = container_filter { + if !filter(idx) { + continue; + } + } + + // slice the op if needed + // PERF: we can skip the slice by using the RichOp::new_slice + let stack_sliced_op; + if op.ctr_last() < start_counter { + continue; + } + + if op.counter < start_counter || op.ctr_end() > end_counter { + stack_sliced_op = Some(op.slice( + (start_counter as usize).saturating_sub(op.counter as usize), + op.atom_len().min((end_counter - op.counter) as usize), + )); + op = stack_sliced_op.as_ref().unwrap(); + } + + let vv = &mut vv.borrow_mut(); + vv.extend_to_include_end_id(ID::new(change.peer(), op.counter)); + let container = op.container; + let depth = oplog.arena.get_depth(container); + let (old_depth, calculator) = self.get_or_create_calc(container, depth); + // checkout use the same diff_calculator, the depth of calculator is not updated + // That may cause the container to be considered deleted + if *old_depth != depth { + *old_depth = depth; + } + + if !started_set.contains(&op.container) { + started_set.insert(container); + calculator.start_tracking(oplog, &lca, diff_mode); + } + + if !vv.includes_vv(before) { + calculator.mark_source_not_in_op_context(); + } + + if visited.contains(&op.container) { + // don't checkout if we have already checked out this container in this round + calculator.apply_change(oplog, RichOp::new_by_change(&change, op), None); + } else { + calculator.apply_change( + oplog, + RichOp::new_by_change(&change, op), + Some(vv), + ); + visited.insert(container); + } + } + } + + Some(started_set) + }; + + // Because we need to get correct `bring_back` value that indicates container is created during this round of diff calc, + // we need to iterate from parents to children. i.e. from smaller depth to larger depth. + let mut new_containers = FxHashSet::default(); + let mut container_id_to_depth = FxHashMap::default(); + let mut all: Vec<(Option, ContainerIdx)> = if let Some(set) = affected_set { + // only visit the affected containers + set.into_iter() + .map(|x| { + let (depth, _) = self.calculators.get_mut(&x).unwrap(); + (*depth, x) + }) + .collect() + } else { + self.calculators + .iter_mut() + .map(|(x, (depth, _))| (*depth, *x)) + .collect() + }; + let mut ans = FxHashMap::default(); + let info = DiffCalcVersionInfo { + from_vv: before, + to_vv: after, + from_frontiers: before_frontiers, + to_frontiers: after_frontiers, + lca_vv: &lca, + }; + while !all.is_empty() { + // sort by depth and lamport, ensure we iterate from top to bottom + all.sort_by_key(|x| x.0); + for (_, container_idx) in std::mem::take(&mut all) { + if ans.contains_key(&container_idx) { + continue; + } + let (depth, calc) = self.calculators.get_mut(&container_idx).unwrap(); + if depth.is_none() { + let d = oplog.arena.get_depth(container_idx); + if d != *depth { + *depth = d; + all.push((*depth, container_idx)); + continue; + } + } + let id = oplog.arena.idx_to_id(container_idx).unwrap(); + let bring_back = new_containers.remove(&id); + + info_span!("CalcDiff", ?id).in_scope(|| { + let (diff, diff_mode) = calc.calculate_diff(container_idx, oplog, info, |c| { + new_containers.insert(c.clone()); + container_id_to_depth + .insert(c.clone(), depth.and_then(|d| d.checked_add(1))); + oplog.arena.register_container(c); + }); + calc.finish_this_round(); + if !diff.is_empty() || bring_back { + ans.insert( + container_idx, + ( + *depth, + InternalContainerDiff { + idx: container_idx, + bring_back, + diff: diff.into(), + diff_mode, + }, + ), + ); + } + }); + } + } + + while !new_containers.is_empty() { + for id in std::mem::take(&mut new_containers) { + // Registration can be lazy; ensure it is registered so we can proceed + let idx = oplog.arena.register_container(&id); + if ans.contains_key(&idx) { + continue; + } + let depth = container_id_to_depth.remove(&id).unwrap(); + ans.insert( + idx, + ( + depth, + InternalContainerDiff { + idx, + bring_back: true, + diff: DiffVariant::None, + diff_mode: DiffMode::Checkout, + }, + ), + ); + } + } + + ( + ans.into_values().map(|x| x.1).collect_vec(), + origin_diff_mode, + ) + } + + // TODO: we may remove depth info + pub(crate) fn get_or_create_calc( + &mut self, + idx: ContainerIdx, + depth: Option, + ) -> &mut (Option, ContainerDiffCalculator) { + self.calculators + .entry(idx) + .or_insert_with(|| match idx.get_type() { + crate::ContainerType::Text => ( + depth, + ContainerDiffCalculator::Richtext(RichtextDiffCalculator::new()), + ), + crate::ContainerType::Map => ( + depth, + ContainerDiffCalculator::Map(MapDiffCalculator::new(idx)), + ), + crate::ContainerType::List => ( + depth, + ContainerDiffCalculator::List(ListDiffCalculator::default()), + ), + crate::ContainerType::Tree => ( + depth, + ContainerDiffCalculator::Tree(TreeDiffCalculator::new(idx)), + ), + crate::ContainerType::Unknown(_) => ( + depth, + ContainerDiffCalculator::Unknown(unknown::UnknownDiffCalculator), + ), + crate::ContainerType::MovableList => ( + depth, + ContainerDiffCalculator::MovableList(MovableListDiffCalculator::new(idx)), + ), + #[cfg(feature = "counter")] + crate::ContainerType::Counter => ( + depth, + ContainerDiffCalculator::Counter(CounterDiffCalculator::new(idx)), + ), + }) + } +} + +/// DiffCalculator should track the history first before it can calculate the difference. +/// +/// So we need it to first apply all the ops between the two versions. +/// +/// NOTE: not every op between two versions are included in a certain container. +/// So there may be some ops that cannot be seen by the container. +/// +#[enum_dispatch] +pub(crate) trait DiffCalculatorTrait { + fn start_tracking(&mut self, oplog: &OpLog, vv: &crate::VersionVector, mode: DiffMode); + fn apply_change( + &mut self, + oplog: &OpLog, + op: crate::op::RichOp, + vv: Option<&crate::VersionVector>, + ); + fn calculate_diff( + &mut self, + idx: ContainerIdx, + oplog: &OpLog, + info: DiffCalcVersionInfo, + on_new_container: impl FnMut(&ContainerID), + ) -> (InternalDiff, DiffMode); + /// This round of diff calc is finished, we can clear the cache + fn finish_this_round(&mut self); +} + +#[enum_dispatch(DiffCalculatorTrait)] +#[derive(Debug)] +pub(crate) enum ContainerDiffCalculator { + Map(MapDiffCalculator), + List(ListDiffCalculator), + Richtext(RichtextDiffCalculator), + Tree(TreeDiffCalculator), + MovableList(MovableListDiffCalculator), + #[cfg(feature = "counter")] + Counter(counter::CounterDiffCalculator), + Unknown(UnknownDiffCalculator), +} + +impl ContainerDiffCalculator { + fn mark_source_not_in_op_context(&mut self) { + match self { + Self::Richtext(calc) => calc.mark_source_not_in_op_context(), + Self::List(calc) => calc.mark_source_not_in_op_context(), + Self::MovableList(calc) => calc.mark_source_not_in_op_context(), + _ => {} + } + } +} + +trait RebuildOpVisitor { + fn visit(&mut self, vv: &VersionVector, op: RichOp<'_>); +} + +#[cold] +#[inline(never)] +fn replay_container_ops_from_empty( + idx: ContainerIdx, + oplog: &OpLog, + vv: &VersionVector, + visitor: &mut dyn RebuildOpVisitor, +) { + let empty_vv = VersionVector::default(); + let empty_frontiers = Frontiers::default(); + let target_frontiers = oplog.dag.vv_to_frontiers(vv); + let (_, _, iter) = + oplog.iter_from_lca_causally(&empty_vv, &empty_frontiers, vv, &target_frontiers); + + for (change, (start_counter, end_counter), vv) in iter { + let iter_start = change + .ops + .binary_search_by(|op| op.ctr_last().cmp(&start_counter)) + .unwrap_or_else(|e| e); + for mut op in &change.ops.vec()[iter_start..] { + if op.counter >= end_counter { + break; + } + + if op.container != idx || op.ctr_last() < start_counter { + continue; + } + + let stack_sliced_op; + if op.counter < start_counter || op.ctr_end() > end_counter { + stack_sliced_op = Some(op.slice( + (start_counter as usize).saturating_sub(op.counter as usize), + op.atom_len().min((end_counter - op.counter) as usize), + )); + op = stack_sliced_op.as_ref().unwrap(); + } + + let vv = &mut vv.borrow_mut(); + vv.extend_to_include_end_id(ID::new(change.peer(), op.counter)); + visitor.visit(vv, RichOp::new_by_change(&change, op)); + } + } +} + +#[derive(Debug)] +pub(crate) struct MapDiffCalculator { + container_idx: ContainerIdx, + changed: FxHashMap>, + current_mode: DiffMode, +} + +impl MapDiffCalculator { + pub(crate) fn new(container_idx: ContainerIdx) -> Self { + Self { + container_idx, + changed: Default::default(), + current_mode: DiffMode::Checkout, + } + } +} + +impl DiffCalculatorTrait for MapDiffCalculator { + fn start_tracking( + &mut self, + _oplog: &crate::OpLog, + _vv: &crate::VersionVector, + mode: DiffMode, + ) { + self.changed.clear(); + self.current_mode = mode; + } + + fn apply_change( + &mut self, + _oplog: &crate::OpLog, + op: crate::op::RichOp, + _vv: Option<&crate::VersionVector>, + ) { + if matches!(self.current_mode, DiffMode::Checkout) { + // We need to use history cache anyway + return; + } + + let map = op.raw_op().content.as_map().unwrap(); + let new_value = MapValue { + value: map.value.clone(), + peer: op.peer, + lamp: op.lamport(), + }; + match self.changed.get(&map.key) { + Some(Some(old_value)) if old_value > &new_value => {} + _ => { + self.changed.insert(map.key.clone(), Some(new_value)); + } + } + } + + fn finish_this_round(&mut self) { + self.changed.clear(); + self.current_mode = DiffMode::Checkout; + } + + fn calculate_diff( + &mut self, + _idx: ContainerIdx, + oplog: &super::oplog::OpLog, + DiffCalcVersionInfo { from_vv, to_vv, .. }: DiffCalcVersionInfo, + mut on_new_container: impl FnMut(&ContainerID), + ) -> (InternalDiff, DiffMode) { + match self.current_mode { + DiffMode::Checkout | DiffMode::Import => oplog.with_history_cache(|h| { + let checkout_index = &h.get_checkout_index().map; + let mut changed = Vec::new(); + let from_map = checkout_index.get_container_latest_op_at_vv( + self.container_idx, + from_vv, + Lamport::MAX, + oplog, + ); + let mut to_map = checkout_index.get_container_latest_op_at_vv( + self.container_idx, + to_vv, + Lamport::MAX, + oplog, + ); + + for (k, peek_from) in from_map.iter() { + let peek_to = to_map.remove(k); + match peek_to { + None => changed.push((k.clone(), None)), + Some(b) => { + if peek_from.value != b.value { + changed.push((k.clone(), Some(b))) + } + } + } + } + + for (k, peek_to) in to_map.into_iter() { + changed.push((k, Some(peek_to))); + } + + let mut updated = + FxHashMap::with_capacity_and_hasher(changed.len(), Default::default()); + for (key, value) in changed { + let value = value.map(|v| { + let value = v.value.clone(); + if let Some(LoroValue::Container(c)) = &value { + on_new_container(c); + } + + MapValue { + value, + lamp: v.lamport, + peer: v.peer, + } + }); + + updated.insert(key, value); + } + + (InternalDiff::Map(MapDelta { updated }), DiffMode::Checkout) + }), + DiffMode::ImportGreaterUpdates | DiffMode::Linear => { + let changed = std::mem::take(&mut self.changed); + let mode = self.current_mode; + // Reset this field to avoid we use `has_all` to cache the diff calc and use it next round + // (In the next round we need to use the checkout mode) + self.current_mode = DiffMode::Checkout; + (InternalDiff::Map(MapDelta { updated: changed }), mode) + } + } + } +} + +use rle::{HasLength as _, Sliceable}; + +#[derive(Default)] +pub(crate) struct ListDiffCalculator { + start_vv: VersionVector, + tracker: Box, + source_not_in_op_context: bool, +} + +impl ListDiffCalculator { + pub(crate) fn get_id_latest_pos(&self, id: ID) -> Option { + self.tracker.get_target_id_latest_index_at_new_version(id) + } + + fn mark_source_not_in_op_context(&mut self) { + self.source_not_in_op_context = true; + } + + #[inline(never)] + fn apply_op_to_tracker(tracker: &mut RichtextTracker, op: &crate::op::RichOp<'_>) { + match &op.op().content { + crate::op::InnerContent::List(l) => match l { + InnerListOp::Insert { slice, pos } => { + tracker.insert(op.id_full(), *pos, RichtextChunk::new_text(slice.0.clone())); + } + InnerListOp::Delete(del) => { + tracker.delete( + op.id_start(), + del.id_start, + del.start() as usize, + del.atom_len(), + del.is_reversed(), + ); + } + _ => unreachable!(), + }, + _ => unreachable!(), + } + } + + #[cold] + #[inline(never)] + fn build_full_tracker(idx: ContainerIdx, oplog: &OpLog, vv: &VersionVector) -> RichtextTracker { + struct ListRebuildVisitor<'a> { + tracker: &'a mut RichtextTracker, + } + + impl RebuildOpVisitor for ListRebuildVisitor<'_> { + fn visit(&mut self, vv: &VersionVector, op: RichOp<'_>) { + self.tracker.checkout(vv); + ListDiffCalculator::apply_op_to_tracker(self.tracker, &op); + } + } + + let mut tracker = RichtextTracker::new_with_unknown(); + let mut visitor = ListRebuildVisitor { + tracker: &mut tracker, + }; + replay_container_ops_from_empty(idx, oplog, vv, &mut visitor); + + tracker + } +} + +impl MovableListDiffCalculator { + pub(crate) fn get_id_latest_pos(&self, id: ID) -> Option { + self.list + .tracker + .get_target_id_latest_index_at_new_version(id) + } + + fn mark_source_not_in_op_context(&mut self) { + self.list.mark_source_not_in_op_context(); + } +} + +impl std::fmt::Debug for ListDiffCalculator { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ListDiffCalculator") + // .field("tracker", &self.tracker) + .finish() + } +} + +impl DiffCalculatorTrait for ListDiffCalculator { + fn start_tracking(&mut self, _oplog: &OpLog, vv: &crate::VersionVector, _mode: DiffMode) { + self.source_not_in_op_context = false; + if !vv.includes_vv(&self.start_vv) || !self.tracker.all_vv().includes_vv(vv) { + *self.tracker = RichtextTracker::new_with_unknown(); + self.start_vv = vv.clone(); + } + + self.tracker.checkout(vv); + } + + fn apply_change( + &mut self, + _oplog: &OpLog, + op: crate::op::RichOp, + vv: Option<&crate::VersionVector>, + ) { + if let Some(vv) = vv { + self.tracker.checkout(vv); + } + + Self::apply_op_to_tracker(&mut self.tracker, &op); + } + + fn finish_this_round(&mut self) {} + + fn calculate_diff( + &mut self, + idx: ContainerIdx, + oplog: &OpLog, + info: DiffCalcVersionInfo, + mut on_new_container: impl FnMut(&ContainerID), + ) -> (InternalDiff, DiffMode) { + let mut delta = Delta::new(); + let (mut retreat, _) = info.from_vv.diff_iter(info.to_vv); + let has_retreat = retreat.next().is_some(); + let should_rebuild = matches!(idx.get_type(), crate::ContainerType::List) + && (has_retreat || info.lca_vv != info.from_vv || self.source_not_in_op_context); + let diff_items = if should_rebuild { + let mut merged = info.from_vv.clone(); + merged.merge(info.to_vv); + let mut full_tracker = Self::build_full_tracker(idx, oplog, &merged); + let diff_items = full_tracker.diff(info.from_vv, info.to_vv).collect_vec(); + *self.tracker = full_tracker; + diff_items + } else { + self.tracker.diff(info.from_vv, info.to_vv).collect_vec() + }; + + for item in diff_items { + match item { + CrdtRopeDelta::Retain(len) => { + delta = delta.retain(len); + } + CrdtRopeDelta::Insert { + chunk: value, + id, + lamport, + } => match value.value() { + RichtextChunkValue::Text(range) => { + for i in range.clone() { + let v = oplog.arena.get_value(i as usize); + if let Some(LoroValue::Container(c)) = &v { + on_new_container(c); + } + } + delta = delta.insert(SliceWithId { + values: Either::Left(SliceRange(range)), + id: IdFull::new(id.peer, id.counter, lamport.unwrap()), + elem_id: None, + }); + } + RichtextChunkValue::StyleAnchor { .. } => unreachable!(), + RichtextChunkValue::Unknown(len) => { + delta = handle_unknown(idx, id, oplog, len, &mut on_new_container, delta); + } + RichtextChunkValue::MoveAnchor => { + delta = handle_unknown(idx, id, oplog, 1, &mut on_new_container, delta); + } + }, + CrdtRopeDelta::Delete(len) => { + delta = delta.delete(len); + } + } + } + + /// Handle span with unknown content when calculating diff + /// + /// We can lookup the content of the span by the id in the oplog + fn handle_unknown( + idx: ContainerIdx, + mut id: ID, + oplog: &OpLog, + len: u32, + on_new_container: &mut dyn FnMut(&ContainerID), + mut delta: Delta, + ) -> Delta { + // assert not unknown id + assert_ne!(id.peer, PeerID::MAX); + let mut acc_len = 0; + let end = id.counter + len as Counter; + let shallow_root = oplog.shallow_since_vv().get(&id.peer).copied().unwrap_or(0); + if id.counter < shallow_root { + // need to find the content between id.counter ~ target_end in gc state + let target_end = shallow_root.min(end); + delta = oplog.with_history_cache(|h| { + let chunks = + h.find_list_chunks_in(idx, IdSpan::new(id.peer, id.counter, target_end)); + for c in chunks { + acc_len += c.length(); + match &c.values { + Either::Left(_) => unreachable!(), + Either::Right(r) => { + if let LoroValue::Container(c) = r { + on_new_container(c) + } + } + } + delta = delta.insert(c); + } + + delta + }); + id.counter = shallow_root; + } + + if id.counter < end { + for rich_op in oplog.iter_ops(IdSpan::new(id.peer, id.counter, end)) { + acc_len += rich_op.content_len(); + let op = rich_op.op(); + let lamport = rich_op.lamport(); + + if let InnerListOp::Insert { slice, pos: _ } = op.content.as_list().unwrap() { + let range = slice.clone(); + for i in slice.0.clone() { + let v = oplog.arena.get_value(i as usize); + if let Some(LoroValue::Container(c)) = &v { + (on_new_container)(c); + } + } + + delta = delta.insert(SliceWithId { + values: Either::Left(range), + id: IdFull::new(id.peer, op.counter, lamport), + elem_id: None, + }); + } else if let InnerListOp::Move { elem_id, .. } = op.content.as_list().unwrap() + { + delta = delta.insert(SliceWithId { + // We do NOT need an actual value range, + // movable list container will only use the id info + values: Either::Right(LoroValue::Null), + id: IdFull::new(id.peer, op.counter, lamport), + elem_id: Some(elem_id.compact()), + }); + } + } + } + + debug_assert_eq!(acc_len, len as usize); + delta + } + + (InternalDiff::ListRaw(delta), DiffMode::Checkout) + } +} + +#[derive(Debug)] +pub(crate) struct RichtextDiffCalculator { + mode: Box, +} + +#[derive(Debug)] +enum RichtextCalcMode { + Crdt { + tracker: Box, + /// (op, end_pos) + styles: Vec<(StyleOp, usize)>, + start_vv: VersionVector, + source_not_in_op_context: bool, + shallow_root_seeded: bool, + }, + Linear { + diff: DeltaRope, + last_style_start: Option<(Arc, u32)>, + }, +} + +impl RichtextDiffCalculator { + pub fn new() -> Self { + Self { + mode: Box::new(RichtextCalcMode::Crdt { + tracker: Box::new(RichtextTracker::new_with_unknown()), + styles: Vec::new(), + start_vv: VersionVector::new(), + source_not_in_op_context: false, + shallow_root_seeded: false, + }), + } + } + + fn mark_source_not_in_op_context(&mut self) { + if let RichtextCalcMode::Crdt { + source_not_in_op_context, + .. + } = &mut *self.mode + { + *source_not_in_op_context = true; + } + } + + fn style_for_end_anchor(oplog: &OpLog, op: &RichOp) -> Option<(StyleOp, usize)> { + let style_start_id = op.id().inc(-1); + if let Some(start_op) = oplog.get_op_that_includes(style_start_id) { + let InnerListOp::StyleStart { + start: _, + end, + key, + value, + info, + } = start_op.content.as_list().unwrap() + else { + unreachable!() + }; + + Some(( + StyleOp { + lamport: start_op.lamport(), + peer: style_start_id.peer, + cnt: style_start_id.counter, + key: key.clone(), + value: value.clone(), + info: *info, + }, + *end as usize, + )) + } else { + oplog.with_history_cache(|history_cache| { + history_cache + .find_text_style_end_in_shallow_root(op.raw_op().container, style_start_id) + }) + } + } + + fn shallow_delete_range_matches_tracker( + tracker: &RichtextTracker, + target_start: ID, + pos: usize, + len: usize, + ) -> bool { + let mut remaining = len; + let mut pos = pos; + let mut expected = target_start; + while remaining > 0 { + let Some((real_id, available)) = tracker.active_real_span_at(pos) else { + return false; + }; + + if real_id.peer != expected.peer || real_id.counter != expected.counter { + return false; + } + + let take = remaining.min(available); + expected = expected.inc(take as Counter); + pos += take; + remaining -= take; + } + + true + } + + fn shallow_clamped_tracker_pos(oplog: &OpLog, tracker: &RichtextTracker, pos: usize) -> usize { + if oplog.shallow_since_vv().is_empty() { + pos + } else { + pos.min(tracker.len()) + } + } + + /// This should be called after calc_diff + /// + /// TODO: Refactor, this can be simplified + pub fn get_id_latest_pos(&self, id: ID) -> Option { + match &*self.mode { + RichtextCalcMode::Crdt { tracker, .. } => { + tracker.get_target_id_latest_index_at_new_version(id) + } + RichtextCalcMode::Linear { .. } => unreachable!(), + } + } + + fn apply_crdt_op_to_tracker( + oplog: &OpLog, + tracker: &mut RichtextTracker, + styles: &mut Vec<(StyleOp, usize)>, + op: RichOp, + ) { + match &op.raw_op().content { + crate::op::InnerContent::List(l) => match l { + InnerListOp::Insert { .. } | InnerListOp::Move { .. } | InnerListOp::Set { .. } => { + unreachable!() + } + InnerListOp::InsertText { + slice: _, + unicode_start, + unicode_len: len, + pos, + } => { + let pos = Self::shallow_clamped_tracker_pos(oplog, tracker, *pos as usize); + tracker.insert( + op.id_full(), + pos, + RichtextChunk::new_text(*unicode_start..*unicode_start + *len), + ); + } + InnerListOp::Delete(del) => { + let is_shallow = !oplog.shallow_since_vv().is_empty(); + let pos = del.start() as usize; + if is_shallow + && !Self::shallow_delete_range_matches_tracker( + tracker, + del.id_start, + pos, + del.atom_len(), + ) + { + let atom_len = del.atom_len(); + let mut segments = + tracker.active_segments_of_real_id_span(del.id_start, atom_len); + if segments.is_empty() { + return; + } + + segments.sort_unstable_by_key(|(_, pos, _)| *pos); + if del.is_reversed() { + for (target_id, pos, len) in segments.into_iter().rev() { + let target_offset = + (target_id.counter - del.id_start.counter) as usize; + debug_assert!(target_offset + len <= atom_len); + let op_offset = atom_len - target_offset - len; + tracker.delete( + op.id_start().inc(op_offset as Counter), + target_id, + pos, + len, + true, + ); + } + } else { + let mut deleted_before = 0; + for (target_id, pos, len) in segments { + let target_offset = + (target_id.counter - del.id_start.counter) as usize; + debug_assert!(pos >= deleted_before); + tracker.delete( + op.id_start().inc(target_offset as Counter), + target_id, + pos.saturating_sub(deleted_before), + len, + false, + ); + deleted_before += len; + } + } + + return; + } + + tracker.delete( + op.id_start(), + del.id_start, + pos, + del.atom_len(), + del.is_reversed(), + ); + } + InnerListOp::StyleStart { + start, + end, + key, + info, + value, + } => { + debug_assert!(start < end, "start: {}, end: {}", start, end); + let style_id = styles.len(); + styles.push(( + StyleOp { + lamport: op.lamport(), + peer: op.peer, + cnt: op.id_start().counter, + key: key.clone(), + value: value.clone(), + info: *info, + }, + *end as usize, + )); + let start = Self::shallow_clamped_tracker_pos(oplog, tracker, *start as usize); + tracker.insert( + op.id_full(), + start, + RichtextChunk::new_style_anchor(style_id as u32, AnchorType::Start), + ); + } + InnerListOp::StyleEnd => { + let id = op.id(); + if let Some(pos) = styles + .iter() + .rev() + .position(|(op, _pos)| op.peer == id.peer && op.cnt == id.counter - 1) + { + let style_id = styles.len() - pos - 1; + let (_start_op, end_pos) = &styles[style_id]; + tracker.insert( + op.id_full(), + // need to shift 1 because we insert the start style anchor before this pos + (*end_pos + 1).min(tracker.len()), + RichtextChunk::new_style_anchor(style_id as u32, AnchorType::End), + ); + } else { + let Some((style, end)) = Self::style_for_end_anchor(oplog, &op) else { + panic!("Unhandled checkout case") + }; + styles.push((style, end)); + let style_id = styles.len() - 1; + tracker.insert( + op.id_full(), + // need to shift 1 because we insert the start style anchor before this pos + (end + 1).min(tracker.len()), + RichtextChunk::new_style_anchor(style_id as u32, AnchorType::End), + ); + } + } + }, + _ => unreachable!(), + } + } + + #[cold] + #[inline(never)] + fn seed_tracker_from_shallow_root( + idx: ContainerIdx, + oplog: &OpLog, + tracker: &mut RichtextTracker, + styles: &mut Vec<(StyleOp, usize)>, + vv: &VersionVector, + ) { + if oplog.shallow_since_vv().is_empty() { + return; + } + + *tracker = RichtextTracker::new_empty(); + styles.clear(); + let shallow_root_vv = oplog + .dag + .frontiers_to_vv(oplog.shallow_since_frontiers()) + .unwrap_or_else(|| oplog.shallow_since_vv().to_vv()); + let seed_vv = if vv.includes_vv(&shallow_root_vv) { + &shallow_root_vv + } else { + vv + }; + + #[derive(Debug, Clone, Copy)] + struct SeedItem { + order: usize, + id: IdFull, + content: RichtextChunk, + } + + struct Fenwick { + tree: Vec, + } + + impl Fenwick { + fn new(len: usize) -> Self { + Self { + tree: vec![0; len + 1], + } + } + + fn add(&mut self, mut index: usize, value: usize) { + index += 1; + while index < self.tree.len() { + self.tree[index] += value; + index += index & index.wrapping_neg(); + } + } + + fn prefix_sum(&self, mut end: usize) -> usize { + let mut sum = 0; + while end > 0 { + sum += self.tree[end]; + end -= end & end.wrapping_neg(); + } + + sum + } + } + + let chunks = oplog.with_history_cache(|h| h.find_text_chunks_in_shallow_root_order(idx)); + let mut pos = 0; + let mut seed_items = Vec::new(); + let mut style_id_to_index = FxHashMap::default(); + for chunk in chunks { + match chunk { + RichtextStateChunk::Text(text) => { + let id = text.id_full(); + let vv_end = seed_vv.get(&id.peer).copied().unwrap_or(0); + if vv_end <= id.counter { + continue; + } + + let end = vv_end.min(id.counter + text.unicode_len() as Counter); + let len = (end - id.counter) as usize; + if len == 0 { + continue; + } + + seed_items.push(SeedItem { + order: seed_items.len(), + id, + content: RichtextChunk::new_unknown(len as u32), + }); + pos += len; + } + RichtextStateChunk::Style { style, anchor_type } => { + let id = match anchor_type { + AnchorType::Start => style.id(), + AnchorType::End => style.id().inc(1), + }; + if !seed_vv.includes_id(id) { + continue; + } + + let style_id = if let Some(id) = style_id_to_index.get(&style.id()) { + *id + } else { + let id = styles.len(); + styles.push((style.as_ref().clone(), pos)); + style_id_to_index.insert(style.id(), id); + id + }; + + if anchor_type == AnchorType::End { + styles[style_id].1 = pos.saturating_sub(1); + } + + seed_items.push(SeedItem { + order: seed_items.len(), + id: IdFull::new( + id.peer, + id.counter, + style.lamport + (id.counter - style.cnt) as u32, + ), + content: RichtextChunk::new_style_anchor(style_id as u32, anchor_type), + }); + pos += 1; + } + } + } + + seed_items.sort_unstable_by_key(|item| (item.id.peer, item.id.counter)); + let mut seen_end_by_peer: FxHashMap = FxHashMap::default(); + let mut normalized_items = Vec::with_capacity(seed_items.len()); + for mut item in seed_items { + let end = item.id.counter + item.content.len() as Counter; + let seen_end = seen_end_by_peer.entry(item.id.peer).or_default(); + if end <= *seen_end { + continue; + } + + if item.id.counter < *seen_end { + let skip = (*seen_end - item.id.counter) as usize; + item.id = item.id.inc(skip as Counter); + item.content = item.content.slice(skip..item.content.len()); + } + + *seen_end = end; + normalized_items.push(item); + } + + let seed_items = normalized_items; + let fenwick_len = seed_items + .iter() + .map(|item| item.order) + .max() + .map_or(0, |order| order + 1); + let mut inserted = Fenwick::new(fenwick_len); + for item in seed_items { + let pos = inserted.prefix_sum(item.order); + tracker.insert_seeded(pos, item.content, item.id); + inserted.add(item.order, item.content.len()); + } + + tracker.mark_shallow_root_applied(seed_vv); + } + + #[cold] + #[inline(never)] + fn build_full_crdt_tracker( + idx: ContainerIdx, + oplog: &OpLog, + vv: &VersionVector, + ) -> (RichtextTracker, Vec<(StyleOp, usize)>) { + struct RichtextRebuildVisitor<'a> { + oplog: &'a OpLog, + tracker: &'a mut RichtextTracker, + styles: &'a mut Vec<(StyleOp, usize)>, + } + + impl RebuildOpVisitor for RichtextRebuildVisitor<'_> { + fn visit(&mut self, vv: &VersionVector, op: RichOp<'_>) { + self.tracker.checkout(vv); + RichtextDiffCalculator::apply_crdt_op_to_tracker( + self.oplog, + self.tracker, + self.styles, + op, + ); + } + } + + let mut tracker = RichtextTracker::new_with_unknown(); + let mut styles = Vec::new(); + if !oplog.shallow_since_vv().is_empty() { + Self::seed_tracker_from_shallow_root(idx, oplog, &mut tracker, &mut styles, vv); + } + + let mut visitor = RichtextRebuildVisitor { + oplog, + tracker: &mut tracker, + styles: &mut styles, + }; + replay_container_ops_from_empty(idx, oplog, vv, &mut visitor); + + (tracker, styles) + } +} + +impl DiffCalculatorTrait for RichtextDiffCalculator { + fn start_tracking( + &mut self, + _oplog: &super::oplog::OpLog, + vv: &crate::VersionVector, + mode: DiffMode, + ) { + match mode { + DiffMode::Linear => { + *self.mode = RichtextCalcMode::Linear { + diff: DeltaRope::new(), + last_style_start: None, + }; + } + _ => { + if !matches!(&*self.mode, RichtextCalcMode::Crdt { .. }) { + unreachable!(); + } + } + } + + match &mut *self.mode { + RichtextCalcMode::Crdt { + tracker, + styles, + start_vv, + source_not_in_op_context, + shallow_root_seeded, + } => { + *source_not_in_op_context = false; + if !vv.includes_vv(start_vv) || !tracker.all_vv().includes_vv(vv) { + **tracker = RichtextTracker::new_with_unknown(); + styles.clear(); + *start_vv = vv.clone(); + *shallow_root_seeded = false; + } + + tracker.checkout(vv); + } + RichtextCalcMode::Linear { .. } => {} + } + } + + fn apply_change( + &mut self, + oplog: &super::oplog::OpLog, + op: crate::op::RichOp, + vv: Option<&crate::VersionVector>, + ) { + match &mut *self.mode { + RichtextCalcMode::Linear { + diff, + last_style_start, + } => match &op.raw_op().content { + crate::op::InnerContent::List(l) => match l { + InnerListOp::Insert { .. } + | InnerListOp::Move { .. } + | InnerListOp::Set { .. } => { + unreachable!() + } + InnerListOp::InsertText { + slice: _, + unicode_start, + unicode_len: len, + pos, + } => { + let s = oplog.arena.slice_by_unicode( + *unicode_start as usize..(*unicode_start + *len) as usize, + ); + diff.insert_value( + *pos as usize, + RichtextStateChunk::new_text(s, op.id_full()), + (), + ); + } + InnerListOp::Delete(del) => { + diff.delete(del.start() as usize, del.atom_len()); + } + InnerListOp::StyleStart { + start, + end, + key, + info, + value, + } => { + debug_assert!(start < end, "start: {}, end: {}", start, end); + let style_op = Arc::new(StyleOp { + lamport: op.lamport(), + peer: op.peer, + cnt: op.id_start().counter, + key: key.clone(), + value: value.clone(), + info: *info, + }); + + *last_style_start = Some((style_op.clone(), *end)); + diff.insert_value( + *start as usize, + RichtextStateChunk::new_style(style_op, AnchorType::Start), + (), + ); + } + InnerListOp::StyleEnd => { + let (style_op, pos) = match last_style_start.take() { + Some((style_op, pos)) => (style_op, pos), + None => { + let Some((style_op, pos)) = Self::style_for_end_anchor(oplog, &op) + else { + panic!("Unhandled checkout case") + }; + + (Arc::new(style_op), pos as u32) + } + }; + assert_eq!(style_op.peer, op.peer); + assert_eq!(style_op.cnt, op.id_start().counter - 1); + diff.insert_value( + pos as usize + 1, + RichtextStateChunk::new_style(style_op, AnchorType::End), + (), + ); + } + }, + _ => unreachable!(), + }, + RichtextCalcMode::Crdt { + tracker, + styles, + start_vv, + source_not_in_op_context: _, + shallow_root_seeded, + } => { + if !*shallow_root_seeded && !oplog.shallow_since_vv().is_empty() { + Self::seed_tracker_from_shallow_root( + op.raw_op().container, + oplog, + tracker, + styles, + start_vv, + ); + *shallow_root_seeded = true; + } + + if let Some(vv) = vv { + tracker.checkout(vv); + } + Self::apply_crdt_op_to_tracker(oplog, tracker, styles, op); + } + } + } + + fn calculate_diff( + &mut self, + idx: ContainerIdx, + oplog: &OpLog, + info: DiffCalcVersionInfo, + _: impl FnMut(&ContainerID), + ) -> (InternalDiff, DiffMode) { + fn push_tracker_chunk( + delta: &mut DeltaRope, + idx: ContainerIdx, + oplog: &OpLog, + styles: &[(StyleOp, usize)], + value: RichtextChunk, + id: ID, + lamport: Option, + ) { + match value.value() { + RichtextChunkValue::Text(text) => { + delta.push_insert( + RichtextStateChunk::Text( + // PERF: can be speedup by acquiring lock on arena + TextChunk::new( + oplog + .arena + .slice_by_unicode(text.start as usize..text.end as usize), + IdFull::new(id.peer, id.counter, lamport.unwrap()), + ), + ), + (), + ); + } + RichtextChunkValue::StyleAnchor { id, anchor_type } => { + delta.push_insert( + RichtextStateChunk::Style { + style: Arc::new(styles[id as usize].0.clone()), + anchor_type, + }, + (), + ); + } + RichtextChunkValue::Unknown(len) => { + // assert not unknown id + assert_ne!(id.peer, PeerID::MAX); + let mut id = id; + let mut acc_len = 0; + let end = id.counter + len as Counter; + let shallow_root = oplog.shallow_since_vv().get(&id.peer).copied().unwrap_or(0); + if id.counter < shallow_root { + // need to find the content between id.counter ~ target_end in gc state + let target_end = shallow_root.min(end); + oplog.with_history_cache(|h| { + let chunks = h.find_text_chunks_in( + idx, + IdSpan::new(id.peer, id.counter, target_end), + ); + for c in chunks { + acc_len += c.rle_len(); + delta.push_insert(c, ()); + } + }); + id.counter = shallow_root; + } + + if id.counter < end { + for rich_op in oplog.iter_ops(IdSpan::new(id.peer, id.counter, end)) { + acc_len += rich_op.content_len(); + let op = rich_op.op(); + let lamport = rich_op.lamport(); + let content = op.content.as_list().unwrap(); + match content { + InnerListOp::InsertText { slice, .. } => { + delta.push_insert( + RichtextStateChunk::Text(TextChunk::new( + slice.clone(), + IdFull::new(id.peer, op.counter, lamport), + )), + (), + ); + } + _ => unreachable!("{:?}", content), + } + } + } + + debug_assert_eq!(acc_len, len as usize); + } + RichtextChunkValue::MoveAnchor => unreachable!(), + } + } + + fn push_tracker_delta_item( + delta: &mut DeltaRope, + idx: ContainerIdx, + oplog: &OpLog, + styles: &[(StyleOp, usize)], + item: CrdtRopeDelta, + ) { + match item { + CrdtRopeDelta::Retain(len) => { + delta.push_retain(len, ()); + } + CrdtRopeDelta::Insert { + chunk: value, + id, + lamport, + } => push_tracker_chunk(delta, idx, oplog, styles, value, id, lamport), + CrdtRopeDelta::Delete(len) => { + delta.push_delete(len); + } + } + } + + match &mut *self.mode { + RichtextCalcMode::Linear { diff, .. } => ( + InternalDiff::RichtextRaw(std::mem::take(diff)), + DiffMode::Linear, + ), + RichtextCalcMode::Crdt { + tracker, + styles, + source_not_in_op_context, + .. + } => { + let (mut retreat, _) = info.from_vv.diff_iter(info.to_vv); + let has_retreat = retreat.next().is_some(); + let should_rebuild = has_retreat + || info.lca_vv != info.from_vv + || *source_not_in_op_context + || !oplog.shallow_since_vv().is_empty(); + if should_rebuild { + // Richtext diffs can start from a tracker that only knows the LCA state as + // unknown spans. Expressing a rollback or an import from `lca != from` as local + // edits can target the wrong visible text when the source state contains + // concurrent inserts or sliced ops. The same risk exists when an op is replayed + // from a dependency version that does not include the visible source state. + // Preserve correctness by replacing the visible source state with the target + // state reconstructed from CRDT ids. Shallow docs seed this tracker from the + // shallow-root state and replay only the retained suffix of history. + let mut merged = info.from_vv.clone(); + merged.merge(info.to_vv); + let (mut full_tracker, full_styles) = + Self::build_full_crdt_tracker(idx, oplog, &merged); + + let mut delta = DeltaRope::new(); + for item in full_tracker.diff(info.from_vv, info.to_vv) { + push_tracker_delta_item(&mut delta, idx, oplog, &full_styles, item); + } + **tracker = full_tracker; + *styles = full_styles; + + return (InternalDiff::RichtextRaw(delta), DiffMode::Checkout); + } + + let mut delta = DeltaRope::new(); + for item in tracker.diff(info.from_vv, info.to_vv) { + push_tracker_delta_item(&mut delta, idx, oplog, styles, item); + } + + (InternalDiff::RichtextRaw(delta), DiffMode::Checkout) + } + } + } + + fn finish_this_round(&mut self) { + match &mut *self.mode { + RichtextCalcMode::Crdt { .. } => {} + RichtextCalcMode::Linear { + diff, + last_style_start, + } => { + *diff = DeltaRope::new(); + last_style_start.take(); + } + } + } +} + +#[derive(Debug)] +pub(crate) struct MovableListDiffCalculator { + list: Box, + inner: Box, +} + +#[derive(Debug)] +struct MovableListInner { + changed_elements: FxHashMap, + move_id_to_elem_id: FxHashMap, + current_mode: DiffMode, +} + +impl DiffCalculatorTrait for MovableListDiffCalculator { + fn start_tracking(&mut self, _oplog: &OpLog, vv: &crate::VersionVector, mode: DiffMode) { + self.list.source_not_in_op_context = false; + if !vv.includes_vv(&self.list.start_vv) || !self.list.tracker.all_vv().includes_vv(vv) { + *self.list.tracker = RichtextTracker::new_with_unknown(); + self.list.start_vv = vv.clone(); + } + + self.list.tracker.checkout(vv); + self.inner.current_mode = mode; + } + + fn apply_change( + &mut self, + oplog: &OpLog, + op: crate::op::RichOp, + vv: Option<&crate::VersionVector>, + ) { + let InnerContent::List(l) = &op.raw_op().content else { + unreachable!() + }; + + // collect the elements that are moved, updated, or inserted + + // If it's checkout mode, we don't need to track the changes + // we only need the element ids + match l { + InnerListOp::Insert { slice, pos: _ } => { + let op_id = op.id_full().idlp(); + for i in 0..slice.atom_len() { + let id = op_id.inc(i as Counter); + let value = oplog.arena.get_value(slice.0.start as usize + i).unwrap(); + + self.inner.changed_elements.insert( + id.compact(), + ElementDelta { + pos: Some(id), + value: value.clone(), + value_updated: true, + value_id: Some(id), + }, + ); + } + } + InnerListOp::Delete(_) => {} + InnerListOp::Move { elem_id, .. } => { + let idlp = IdLp::new(op.peer, op.lamport()); + match self.inner.changed_elements.get_mut(&elem_id.compact()) { + Some(change) => { + if change.pos.is_some() && change.pos.as_ref().unwrap() > &idlp { + } else { + change.pos = Some(idlp); + } + } + None => { + self.inner.changed_elements.insert( + elem_id.compact(), + ElementDelta { + pos: Some(idlp), + value: LoroValue::Null, + value_updated: false, + value_id: None, + }, + ); + } + } + } + InnerListOp::Set { elem_id, value } => { + let idlp = IdLp::new(op.peer, op.lamport()); + match self.inner.changed_elements.get_mut(&elem_id.compact()) { + Some(change) => { + if change.value_id.is_some() && change.value_id.as_ref().unwrap() > &idlp { + } else { + change.value_id = Some(idlp); + change.value = value.clone(); + } + } + None => { + self.inner.changed_elements.insert( + elem_id.compact(), + ElementDelta { + pos: None, + value: value.clone(), + value_updated: true, + value_id: Some(idlp), + }, + ); + } + } + } + + InnerListOp::StyleStart { .. } => unreachable!(), + InnerListOp::StyleEnd => unreachable!(), + InnerListOp::InsertText { .. } => unreachable!(), + } + + let is_checkout = matches!(self.inner.current_mode, DiffMode::Checkout); + + { + // Apply change on the list items + if let Some(vv) = vv { + self.list.tracker.checkout(vv); + } + Self::apply_op_to_tracker( + &mut self.list.tracker, + &mut self.inner.move_id_to_elem_id, + oplog, + &op, + is_checkout, + ); + }; + } + + fn finish_this_round(&mut self) { + self.list.finish_this_round(); + } + + #[instrument(skip(self, oplog, on_new_container))] + fn calculate_diff( + &mut self, + idx: ContainerIdx, + oplog: &OpLog, + info: DiffCalcVersionInfo, + mut on_new_container: impl FnMut(&ContainerID), + ) -> (InternalDiff, DiffMode) { + let (mut retreat, _) = info.from_vv.diff_iter(info.to_vv); + let has_retreat = retreat.next().is_some(); + if has_retreat || info.lca_vv != info.from_vv || self.list.source_not_in_op_context { + let mut merged = info.from_vv.clone(); + merged.merge(info.to_vv); + self.rebuild_full_tracker(idx, oplog, &merged); + } + + let (InternalDiff::ListRaw(list_diff), diff_mode) = + self.list.calculate_diff(idx, oplog, info, |_| {}) + else { + unreachable!() + }; + + assert_eq!(diff_mode, DiffMode::Checkout); + let is_checkout = matches!( + self.inner.current_mode, + DiffMode::Checkout | DiffMode::Import + ); + let mut element_changes: FxHashMap = if is_checkout { + FxHashMap::default() + } else { + std::mem::take(&mut self.inner.changed_elements) + }; + + if is_checkout { + for id in self.inner.changed_elements.keys() { + element_changes.insert(*id, ElementDelta::placeholder()); + } + } + + let list_diff: Delta, ()> = Delta { + vec: list_diff + .iter() + .map(|x| match x { + &DeltaItem::Retain { retain, .. } => DeltaItem::Retain { + retain, + attributes: (), + }, + DeltaItem::Insert { insert, .. } => { + let len = insert.length(); + let id = insert.id; + let mut new_insert = SmallVec::with_capacity(len); + for i in 0..len { + let id = id.inc(i as i32); + let elem_id = + if let Some(e) = self.inner.move_id_to_elem_id.get(&id.id()) { + e.compact() + } else { + insert.elem_id.unwrap_or_else(|| id.idlp().compact()) + }; + if is_checkout { + // add the related element id + element_changes.insert(elem_id, ElementDelta::placeholder()); + } + new_insert.push(id); + } + + DeltaItem::Insert { + insert: new_insert, + attributes: (), + } + } + &DeltaItem::Delete { delete, .. } => DeltaItem::Delete { + delete, + attributes: (), + }, + }) + .collect(), + }; + + if is_checkout { + oplog.with_history_cache(|history_cache| { + let checkout_index = &history_cache.get_checkout_index().movable_list; + element_changes.retain(|id, change| { + let id = id.to_id(); + // It can be None if the target does not exist before the `to` version + // But we don't need to calc from, because the deletion is handled by the diff from list items + + // TODO: PERF: Provide the lamport of to version + let Some(pos) = checkout_index.last_pos(id, info.to_vv, Lamport::MAX, oplog) + else { + return false; + }; + // TODO: PERF: Provide the lamport of to version + let value = checkout_index + .last_value(id, info.to_vv, Lamport::MAX, oplog) + .unwrap(); + // TODO: PERF: Provide the lamport of to version + let old_pos = checkout_index.last_pos(id, info.from_vv, Lamport::MAX, oplog); + // TODO: PERF: Provide the lamport of to version + let old_value = + checkout_index.last_value(id, info.from_vv, Lamport::MAX, oplog); + if old_pos.is_none() && old_value.is_none() { + if let LoroValue::Container(c) = &value.value { + on_new_container(c); + } + *change = ElementDelta { + pos: Some(pos.idlp()), + value: value.value.clone(), + value_id: Some(IdLp::new(value.peer, value.lamport)), + value_updated: true, + }; + } else { + // TODO: PERF: can be filtered based on the list_diff and whether the pos/value are updated + *change = ElementDelta { + pos: Some(pos.idlp()), + value: value.value.clone(), + value_updated: old_value.unwrap().value != value.value, + value_id: Some(IdLp::new(value.peer, value.lamport)), + }; + } + + true + }); + }); + } + + let diff = MovableListInnerDelta { + list: list_diff, + elements: element_changes, + }; + + (InternalDiff::MovableList(diff), self.inner.current_mode) + } +} + +impl MovableListDiffCalculator { + fn new(_container: ContainerIdx) -> MovableListDiffCalculator { + MovableListDiffCalculator { + list: Default::default(), + inner: Box::new(MovableListInner { + changed_elements: Default::default(), + current_mode: DiffMode::Checkout, + move_id_to_elem_id: Default::default(), + }), + } + } + + fn apply_op_to_tracker( + tracker: &mut RichtextTracker, + move_id_to_elem_id: &mut FxHashMap, + oplog: &OpLog, + op: &RichOp<'_>, + is_checkout: bool, + ) { + let real_op = op.op(); + match &real_op.content { + InnerContent::List(l) => match l { + InnerListOp::Insert { .. } | InnerListOp::Delete(_) => { + ListDiffCalculator::apply_op_to_tracker(tracker, op); + } + InnerListOp::Move { from, elem_id, to } => { + move_id_to_elem_id.insert(op.id(), *elem_id); + if !tracker.current_vv().includes_id(op.id()) { + let last_pos = if is_checkout { + oplog.with_history_cache(|h| { + let list = &h.get_checkout_index().movable_list; + list.last_pos(*elem_id, tracker.current_vv(), Lamport::MAX, oplog) + .expect("moved element should have a visible source position") + .id() + }) + } else { + // In import/linear mode this id is only needed if the tracker is later + // checked out before the source version, which those modes do not do. + ID::new(PeerID::MAX - 2, 0) + }; + tracker.move_item(op.id_full(), last_pos, *from as usize, *to as usize); + } + } + InnerListOp::Set { .. } => {} + InnerListOp::InsertText { .. } + | InnerListOp::StyleStart { .. } + | InnerListOp::StyleEnd => unreachable!(), + }, + _ => unreachable!(), + } + } + + #[cold] + #[inline(never)] + fn rebuild_full_tracker(&mut self, idx: ContainerIdx, oplog: &OpLog, vv: &VersionVector) { + struct MovableListRebuildVisitor<'a> { + oplog: &'a OpLog, + tracker: &'a mut RichtextTracker, + move_id_to_elem_id: &'a mut FxHashMap, + } + + impl RebuildOpVisitor for MovableListRebuildVisitor<'_> { + fn visit(&mut self, vv: &VersionVector, op: RichOp<'_>) { + self.tracker.checkout(vv); + MovableListDiffCalculator::apply_op_to_tracker( + self.tracker, + self.move_id_to_elem_id, + self.oplog, + &op, + true, + ); + } + } + + let mut tracker = RichtextTracker::new_with_unknown(); + let mut move_id_to_elem_id = FxHashMap::default(); + let mut visitor = MovableListRebuildVisitor { + oplog, + tracker: &mut tracker, + move_id_to_elem_id: &mut move_id_to_elem_id, + }; + replay_container_ops_from_empty(idx, oplog, vv, &mut visitor); + + *self.list.tracker = tracker; + self.inner.move_id_to_elem_id = move_id_to_elem_id; + } +} + +#[test] +fn test_size() { + let text = RichtextDiffCalculator::new(); + let size = std::mem::size_of_val(&text); + assert!(size < 50, "RichtextDiffCalculator size: {}", size); + let list = MovableListDiffCalculator::new(ContainerIdx::from_index_and_type( + 0, + loro_common::ContainerType::MovableList, + )); + let size = std::mem::size_of_val(&list); + assert!(size < 50, "MovableListDiffCalculator size: {}", size); + let calc = ContainerDiffCalculator::Richtext(text); + let size = std::mem::size_of_val(&calc); + assert!(size < 50, "ContainerDiffCalculator size: {}", size); +} diff --git a/3rdparty/loro/crates/loro-internal/src/diff_calc/counter.rs b/3rdparty/loro/crates/loro-internal/src/diff_calc/counter.rs new file mode 100644 index 0000000000..bf0663b54d --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/diff_calc/counter.rs @@ -0,0 +1,63 @@ +use std::collections::BTreeMap; + +use loro_common::{ContainerID, ID}; + +use crate::{container::idx::ContainerIdx, event::InternalDiff, OpLog}; + +use super::{DiffCalcVersionInfo, DiffCalculatorTrait, DiffMode}; + +#[derive(Debug)] +pub(crate) struct CounterDiffCalculator { + ops: BTreeMap, +} + +impl CounterDiffCalculator { + pub(crate) fn new(_idx: ContainerIdx) -> Self { + Self { + ops: BTreeMap::new(), + } + } +} + +impl DiffCalculatorTrait for CounterDiffCalculator { + fn start_tracking(&mut self, _oplog: &OpLog, _vv: &crate::VersionVector, _mode: DiffMode) {} + + fn apply_change( + &mut self, + _oplog: &OpLog, + op: crate::op::RichOp, + _vv: Option<&crate::VersionVector>, + ) { + let id = op.id(); + self.ops.insert( + id, + *op.op().content.as_future().unwrap().as_counter().unwrap(), + ); + } + + fn finish_this_round(&mut self) {} + + fn calculate_diff( + &mut self, + _idx: ContainerIdx, + _oplog: &OpLog, + info: DiffCalcVersionInfo, + _on_new_container: impl FnMut(&ContainerID), + ) -> (InternalDiff, DiffMode) { + let mut diff = 0.; + let (b, a) = info.from_vv.diff_iter(info.to_vv); + + for sub in b { + for (_, c) in self.ops.range(sub.norm_id_start()..sub.norm_id_end()) { + diff -= c; + } + } + for sub in a { + for (_, c) in self.ops.range(sub.norm_id_start()..sub.norm_id_end()) { + diff += c; + } + } + + (InternalDiff::Counter(diff), DiffMode::Linear) + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/diff_calc/tree.rs b/3rdparty/loro/crates/loro-internal/src/diff_calc/tree.rs new file mode 100644 index 0000000000..58c410316d --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/diff_calc/tree.rs @@ -0,0 +1,654 @@ +use std::{collections::BTreeSet, sync::Arc}; + +use fractional_index::FractionalIndex; +use itertools::Itertools; +use loro_common::{ContainerID, IdFull, IdLp, Lamport, PeerID, TreeID, ID}; +use rustc_hash::FxHashMap; + +use crate::{ + container::{idx::ContainerIdx, tree::tree_op::TreeOp}, + dag::DagUtils, + delta::{TreeDelta, TreeDeltaItem, TreeInternalDiff}, + event::InternalDiff, + state::TreeParentId, + version::Frontiers, + OpLog, VersionVector, +}; + +use super::{DiffCalcVersionInfo, DiffCalculatorTrait, DiffMode}; + +#[derive(Debug)] +pub(crate) struct TreeDiffCalculator { + container: ContainerIdx, + mode: TreeDiffCalculatorMode, +} + +#[derive(Debug)] +enum TreeDiffCalculatorMode { + Crdt, + Linear(TreeDelta), + ImportGreaterUpdates(TreeDelta), +} + +impl DiffCalculatorTrait for TreeDiffCalculator { + fn start_tracking(&mut self, _oplog: &OpLog, _vv: &crate::VersionVector, mode: DiffMode) { + match mode { + DiffMode::Checkout => { + self.mode = TreeDiffCalculatorMode::Crdt; + } + DiffMode::Import => { + self.mode = TreeDiffCalculatorMode::Crdt; + } + DiffMode::ImportGreaterUpdates => { + self.mode = TreeDiffCalculatorMode::ImportGreaterUpdates(TreeDelta::default()); + } + DiffMode::Linear => { + self.mode = TreeDiffCalculatorMode::Linear(TreeDelta::default()); + } + } + } + + fn apply_change( + &mut self, + _oplog: &OpLog, + op: crate::op::RichOp, + _vv: Option<&crate::VersionVector>, + ) { + match &mut self.mode { + TreeDiffCalculatorMode::Crdt => {} + TreeDiffCalculatorMode::Linear(ref mut delta) + | TreeDiffCalculatorMode::ImportGreaterUpdates(ref mut delta) => { + let id_full = op.id_full(); + let op = op.op(); + let content = op.content.as_tree().unwrap(); + + let item: TreeDeltaItem = match &**content { + crate::container::tree::tree_op::TreeOp::Create { + target, + parent, + position, + } => TreeDeltaItem { + target: *target, + action: TreeInternalDiff::Create { + parent: (*parent).into(), + position: position.clone(), + }, + last_effective_move_op_id: id_full, + }, + crate::container::tree::tree_op::TreeOp::Move { + target, + parent, + position, + } => TreeDeltaItem { + target: *target, + action: TreeInternalDiff::Move { + parent: (*parent).into(), + position: position.clone(), + }, + last_effective_move_op_id: id_full, + }, + crate::container::tree::tree_op::TreeOp::Delete { target } => TreeDeltaItem { + target: *target, + action: TreeInternalDiff::Delete { + parent: TreeParentId::Deleted, + position: None, + }, + last_effective_move_op_id: id_full, + }, + }; + + delta.diff.push(item); + } + } + } + + fn finish_this_round(&mut self) { + self.mode = TreeDiffCalculatorMode::Crdt; + } + + fn calculate_diff( + &mut self, + _idx: ContainerIdx, + oplog: &OpLog, + info: DiffCalcVersionInfo, + mut on_new_container: impl FnMut(&ContainerID), + ) -> (InternalDiff, DiffMode) { + match &mut self.mode { + TreeDiffCalculatorMode::Crdt => { + let diff = self.diff(oplog, info); + diff.diff.iter().for_each(|d| { + // the metadata could be modified before, so (re)create a node need emit the map container diffs + // `Create` here is because maybe in a diff calc uncreate and then create back + if matches!(d.action, TreeInternalDiff::Create { .. }) { + on_new_container(&d.target.associated_meta_container()) + } + }); + + (InternalDiff::Tree(diff), DiffMode::Checkout) + } + TreeDiffCalculatorMode::Linear(ans) => { + (InternalDiff::Tree(std::mem::take(ans)), DiffMode::Linear) + } + TreeDiffCalculatorMode::ImportGreaterUpdates(ans) => { + let mut ans = std::mem::take(ans); + ans.diff.sort_unstable_by(|a, b| { + a.last_effective_move_op_id + .lamport + .cmp(&b.last_effective_move_op_id.lamport) + .then_with(|| { + a.last_effective_move_op_id + .peer + .cmp(&b.last_effective_move_op_id.peer) + }) + }); + (InternalDiff::Tree(ans), DiffMode::ImportGreaterUpdates) + } + } + } +} + +impl TreeDiffCalculator { + pub(crate) fn new(container: ContainerIdx) -> Self { + Self { + container, + mode: TreeDiffCalculatorMode::Crdt, + } + } + + fn diff(&mut self, oplog: &OpLog, info: DiffCalcVersionInfo) -> TreeDelta { + self.checkout(info.from_vv, info.from_frontiers, oplog); + self.checkout_diff(info, oplog) + } + + fn checkout(&mut self, to: &VersionVector, to_frontiers: &Frontiers, oplog: &OpLog) { + oplog.with_history_cache(|h| { + let mark = h.ensure_importing_caches_exist(); + let tree_ops = h.get_tree(&self.container, mark).unwrap(); + let mut tree_cache = tree_ops.tree().lock(); + let s = format!("checkout current {:?} to {:?}", &tree_cache.current_vv, &to); + let s = tracing::span!(tracing::Level::INFO, "checkout", s = s); + let _e = s.enter(); + if to == &tree_cache.current_vv { + return; + } + let min_lamport = self.get_min_lamport_by_frontiers(to_frontiers, oplog); + // retreat + let mut retreat_ops = vec![]; + for (_target, ops) in tree_cache.tree.iter() { + for op in ops.iter().rev() { + if op.id.lamport < min_lamport { + break; + } + if !to.includes_id(op.id.id()) { + retreat_ops.push(op.clone()); + } + } + } + + for op in retreat_ops { + tree_cache.retreat_op(&op); + } + + // forward and apply + let max_lamport = self.get_max_lamport_by_frontiers(to_frontiers, oplog); + let mut forward_ops = vec![]; + let group = h + .get_importing_cache(&self.container, mark) + .unwrap() + .as_tree() + .unwrap(); + for (idlp, op) in group.ops().range( + IdLp { + lamport: 0, + peer: 0, + }..=IdLp { + lamport: max_lamport, + peer: PeerID::MAX, + }, + ) { + if !tree_cache + .current_vv + .includes_id(ID::new(idlp.peer, op.counter)) + && to.includes_id(ID::new(idlp.peer, op.counter)) + { + forward_ops.push((IdFull::new(idlp.peer, op.counter, idlp.lamport), op)); + } + } + + for (id, op) in forward_ops { + let op = MoveLamportAndID { + id, + op: op.value.clone(), + effected: false, + }; + tree_cache.apply(op); + } + tree_cache.current_vv = to.clone(); + }); + } + + fn checkout_diff(&mut self, info: DiffCalcVersionInfo, oplog: &OpLog) -> TreeDelta { + oplog.with_history_cache(|h| { + let mark = h.ensure_importing_caches_exist(); + let tree_ops = h.get_tree(&self.container, mark).unwrap(); + let mut tree_cache = tree_ops.tree().lock(); + let mut parent_to_children_cache = + TreeParentToChildrenCache::init_from_tree_cache(&tree_cache); + let s = tracing::span!(tracing::Level::INFO, "checkout_diff"); + let _e = s.enter(); + let to_frontiers = info.to_frontiers; + let from_frontiers = info.from_frontiers; + let (common_ancestors, _mode) = + oplog.dag.find_common_ancestor(from_frontiers, to_frontiers); + let lca_vv = oplog.dag.frontiers_to_vv(&common_ancestors).unwrap(); + let lca_frontiers = common_ancestors; + let to_max_lamport = self.get_max_lamport_by_frontiers(to_frontiers, oplog); + let lca_min_lamport = self.get_min_lamport_by_frontiers(&lca_frontiers, oplog); + + // retreat for diff + let mut diffs = vec![]; + + if !(tree_cache.current_vv == lca_vv && &lca_vv == info.from_vv) { + let mut retreat_ops = vec![]; + for (_target, ops) in tree_cache.tree.iter() { + for op in ops.iter().rev() { + if op.id.lamport < lca_min_lamport { + break; + } + if !lca_vv.includes_id(op.id.id()) { + retreat_ops.push(op.clone()); + } + } + } + + for op in retreat_ops.into_iter().sorted().rev() { + tree_cache.retreat_op(&op); + let (old_parent, position, last_effective_move_op_id) = + tree_cache.get_parent_with_id(op.op.target()); + if op.effected { + // we need to know whether old_parent is deleted + let is_parent_deleted = tree_cache.is_parent_deleted(op.op.parent_id()); + let is_old_parent_deleted = tree_cache.is_parent_deleted(old_parent); + if op.op.target().id() == op.id.id() { + assert_eq!( + old_parent, + TreeParentId::Unexist, + "old_parent = {:?} instead", + &old_parent + ); + } + parent_to_children_cache.record_change( + op.op.target(), + op.op.parent_id(), + old_parent, + ); + let this_diff = TreeDeltaItem::new( + op.op.target(), + old_parent, + op.op.parent_id(), + last_effective_move_op_id, + is_old_parent_deleted, + is_parent_deleted, + position, + ); + let is_create = matches!(this_diff.action, TreeInternalDiff::Create { .. }); + diffs.push(this_diff); + if is_create { + let mut s = vec![op.op.target()]; + while let Some(t) = s.pop() { + let children = tree_cache.get_children_with_id( + TreeParentId::Node(t), + &parent_to_children_cache, + ); + children.iter().for_each(|c| { + diffs.push(TreeDeltaItem { + target: c.0, + action: TreeInternalDiff::Create { + parent: TreeParentId::Node(t), + position: c.1.clone().unwrap(), + }, + last_effective_move_op_id: c.2, + }) + }); + s.extend(children.iter().map(|c| c.0)); + } + } + } + } + } + tree_cache.current_vv = lca_vv; + // forward + let group = h + .get_importing_cache(&self.container, mark) + .unwrap() + .as_tree() + .unwrap(); + for (idlp, op) in group.ops().range( + IdLp { + lamport: lca_min_lamport, + peer: 0, + }..=IdLp { + lamport: to_max_lamport, + peer: PeerID::MAX, + }, + ) { + let id = ID::new(idlp.peer, op.counter); + if !tree_cache.current_vv.includes_id(id) && info.to_vv.includes_id(id) { + let op = MoveLamportAndID { + id: IdFull { + peer: id.peer, + lamport: idlp.lamport, + counter: id.counter, + }, + op: op.value.clone(), + effected: false, + }; + let (old_parent, _position, _id) = + tree_cache.get_parent_with_id(op.op.target()); + let is_parent_deleted = tree_cache.is_parent_deleted(op.op.parent_id()); + let is_old_parent_deleted = tree_cache.is_parent_deleted(old_parent); + let effected = tree_cache.apply(op.clone()); + if effected { + let this_diff = TreeDeltaItem::new( + op.op.target(), + op.op.parent_id(), + old_parent, + op.id_full(), + is_parent_deleted, + is_old_parent_deleted, + op.op.fractional_index(), + ); + parent_to_children_cache.record_change( + op.op.target(), + old_parent, + op.op.parent_id(), + ); + let is_create = matches!(this_diff.action, TreeInternalDiff::Create { .. }); + diffs.push(this_diff); + if is_create { + // TODO: per + let mut s = vec![op.op.target()]; + while let Some(t) = s.pop() { + let children = tree_cache.get_children_with_id( + TreeParentId::Node(t), + &parent_to_children_cache, + ); + children.iter().for_each(|c| { + diffs.push(TreeDeltaItem { + target: c.0, + action: TreeInternalDiff::Create { + parent: TreeParentId::Node(t), + position: c.1.clone().unwrap(), + }, + last_effective_move_op_id: c.2, + }) + }); + s.extend(children.iter().map(|x| x.0)); + } + } + } + } + } + + tree_cache.current_vv = info.to_vv.clone(); + TreeDelta { diff: diffs } + }) + } + + fn get_min_lamport_by_frontiers(&self, frontiers: &Frontiers, oplog: &OpLog) -> Lamport { + frontiers + .iter() + .map(|id| oplog.get_min_lamport_at(id)) + .min() + .unwrap_or(0) + } + + fn get_max_lamport_by_frontiers(&self, frontiers: &Frontiers, oplog: &OpLog) -> Lamport { + frontiers + .iter() + .map(|id| oplog.get_max_lamport_at(id)) + .max() + .unwrap_or(Lamport::MAX) + } +} + +/// All information of an operation for diff calculating of movable tree. +#[derive(Debug, Clone)] +pub struct MoveLamportAndID { + pub(crate) id: IdFull, + pub(crate) op: Arc, + /// Whether this action is applied in the current version. + /// If this action will cause a circular reference, then this action will not be applied. + pub(crate) effected: bool, +} + +impl MoveLamportAndID { + fn id_full(&self) -> IdFull { + self.id + } +} + +impl PartialEq for MoveLamportAndID { + fn eq(&self, other: &Self) -> bool { + self.id == other.id + } +} + +impl Eq for MoveLamportAndID {} + +impl PartialOrd for MoveLamportAndID { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for MoveLamportAndID { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.id + .lamport + .cmp(&other.id.lamport) + .then_with(|| self.id.peer.cmp(&other.id.peer)) + } +} + +#[derive(Clone, Default)] +pub(crate) struct TreeCacheForDiff { + tree: FxHashMap>, + current_vv: VersionVector, +} + +impl std::fmt::Debug for TreeCacheForDiff { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + writeln!(f, "TreeCacheForDiff {{ tree: ")?; + for (id, ops) in self.tree.iter() { + writeln!(f, " {} -> {:?}", id, ops)?; + } + writeln!(f, " current_vv: {:?}", self.current_vv)?; + Ok(()) + } +} + +impl TreeCacheForDiff { + fn retreat_op(&mut self, op: &MoveLamportAndID) { + self.tree.get_mut(&op.op.target()).unwrap().remove(op); + self.current_vv.set_end(op.id.id()); + } + + fn is_ancestor_of(&self, maybe_ancestor: &TreeID, node_id: &TreeParentId) -> bool { + if !self.tree.contains_key(maybe_ancestor) { + return false; + } + if let TreeParentId::Node(id) = node_id { + if id == maybe_ancestor { + return true; + } + } + match node_id { + TreeParentId::Node(id) => { + let (parent, _, _) = &self.get_parent_with_id(*id); + if parent == node_id { + panic!("is_ancestor_of loop") + } + self.is_ancestor_of(maybe_ancestor, parent) + } + TreeParentId::Deleted | TreeParentId::Root => false, + TreeParentId::Unexist => false, + } + } + + fn apply(&mut self, mut node: MoveLamportAndID) -> bool { + let mut effected = true; + if self.is_ancestor_of(&node.op.target(), &node.op.parent_id()) { + effected = false; + } + node.effected = effected; + self.current_vv.set_last(node.id.id()); + self.tree.entry(node.op.target()).or_default().insert(node); + effected + } + + pub(crate) fn init_tree_with_shallow_root_version(&mut self, nodes: Vec) { + if nodes.is_empty() { + return; + } + + debug_assert!(self.tree.is_empty()); + for node in nodes.into_iter() { + self.current_vv.extend_to_include_last_id(node.id.id()); + self.current_vv + .extend_to_include_last_id(node.op.target().id()); + self.tree.entry(node.op.target()).or_default().insert(node); + } + } + + fn is_parent_deleted(&self, parent: TreeParentId) -> bool { + match parent { + TreeParentId::Deleted => true, + TreeParentId::Node(id) => self.is_parent_deleted(self.get_parent_with_id(id).0), + TreeParentId::Root => false, + TreeParentId::Unexist => false, + } + } + + /// get the parent of the first effected op and its id + fn get_parent_with_id( + &self, + tree_id: TreeID, + ) -> (TreeParentId, Option, IdFull) { + let mut ans = (TreeParentId::Unexist, None, IdFull::NONE_ID); + if let Some(cache) = self.tree.get(&tree_id) { + for op in cache.iter().rev() { + if op.effected { + ans = ( + op.op.parent_id(), + op.op.fractional_index().clone(), + op.id_full(), + ); + break; + } + } + } + ans + } + + /// get the parent of the last effected op + fn get_last_effective_move(&self, tree_id: TreeID) -> Option<&MoveLamportAndID> { + if TreeID::is_deleted_root(&tree_id) { + return None; + } + + let mut ans = None; + if let Some(set) = self.tree.get(&tree_id) { + for op in set.iter().rev() { + if op.effected { + ans = Some(op); + break; + } + } + } + + ans + } + + fn get_children_with_id( + &self, + parent: TreeParentId, + cache: &TreeParentToChildrenCache, + ) -> Vec<(TreeID, Option, IdFull)> { + let Some(children_ids) = cache.get_children(parent) else { + return vec![]; + }; + let mut ans = Vec::with_capacity(children_ids.len()); + for child in children_ids.iter() { + let Some(op) = self.get_last_effective_move(*child) else { + panic!("child {:?} has no last effective move", child); + }; + + assert_eq!(op.op.parent_id(), parent); + ans.push((*child, op.op.fractional_index().clone(), op.id_full())); + } + // The children should be sorted by the position. + // If the fractional index is the same, then sort by the lamport and peer. + ans.sort_by(|a, b| { + a.1.cmp(&b.1) + .then(a.2.lamport.cmp(&b.2.lamport).then(a.2.peer.cmp(&b.2.peer))) + }); + ans + } +} + +#[derive(Debug)] +struct TreeParentToChildrenCache { + cache: FxHashMap>, +} + +impl TreeParentToChildrenCache { + fn get_children(&self, parent: TreeParentId) -> Option<&BTreeSet> { + self.cache.get(&parent) + } + + fn init_from_tree_cache(tree_cache: &TreeCacheForDiff) -> Self { + let mut cache = Self { + cache: FxHashMap::default(), + }; + for (tree_id, _) in tree_cache.tree.iter() { + let Some(op) = tree_cache.get_last_effective_move(*tree_id) else { + continue; + }; + + cache + .cache + .entry(op.op.parent_id()) + .or_default() + .insert(op.op.target()); + } + cache + } + + fn record_change( + &mut self, + target: TreeID, + old_parent: TreeParentId, + new_parent: TreeParentId, + ) { + if !old_parent.is_unexist() { + let removed = if let Some(children) = self.cache.get_mut(&old_parent) { + children.remove(&target) + } else { + false + }; + + if !removed { + let removed = self + .cache + .values_mut() + .any(|children| children.remove(&target)); + assert!( + removed, + "target {target:?} should be present in TreeParentToChildrenCache before moving from {old_parent:?} to {new_parent:?}", + ); + } + } + self.cache.entry(new_parent).or_default().insert(target); + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/diff_calc/unknown.rs b/3rdparty/loro/crates/loro-internal/src/diff_calc/unknown.rs new file mode 100644 index 0000000000..25a6fbff38 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/diff_calc/unknown.rs @@ -0,0 +1,32 @@ +use loro_common::ContainerID; + +use crate::{container::idx::ContainerIdx, event::InternalDiff, OpLog}; + +use super::{DiffCalcVersionInfo, DiffCalculatorTrait, DiffMode}; + +#[derive(Debug, Default)] +pub struct UnknownDiffCalculator; + +impl DiffCalculatorTrait for UnknownDiffCalculator { + fn start_tracking(&mut self, _oplog: &OpLog, _vv: &crate::VersionVector, _mode: DiffMode) {} + + fn apply_change( + &mut self, + _oplog: &OpLog, + _op: crate::op::RichOp, + _vv: Option<&crate::VersionVector>, + ) { + } + + fn finish_this_round(&mut self) {} + + fn calculate_diff( + &mut self, + _idx: ContainerIdx, + _oplog: &OpLog, + _info: DiffCalcVersionInfo, + _on_new_container: impl FnMut(&ContainerID), + ) -> (InternalDiff, DiffMode) { + (InternalDiff::Unknown, DiffMode::Checkout) + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/encoding.rs b/3rdparty/loro/crates/loro-internal/src/encoding.rs new file mode 100644 index 0000000000..c234fb950a --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/encoding.rs @@ -0,0 +1,598 @@ +pub(crate) mod arena; +pub(crate) mod fast_snapshot; +pub(crate) mod json_schema; +mod outdated_encode_reordered; +mod shallow_snapshot; +pub(crate) mod value; +pub(crate) mod value_register; +pub(crate) use outdated_encode_reordered::{ + decode_op, encode_op, get_op_prop, EncodedDeleteStartId, IterableEncodedDeleteStartId, +}; +use outdated_encode_reordered::{import_changes_to_oplog, ImportChangesResult}; +pub(crate) use value::OwnedValue; + +use crate::change::Change; +use crate::version::{Frontiers, VersionRange}; +use crate::LoroDoc; +use crate::{oplog::OpLog, LoroError, VersionVector}; +use loro_common::{HasIdSpan, IdSpan, InternalString, LoroEncodeError, LoroResult, ID}; +use num_traits::{FromPrimitive, ToPrimitive}; +use std::borrow::Cow; + +/// The mode of the export. +/// +/// Loro CRDT internally consists of two parts: document history and current document state. +/// The export modes offer various options to meet different requirements. +/// +/// - CRDT property: Documents maintain consistent states when they receive the same set of updates. +/// - In real-time collaboration, peers typically only need to synchronize updates +/// (operations/history) to achieve consistency. +/// +/// ## Update Export +/// +/// - Exports only the history part, containing multiple operations. +/// - Suitable for real-time collaboration scenarios where peers only need to synchronize updates. +/// +/// ## Snapshot Export +/// +/// ### Default Snapshot +/// +/// - Includes complete history and current full state. +/// +/// ### Shallow Snapshot +/// +/// - Contains the complete current state. +/// - Retains partial history starting from a specified version. +/// +/// ### State-only Snapshot +/// +/// - Exports the state of the target version. +/// - Includes a minimal set of operation history. +#[non_exhaustive] +#[derive(Debug, Clone)] +pub enum ExportMode<'a> { + /// It contains the full history and the current state of the document. + Snapshot, + /// It contains the history since the `from` version vector. + Updates { from: Cow<'a, VersionVector> }, + /// This mode exports the history in the specified range. + UpdatesInRange { spans: Cow<'a, [IdSpan]> }, + /// The shallow snapshot only contains the history since the target frontiers + ShallowSnapshot(Cow<'a, Frontiers>), + /// The state only snapshot exports the state of the target version + /// with a minimal set of history (a few ops). + /// + /// It's a shallow snapshot with depth=1 at the target version. + /// If the target version is None, it will use the latest version as the target version. + StateOnly(Option>), + /// The snapshot at the specified frontiers. It contains the full history + /// till the target frontiers and the state at the target frontiers. + SnapshotAt { version: Cow<'a, Frontiers> }, +} + +impl<'a> ExportMode<'a> { + /// It contains the full history and the current state of the document. + pub fn snapshot() -> Self { + ExportMode::Snapshot + } + + /// It contains the history since the `from` version vector. + pub fn updates(from: &'a VersionVector) -> Self { + ExportMode::Updates { + from: Cow::Borrowed(from), + } + } + + /// It contains the history since the `from` version vector. + pub fn updates_owned(from: VersionVector) -> Self { + ExportMode::Updates { + from: Cow::Owned(from), + } + } + + /// It contains all the history of the document. + pub fn all_updates() -> Self { + ExportMode::Updates { + from: Cow::Owned(Default::default()), + } + } + + /// This mode exports the history in the specified range. + pub fn updates_in_range(spans: impl Into>) -> Self { + ExportMode::UpdatesInRange { + spans: spans.into(), + } + } + + /// The shallow snapshot only contains the history since the target frontiers. + pub fn shallow_snapshot(frontiers: &'a Frontiers) -> Self { + ExportMode::ShallowSnapshot(Cow::Borrowed(frontiers)) + } + + /// The shallow snapshot only contains the history since the target frontiers. + pub fn shallow_snapshot_owned(frontiers: Frontiers) -> Self { + ExportMode::ShallowSnapshot(Cow::Owned(frontiers)) + } + + /// The shallow snapshot only contains the history since the target frontiers. + pub fn shallow_snapshot_since(id: ID) -> Self { + let frontiers = Frontiers::from_id(id); + ExportMode::ShallowSnapshot(Cow::Owned(frontiers)) + } + + /// The state only snapshot exports the state of the target version + /// with a minimal set of history (a few ops). + /// + /// It's a shallow snapshot with depth=1 at the target version. + /// If the target version is None, it will use the latest version as the target version. + pub fn state_only(frontiers: Option<&'a Frontiers>) -> Self { + ExportMode::StateOnly(frontiers.map(Cow::Borrowed)) + } + + /// The snapshot at the specified frontiers. It contains the full history + /// till the target frontiers and the state at the target frontiers. + pub fn snapshot_at(frontiers: &'a Frontiers) -> Self { + ExportMode::SnapshotAt { + version: Cow::Borrowed(frontiers), + } + } + + /// This mode exports the history within the specified version vector. + pub fn updates_till(vv: &VersionVector) -> ExportMode<'static> { + let mut spans = Vec::with_capacity(vv.len()); + for (peer, counter) in vv.iter() { + if *counter > 0 { + spans.push(IdSpan::new(*peer, 0, *counter)); + } + } + + ExportMode::UpdatesInRange { + spans: Cow::Owned(spans), + } + } +} + +const MAGIC_BYTES: [u8; 4] = *b"loro"; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum EncodeMode { + // This is a config option, it won't be used in encoding. + Auto = 255, + OutdatedRle = 1, + OutdatedSnapshot = 2, + FastSnapshot = 3, + FastUpdates = 4, +} + +impl num_traits::FromPrimitive for EncodeMode { + #[allow(trivial_numeric_casts)] + #[inline] + fn from_i64(n: i64) -> Option { + match n { + n if n == EncodeMode::Auto as i64 => Some(EncodeMode::Auto), + n if n == EncodeMode::OutdatedRle as i64 => Some(EncodeMode::OutdatedRle), + n if n == EncodeMode::OutdatedSnapshot as i64 => Some(EncodeMode::OutdatedSnapshot), + n if n == EncodeMode::FastSnapshot as i64 => Some(EncodeMode::FastSnapshot), + n if n == EncodeMode::FastUpdates as i64 => Some(EncodeMode::FastUpdates), + _ => None, + } + } + #[inline] + fn from_u64(n: u64) -> Option { + Self::from_i64(n as i64) + } +} + +impl num_traits::ToPrimitive for EncodeMode { + #[inline] + #[allow(trivial_numeric_casts)] + fn to_i64(&self) -> Option { + Some(match *self { + EncodeMode::Auto => EncodeMode::Auto as i64, + EncodeMode::OutdatedRle => EncodeMode::OutdatedRle as i64, + EncodeMode::OutdatedSnapshot => EncodeMode::OutdatedSnapshot as i64, + EncodeMode::FastSnapshot => EncodeMode::FastSnapshot as i64, + EncodeMode::FastUpdates => EncodeMode::FastUpdates as i64, + }) + } + #[inline] + fn to_u64(&self) -> Option { + self.to_i64().map(|x| x as u64) + } +} + +impl EncodeMode { + pub fn to_bytes(self) -> [u8; 2] { + let value = self.to_u16().unwrap(); + value.to_be_bytes() + } + + pub fn is_snapshot(self) -> bool { + matches!( + self, + EncodeMode::OutdatedSnapshot | EncodeMode::FastSnapshot + ) + } +} + +impl TryFrom<[u8; 2]> for EncodeMode { + type Error = LoroError; + + fn try_from(value: [u8; 2]) -> Result { + let value = u16::from_be_bytes(value); + Self::from_u16(value).ok_or(LoroError::IncompatibleFutureEncodingError(value as usize)) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct ImportStatus { + pub success: VersionRange, + pub pending: Option, +} + +pub(crate) fn decode_oplog( + oplog: &mut OpLog, + parsed: ParsedHeaderAndBody, +) -> Result { + let changes = decode_oplog_changes(oplog, parsed)?; + let result = apply_decoded_changes_to_oplog(oplog, changes); + if result.has_deps_before_shallow_root { + return Err(LoroError::ImportUpdatesThatDependsOnOutdatedVersion); + } + + Ok(result.status) +} + +pub(crate) fn decode_oplog_changes( + oplog: &mut OpLog, + parsed: ParsedHeaderAndBody, +) -> Result, LoroError> { + let ParsedHeaderAndBody { mode, body, .. } = parsed; + match mode { + EncodeMode::OutdatedRle | EncodeMode::OutdatedSnapshot => { + Err(LoroError::ImportUnsupportedEncodingMode) + } + EncodeMode::FastSnapshot => fast_snapshot::decode_oplog(oplog, body), + EncodeMode::FastUpdates => fast_snapshot::decode_updates(oplog, body.to_vec().into()), + EncodeMode::Auto => unreachable!(), + } +} + +pub(crate) struct ApplyDecodedChangesResult { + pub status: ImportStatus, + pub has_deps_before_shallow_root: bool, +} + +pub(crate) fn apply_decoded_changes_to_oplog( + oplog: &mut OpLog, + changes: Vec, +) -> ApplyDecodedChangesResult { + let ImportChangesResult { + mut imported, + latest_ids, + pending_changes, + changes_that_have_deps_before_shallow_root, + } = import_changes_to_oplog(changes, oplog); + + let mut pending = VersionRange::default(); + pending_changes.iter().for_each(|c| { + pending.extends_to_include_id_span(c.id_span()); + }); + // TODO: PERF: should we use hashmap to filter latest_ids with the same peer first? + oplog.try_apply_pending(latest_ids, Some(&mut imported)); + oplog + .import_unknown_lamport_pending_changes(pending_changes) + .expect("importing unknown-lamport pending changes is infallible"); + ApplyDecodedChangesResult { + status: ImportStatus { + success: imported, + pending: (!pending.is_empty()).then_some(pending), + }, + has_deps_before_shallow_root: !changes_that_have_deps_before_shallow_root.is_empty(), + } +} + +pub(crate) struct ParsedHeaderAndBody<'a> { + pub checksum: [u8; 16], + pub checksum_body: &'a [u8], + pub mode: EncodeMode, + pub body: &'a [u8], +} + +const XXH_SEED: u32 = u32::from_le_bytes(*b"LORO"); +impl ParsedHeaderAndBody<'_> { + /// Return if the checksum is correct. + fn check_checksum(&self) -> LoroResult<()> { + match self.mode { + EncodeMode::OutdatedRle | EncodeMode::OutdatedSnapshot => { + if md5::compute(self.checksum_body).0 != self.checksum { + return Err(LoroError::DecodeChecksumMismatchError); + } + } + EncodeMode::FastSnapshot | EncodeMode::FastUpdates => { + let mut expected_bytes = [0; 4]; + expected_bytes.copy_from_slice(&self.checksum[12..16]); + let expected = u32::from_le_bytes(expected_bytes); + if xxhash_rust::xxh32::xxh32(self.checksum_body, XXH_SEED) != expected { + return Err(LoroError::DecodeChecksumMismatchError); + } + } + EncodeMode::Auto => { + return Err(LoroError::DecodeError( + "Invalid import mode `Auto` in encoded blob" + .to_string() + .into_boxed_str(), + )); + } + } + + Ok(()) + } +} + +const MIN_HEADER_SIZE: usize = 22; +pub(crate) fn parse_header_and_body( + bytes: &[u8], + check_checksum: bool, +) -> Result, LoroError> { + let reader = &bytes; + if bytes.len() < MIN_HEADER_SIZE { + return Err(LoroError::DecodeError("Invalid import data".into())); + } + + let (magic_bytes, reader) = reader.split_at(4); + if magic_bytes != MAGIC_BYTES { + return Err(LoroError::DecodeError("Invalid magic bytes".into())); + } + + let (checksum, reader) = reader.split_at(16); + let checksum_body = reader; + let (mode_bytes, reader) = reader.split_at(2); + let mode: EncodeMode = [mode_bytes[0], mode_bytes[1]].try_into()?; + if mode == EncodeMode::Auto { + return Err(LoroError::DecodeError( + "Invalid import mode `Auto` in encoded blob" + .to_string() + .into_boxed_str(), + )); + } + let mut checksum_arr = [0; 16]; + checksum_arr.copy_from_slice(checksum); + + let ans = ParsedHeaderAndBody { + mode, + checksum_body, + checksum: checksum_arr, + body: reader, + }; + + if check_checksum { + ans.check_checksum()?; + } + Ok(ans) +} + +pub(crate) fn export_fast_snapshot(doc: &LoroDoc) -> Vec { + encode_with(EncodeMode::FastSnapshot, &mut |ans| { + fast_snapshot::encode_snapshot(doc, ans); + Ok(()) + }) + .unwrap() +} + +pub(crate) fn export_snapshot_at( + doc: &LoroDoc, + frontiers: &Frontiers, +) -> Result, LoroEncodeError> { + check_target_version_reachable(doc, frontiers)?; + encode_with(EncodeMode::FastSnapshot, &mut |ans| { + shallow_snapshot::encode_snapshot_at(doc, frontiers, ans) + }) +} + +pub(crate) fn export_fast_updates(doc: &LoroDoc, vv: &VersionVector) -> Vec { + encode_with(EncodeMode::FastUpdates, &mut |ans| { + fast_snapshot::encode_updates(doc, vv, ans); + Ok(()) + }) + .unwrap() +} + +pub(crate) fn export_fast_updates_in_range(oplog: &OpLog, spans: &[IdSpan]) -> Vec { + encode_with(EncodeMode::FastUpdates, &mut |ans| { + fast_snapshot::encode_updates_in_range(oplog, spans, ans); + Ok(()) + }) + .unwrap() +} + +pub(crate) fn export_shallow_snapshot( + doc: &LoroDoc, + f: &Frontiers, +) -> Result, LoroEncodeError> { + check_target_version_reachable(doc, f)?; + encode_with(EncodeMode::FastSnapshot, &mut |ans| { + shallow_snapshot::export_shallow_snapshot(doc, f, ans)?; + Ok(()) + }) +} + +fn check_target_version_reachable(doc: &LoroDoc, f: &Frontiers) -> Result<(), LoroEncodeError> { + let oplog = doc.oplog.lock(); + if !oplog.dag.can_export_shallow_snapshot_on(f) { + return Err(LoroEncodeError::FrontiersNotFound(format!("{f:?}"))); + } + + Ok(()) +} + +pub(crate) fn export_state_only_snapshot( + doc: &LoroDoc, + f: &Frontiers, +) -> Result, LoroEncodeError> { + check_target_version_reachable(doc, f)?; + encode_with(EncodeMode::FastSnapshot, &mut |ans| { + shallow_snapshot::export_state_only_snapshot(doc, f, ans)?; + Ok(()) + }) +} + +fn encode_with( + mode: EncodeMode, + f: &mut dyn FnMut(&mut Vec) -> Result<(), LoroEncodeError>, +) -> Result, LoroEncodeError> { + // HEADER + let mut ans = Vec::with_capacity(MIN_HEADER_SIZE); + ans.extend(MAGIC_BYTES); + let checksum = [0; 16]; + ans.extend(checksum); + ans.extend(mode.to_bytes()); + + // BODY + f(&mut ans)?; + + // CHECKSUM in HEADER + let checksum_body = &ans[20..]; + let checksum = xxhash_rust::xxh32::xxh32(checksum_body, XXH_SEED); + ans[16..20].copy_from_slice(&checksum.to_le_bytes()); + Ok(ans) +} + +pub(crate) fn decode_snapshot( + doc: &LoroDoc, + mode: EncodeMode, + body: &[u8], + origin: InternalString, +) -> Result { + match mode { + EncodeMode::OutdatedSnapshot => { + return Err(LoroError::ImportUnsupportedEncodingMode); + } + EncodeMode::FastSnapshot => { + fast_snapshot::decode_snapshot(doc, body.to_vec().into(), origin)? + } + _ => { + return Err(LoroError::DecodeError( + format!("Invalid snapshot encoding mode: {mode:?}").into_boxed_str(), + )); + } + }; + Ok(ImportStatus { + success: VersionRange::from_vv(&doc.oplog_vv()), + pending: None, + }) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum EncodedBlobMode { + Snapshot, + OutdatedSnapshot, + ShallowSnapshot, + OutdatedRle, + Updates, +} + +impl std::fmt::Display for EncodedBlobMode { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + EncodedBlobMode::OutdatedRle => "outdated-update", + EncodedBlobMode::OutdatedSnapshot => "outdated-snapshot", + EncodedBlobMode::Snapshot => "snapshot", + EncodedBlobMode::ShallowSnapshot => "shallow-snapshot", + EncodedBlobMode::Updates => "update", + }) + } +} + +impl EncodedBlobMode { + pub fn is_snapshot(&self) -> bool { + matches!( + self, + EncodedBlobMode::Snapshot + | EncodedBlobMode::ShallowSnapshot + | EncodedBlobMode::OutdatedSnapshot + ) + } +} + +#[derive(Debug, Clone)] +pub struct ImportBlobMetadata { + /// The partial start version vector. + /// + /// Import blob includes all the ops from `partial_start_vv` to `partial_end_vv`. + /// However, it does not constitute a complete version vector, as it only contains counters + /// from peers included within the import blob. + pub partial_start_vv: VersionVector, + /// The partial end version vector. + /// + /// Import blob includes all the ops from `partial_start_vv` to `partial_end_vv`. + /// However, it does not constitute a complete version vector, as it only contains counters + /// from peers included within the import blob. + pub partial_end_vv: VersionVector, + pub start_timestamp: i64, + pub start_frontiers: Frontiers, + pub end_timestamp: i64, + pub change_num: u32, + pub mode: EncodedBlobMode, +} + +impl LoroDoc { + /// Decodes the metadata for an imported blob from the provided bytes. + pub fn decode_import_blob_meta( + blob: &[u8], + check_checksum: bool, + ) -> LoroResult { + let parsed = parse_header_and_body(blob, check_checksum)?; + match parsed.mode { + EncodeMode::Auto => unreachable!(), + EncodeMode::OutdatedRle | EncodeMode::OutdatedSnapshot => { + Err(LoroError::ImportUnsupportedEncodingMode) + } + EncodeMode::FastSnapshot => fast_snapshot::decode_snapshot_blob_meta(parsed), + EncodeMode::FastUpdates => fast_snapshot::decode_updates_blob_meta(parsed), + } + } +} + +#[cfg(test)] +mod test { + + use loro_common::{loro_value, ContainerID, ContainerType, LoroValue, ID}; + + #[test] + fn test_value_encode_size() { + fn assert_size(value: LoroValue, max_size: usize) { + let size = postcard::to_allocvec(&value).unwrap().len(); + assert!( + size <= max_size, + "value: {:?}, size: {}, max_size: {}", + value, + size, + max_size + ); + } + + assert_size(LoroValue::Null, 1); + assert_size(LoroValue::I64(1), 2); + assert_size(LoroValue::Double(1.), 9); + assert_size(LoroValue::Bool(true), 2); + assert_size(LoroValue::String("123".to_string().into()), 5); + assert_size(LoroValue::Binary(vec![1, 2, 3].into()), 5); + assert_size( + loro_value!({ + "a": 1, + "b": 2, + }), + 10, + ); + assert_size(loro_value!([1, 2, 3]), 8); + assert_size( + LoroValue::Container(ContainerID::new_normal(ID::new(1, 1), ContainerType::Map)), + 5, + ); + assert_size( + LoroValue::Container(ContainerID::new_root("a", ContainerType::Map)), + 5, + ); + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/encoding/AGENTS.md b/3rdparty/loro/crates/loro-internal/src/encoding/AGENTS.md new file mode 100644 index 0000000000..5a50ce70d3 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/encoding/AGENTS.md @@ -0,0 +1,38 @@ +# Encoding Guidelines + +This module owns Loro import/export formats. Read +[../../../../context/internal-encoding.md](../../../../context/internal-encoding.md) +for the verified map of supported modes, outdated modes, shallow snapshots, JSON +schema, and validation entry points. + +## Local Entry Points + +- `../encoding.rs`: `ExportMode`, `EncodeMode`, 22-byte `loro` header, checksum + validation, top-level dispatch, and `decode_import_blob_meta`. +- `fast_snapshot.rs`: current `FastSnapshot` and `FastUpdates` body layouts. +- `shallow_snapshot.rs`: `ShallowSnapshot`, `StateOnly`, and `SnapshotAt`. +- `json_schema.rs`: JSON updates, peer compression, validation, import/export, + and redaction. +- `outdated_encode_reordered.rs`: legacy-named op/value columnar helpers still + used by current fast paths; do not confuse this with unsupported top-level + outdated blob modes. +- `value.rs`, `value_register.rs`, `arena.rs`: op/value encoding support. + +## Rules + +- Current binary modes are `FastSnapshot = 3` and `FastUpdates = 4`. +- Top-level `OutdatedRle = 1` and `OutdatedSnapshot = 2` are compatibility + detections, not formats to extend. +- Malformed bytes or JSON schema should return `Err`, not partially import. +- Snapshot import/export must preserve rollback and attached/detached state + invariants. +- Unknown container types must block shallow/state snapshot export rather than + producing a blob that cannot be decoded correctly. + +## Validation + +- `cargo test -p loro-internal import_atomicity` +- `cargo test -p loro-internal decode_updates_rejects_truncated_block` +- `cargo test -p loro-internal --test mergeable_container` when snapshot changes + can affect mergeable child retention. +- Root `pnpm test` for shared import/export semantic changes. diff --git a/3rdparty/loro/crates/loro-internal/src/encoding/CLAUDE.md b/3rdparty/loro/crates/loro-internal/src/encoding/CLAUDE.md new file mode 120000 index 0000000000..47dc3e3d86 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/encoding/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/3rdparty/loro/crates/loro-internal/src/encoding/arena.rs b/3rdparty/loro/crates/loro-internal/src/encoding/arena.rs new file mode 100644 index 0000000000..5ede1b7172 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/encoding/arena.rs @@ -0,0 +1,277 @@ +use std::{borrow::Cow, ops::Deref}; + +use crate::InternalString; +use loro_common::{ContainerID, ContainerType, Counter, LoroError, LoroResult, PeerID}; +use rustc_hash::FxHashSet; +use serde_columnar::columnar; + +use super::{ + outdated_encode_reordered::PeerIdx, + value::{Value, ValueDecodedArenasTrait, ValueEncodeRegister}, + value_register::ValueRegister, +}; + +#[derive(Debug)] +pub struct EncodedRegisters<'a> { + pub(super) peer: ValueRegister, + pub(super) key: ValueRegister, + pub(super) tree_id: ValueRegister, + pub(super) position: either::Either, ValueRegister<&'a [u8]>>, +} + +impl ValueEncodeRegister for EncodedRegisters<'_> { + fn key_mut(&mut self) -> &mut ValueRegister { + &mut self.key + } + + fn peer_mut(&mut self) -> &mut ValueRegister { + &mut self.peer + } + + fn encode_tree_op( + &mut self, + op: &crate::container::tree::tree_op::TreeOp, + ) -> super::value::Value<'static> { + Value::TreeMove(super::value::EncodedTreeMove::from_tree_op(op, self)) + } +} + +#[columnar(vec, ser, de, iterable)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub(crate) struct EncodedContainer { + #[columnar(strategy = "BoolRle")] + is_root: bool, + #[columnar(strategy = "Rle")] + kind: u8, + #[columnar(strategy = "Rle")] + peer_idx: usize, + #[columnar(strategy = "DeltaRle")] + key_idx_or_counter: i32, +} + +impl EncodedContainer { + pub fn as_container_id(&self, arenas: &dyn ValueDecodedArenasTrait) -> LoroResult { + if self.is_root { + Ok(ContainerID::Root { + container_type: ContainerType::try_from_u8(self.kind) + .unwrap_or(ContainerType::Unknown(self.kind)), + name: arenas + .keys() + .get(self.key_idx_or_counter as usize) + .ok_or(LoroError::DecodeDataCorruptionError)? + .clone(), + }) + } else { + Ok(ContainerID::Normal { + container_type: ContainerType::try_from_u8(self.kind) + .unwrap_or(ContainerType::Unknown(self.kind)), + peer: *(arenas + .peers() + .get(self.peer_idx) + .ok_or(LoroError::DecodeDataCorruptionError)?), + counter: self.key_idx_or_counter, + }) + } + } +} + +#[columnar(ser, de)] +#[derive(Default)] +pub(crate) struct ContainerArena { + #[columnar(class = "vec", iter = "EncodedContainer")] + pub(super) containers: Vec, +} + +impl Deref for ContainerArena { + type Target = [EncodedContainer]; + + fn deref(&self) -> &Self::Target { + &self.containers + } +} + +impl ContainerArena { + pub fn encode(&self) -> Vec { + serde_columnar::to_vec(&self.containers).unwrap() + } + + pub fn decode(bytes: &[u8]) -> LoroResult { + Ok(ContainerArena { + containers: serde_columnar::from_bytes(bytes)?, + }) + } + + pub fn from_containers( + cids: Vec, + peer_register: &mut ValueRegister, + key_reg: &mut ValueRegister, + ) -> Self { + let mut ans = Self { + containers: Vec::with_capacity(cids.len()), + }; + for cid in cids { + ans.push(cid, peer_register, key_reg); + } + + ans + } + + pub fn push( + &mut self, + id: ContainerID, + peer_register: &mut ValueRegister, + register_key: &mut ValueRegister, + ) { + let (is_root, kind, peer_idx, key_idx_or_counter) = match id { + ContainerID::Root { + container_type, + name, + } => (true, container_type, 0, register_key.register(&name) as i32), + ContainerID::Normal { + container_type, + peer, + counter, + } => ( + false, + container_type, + peer_register.register(&peer), + counter, + ), + }; + self.containers.push(EncodedContainer { + is_root, + kind: kind.to_u8(), + peer_idx, + key_idx_or_counter, + }); + } +} + +#[derive(Clone, Hash, PartialEq, Eq, Debug)] +#[columnar(vec, ser, de, iterable)] +pub struct EncodedTreeID { + #[columnar(strategy = "Rle")] + pub peer_idx: PeerIdx, + #[columnar(strategy = "DeltaRle")] + pub counter: Counter, +} + +#[derive(Clone)] +#[columnar(vec, ser, de, iterable)] +pub(super) struct PositionDelta<'a> { + #[columnar(strategy = "Rle")] + common_prefix_length: usize, + #[columnar(borrow)] + rest: Cow<'a, [u8]>, +} + +#[derive(Default)] +#[columnar(ser, de)] +pub(crate) struct PositionArena<'a> { + #[columnar(class = "vec", iter = "PositionDelta<'a>")] + pub(super) positions: Vec>, +} + +impl<'a> PositionArena<'a> { + pub fn from_positions(positions: impl IntoIterator) -> Self { + let iter = positions.into_iter(); + let mut ans = Vec::with_capacity(iter.size_hint().0); + let mut last_bytes: &[u8] = &[]; + for p in iter { + let common = longest_common_prefix_length(last_bytes, p); + let rest = &p[common..]; + last_bytes = p; + ans.push(PositionDelta { + common_prefix_length: common, + rest: Cow::Borrowed(rest), + }) + } + Self { positions: ans } + } + + pub fn try_parse_to_positions(self) -> LoroResult>> { + let mut ans: Vec> = Vec::with_capacity(self.positions.len()); + for PositionDelta { + common_prefix_length, + rest, + } in self.positions + { + if let Some(last_bytes) = ans.last() { + if common_prefix_length > last_bytes.len() { + return Err(LoroError::DecodeError( + "Decode position arena failed: common prefix out of range".into(), + )); + } + } else if common_prefix_length != 0 { + return Err(LoroError::DecodeError( + "Decode position arena failed: first common prefix is nonzero".into(), + )); + } + + // +1 for Fractional Index + let capacity = rest + .len() + .checked_add(common_prefix_length) + .and_then(|len| len.checked_add(1)) + .ok_or_else(|| { + LoroError::DecodeError( + "Decode position arena failed: position length overflow".into(), + ) + })?; + let mut p = Vec::with_capacity(capacity); + if let Some(last_bytes) = ans.last() { + p.extend_from_slice(&last_bytes[0..common_prefix_length]); + } + p.extend_from_slice(rest.as_ref()); + ans.push(p); + } + Ok(ans) + } + + pub fn encode(&self) -> Vec { + serde_columnar::to_vec(&self).unwrap() + } + + pub fn decode<'de: 'a>(bytes: &'de [u8]) -> LoroResult { + Ok(serde_columnar::from_bytes(bytes)?) + } + + pub fn encode_v2(&self) -> Vec { + if self.positions.is_empty() { + return Vec::new(); + } + + serde_columnar::to_vec(&self).unwrap() + } + + pub fn decode_v2<'de: 'a>(bytes: &'de [u8]) -> LoroResult { + if bytes.is_empty() { + return Ok(Self::default()); + } + + Ok(serde_columnar::from_bytes(bytes)?) + } +} + +fn longest_common_prefix_length(a: &[u8], b: &[u8]) -> usize { + a.iter().zip(b.iter()).take_while(|(x, y)| x == y).count() +} + +#[cfg(test)] +mod tests { + use std::borrow::Cow; + + use super::{PositionArena, PositionDelta}; + + #[test] + fn position_arena_rejects_invalid_common_prefix() { + let arena = PositionArena { + positions: vec![PositionDelta { + common_prefix_length: 1, + rest: Cow::Borrowed(&[]), + }], + }; + + assert!(arena.try_parse_to_positions().is_err()); + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/encoding/fast_snapshot.rs b/3rdparty/loro/crates/loro-internal/src/encoding/fast_snapshot.rs new file mode 100644 index 0000000000..67f466bdcf --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/encoding/fast_snapshot.rs @@ -0,0 +1,478 @@ +//! Fast snapshot encoding and decoding. +//! +//! # Layout +//! +//! - u32 in little endian for len of bytes for oplog +//! - oplog bytes +//! - u32 in little endian for len of bytes for state +//! - state bytes +//! - u32 in little endian for len of bytes for gc +//! - gc bytes +//! +//! All of `oplog bytes`, `state bytes` and `gc bytes` are encoded KV store bytes. +//! +//! +//! +use std::io::{Read, Write}; + +use crate::{ + change::Change, encoding::shallow_snapshot, oplog::ChangeStore, version::Frontiers, LoroDoc, + OpLog, VersionVector, +}; +use bytes::{Buf, Bytes}; +use loro_common::{HasCounterSpan, IdSpan, InternalString, LoroError, LoroResult}; + +use super::{EncodedBlobMode, ImportBlobMetadata, ParsedHeaderAndBody}; +pub(crate) const EMPTY_MARK: &[u8] = b"E"; +pub(crate) struct Snapshot { + pub oplog_bytes: Bytes, + pub state_bytes: Option, + pub shallow_root_state_bytes: Bytes, +} + +pub(super) fn _encode_snapshot(s: Snapshot, w: &mut W) { + w.write_all(&(s.oplog_bytes.len() as u32).to_le_bytes()) + .unwrap(); + w.write_all(&s.oplog_bytes).unwrap(); + let state_bytes = s + .state_bytes + .unwrap_or_else(|| Bytes::from_static(EMPTY_MARK)); + w.write_all(&(state_bytes.len() as u32).to_le_bytes()) + .unwrap(); + w.write_all(&state_bytes).unwrap(); + w.write_all(&(s.shallow_root_state_bytes.len() as u32).to_le_bytes()) + .unwrap(); + w.write_all(&s.shallow_root_state_bytes).unwrap(); +} + +pub(super) fn _decode_snapshot_bytes(bytes: Bytes) -> LoroResult { + let mut r = bytes.reader(); + let oplog_bytes_len = read_u32_le(&mut r)? as usize; + if r.get_ref().len() < oplog_bytes_len { + return Err(LoroError::DecodeError( + "decode_snapshot: invalid oplog bytes length" + .to_string() + .into_boxed_str(), + )); + } + let oplog_bytes = r.get_mut().copy_to_bytes(oplog_bytes_len); + let state_bytes_len = read_u32_le(&mut r)? as usize; + if r.get_ref().len() < state_bytes_len { + return Err(LoroError::DecodeError( + "decode_snapshot: invalid state bytes length" + .to_string() + .into_boxed_str(), + )); + } + let state_bytes = r.get_mut().copy_to_bytes(state_bytes_len); + let state_bytes = if state_bytes == EMPTY_MARK { + None + } else { + Some(state_bytes) + }; + let shallow_bytes_len = read_u32_le(&mut r)? as usize; + if r.get_ref().len() < shallow_bytes_len { + return Err(LoroError::DecodeError( + "decode_snapshot: invalid shallow root bytes length" + .to_string() + .into_boxed_str(), + )); + } + let shallow_root_state_bytes = r.get_mut().copy_to_bytes(shallow_bytes_len); + if r.get_ref().has_remaining() { + return Err(LoroError::DecodeError( + "decode_snapshot: trailing bytes after snapshot" + .to_string() + .into_boxed_str(), + )); + } + + Ok(Snapshot { + oplog_bytes, + state_bytes, + shallow_root_state_bytes, + }) +} + +pub(super) fn _decode_snapshot_meta_partial(bytes: &[u8]) -> LoroResult<(&[u8], bool)> { + let mut r = bytes; + let oplog_bytes_len = read_u32_le_slice(&mut r)? as usize; + if r.len() < oplog_bytes_len { + return Err(LoroError::DecodeDataCorruptionError); + } + let oplog_bytes = &r[..oplog_bytes_len]; + r = &r[oplog_bytes_len..]; + let state_bytes_len = read_u32_le_slice(&mut r)? as usize; + if r.len() < state_bytes_len { + return Err(LoroError::DecodeDataCorruptionError); + } + r = &r[state_bytes_len..]; + let shallow_bytes_len = read_u32_le_slice(&mut r)? as usize; + if r.len() < shallow_bytes_len { + return Err(LoroError::DecodeDataCorruptionError); + } + r = &r[shallow_bytes_len..]; + if !r.is_empty() { + return Err(LoroError::DecodeDataCorruptionError); + } + + Ok((oplog_bytes, shallow_bytes_len > 0)) +} + +fn read_u32_le_slice(r: &mut &[u8]) -> LoroResult { + let mut buf = [0; 4]; + r.read_exact(&mut buf) + .map_err(|_| LoroError::DecodeDataCorruptionError)?; + Ok(u32::from_le_bytes(buf)) +} + +fn read_u32_le(r: &mut bytes::buf::Reader) -> LoroResult { + let mut buf = [0; 4]; + r.read_exact(&mut buf).map_err(|_| { + LoroError::DecodeError( + "decode_snapshot: unexpected end of snapshot bytes" + .to_string() + .into_boxed_str(), + ) + })?; + Ok(u32::from_le_bytes(buf)) +} + +pub(crate) fn decode_snapshot( + doc: &LoroDoc, + bytes: Bytes, + origin: InternalString, +) -> LoroResult<()> { + let snapshot = _decode_snapshot_bytes(bytes)?; + decode_snapshot_inner(snapshot, doc, origin) +} + +pub(crate) fn decode_snapshot_inner( + snapshot: Snapshot, + doc: &LoroDoc, + origin: InternalString, +) -> Result<(), LoroError> { + let Snapshot { + oplog_bytes, + state_bytes, + shallow_root_state_bytes, + } = snapshot; + ensure_cov::notify_cov("loro_internal::import::fast_snapshot::decode_snapshot"); + let mut oplog = doc.oplog().lock(); + if !oplog.is_empty() { + return Err(LoroError::DecodeError( + "decode_snapshot: cannot import snapshot into a non-empty doc" + .to_string() + .into_boxed_str(), + )); + } + + let mut state = doc.app_state().lock(); + + state.check_before_decode_snapshot()?; + + if !state.frontiers.is_empty() { + return Err(LoroError::DecodeError( + "decode_snapshot: app state frontiers must be empty before import" + .to_string() + .into_boxed_str(), + )); + } + + if !oplog.frontiers().is_empty() { + return Err(LoroError::DecodeError( + "decode_snapshot: oplog frontiers must be empty before import" + .to_string() + .into_boxed_str(), + )); + } + let need_calc = state_bytes.is_none(); + let checkout_origin = origin.clone(); + + let arena_checkpoint = oplog.arena.checkpoint_for_rollback(); + let decode_result = (|| -> LoroResult<()> { + oplog.decode_change_store(oplog_bytes)?; + let state_frontiers; + if shallow_root_state_bytes.is_empty() { + ensure_cov::notify_cov("loro_internal::import::snapshot::normal"); + if let Some(bytes) = state_bytes { + state.store.decode(bytes)?; + } + state_frontiers = oplog.frontiers().clone(); + } else { + ensure_cov::notify_cov("loro_internal::import::snapshot::gc"); + let shallow_root_state_frontiers = state.store.decode_gc( + shallow_root_state_bytes.clone(), + oplog.dag().shallow_since_frontiers().clone(), + doc.config.clone(), + )?; + state.store.decode_state_by_two_bytes( + shallow_root_state_bytes, + state_bytes.unwrap_or_default(), + )?; + + let shallow_root_store = state.shallow_root_store().cloned(); + oplog.with_history_cache(|h| { + h.set_shallow_root_store(shallow_root_store); + }); + + if need_calc { + ensure_cov::notify_cov("shallow_snapshot::need_calc"); + state_frontiers = shallow_root_state_frontiers.ok_or_else(|| { + LoroError::DecodeError( + "decode_snapshot: shallow root frontiers are missing" + .to_string() + .into_boxed_str(), + ) + })?; + } else { + ensure_cov::notify_cov("shallow_snapshot::dont_need_calc"); + state_frontiers = oplog.frontiers().clone(); + } + } + + state.init_with_states_and_version(state_frontiers, &oplog, vec![], false, origin)?; + Ok(()) + })(); + + if let Err(e) = decode_result { + state.reset_to_empty_for_failed_snapshot_import(); + oplog.reset_to_empty_for_failed_snapshot_import(arena_checkpoint); + return Err(e); + } + drop(state); + drop(oplog); + if need_calc { + doc.set_detached(true); + if let Err(e) = doc._checkout_to_latest_without_commit_as_import(false, checkout_origin) { + doc.set_detached(false); + doc.app_state() + .lock() + .reset_to_empty_for_failed_snapshot_import(); + doc.oplog() + .lock() + .reset_to_empty_for_failed_snapshot_import(arena_checkpoint); + return Err(e); + } + debug_assert_eq!(doc.state_frontiers(), doc.oplog_frontiers()); + } + + Ok(()) +} + +impl OpLog { + pub(super) fn decode_change_store(&mut self, bytes: bytes::Bytes) -> LoroResult<()> { + let v = self.change_store().import_all(bytes)?; + self.dag.set_version_by_fast_snapshot_import(v); + self.refresh_visible_op_count(); + Ok(()) + } +} + +pub(crate) fn encode_snapshot(doc: &LoroDoc, w: &mut W) { + let snapshot = encode_snapshot_inner(doc); + _encode_snapshot(snapshot, w); +} + +pub(crate) fn encode_snapshot_inner(doc: &LoroDoc) -> Snapshot { + assert!(doc.drop_pending_events().is_empty()); + let old_state_frontiers = doc.state_frontiers(); + let was_detached = doc.is_detached(); + let oplog = doc.oplog().lock(); + let mut state = doc.app_state().lock(); + let is_gc = state.store.shallow_root_store().is_some(); + if is_gc { + // TODO: PERF: this can be optimized by reusing the bytes of gc store + let f = oplog.shallow_since_frontiers().clone(); + drop(state); + drop(oplog); + let (snapshot, _) = shallow_snapshot::export_shallow_snapshot_inner(doc, &f).unwrap(); + return snapshot; + } + + assert!(!state.is_in_txn()); + let oplog_bytes = oplog.encode_change_store(); + if oplog.is_shallow() { + assert_eq!( + oplog.shallow_since_frontiers(), + state.store.shallow_root_frontiers().unwrap() + ); + } + if was_detached { + let latest = oplog.frontiers().clone(); + drop(state); + drop(oplog); + doc._checkout_without_emitting(&latest, false, true) + .unwrap(); + state = doc.app_state().lock(); + } + state.ensure_all_alive_containers(); + let state_bytes = state.store.encode(); + let snapshot = Snapshot { + oplog_bytes, + state_bytes: Some(state_bytes), + shallow_root_state_bytes: Bytes::new(), + }; + if was_detached { + drop(state); + doc._checkout_without_emitting(&old_state_frontiers, false, true) + .unwrap(); + doc.drop_pending_events(); + } + + snapshot +} + +pub(crate) fn decode_oplog(oplog: &mut OpLog, bytes: &[u8]) -> Result, LoroError> { + let oplog_len = bytes + .get(0..4) + .ok_or_else(|| LoroError::DecodeError("decode_oplog: missing length prefix".into()))?; + let oplog_len = u32::from_le_bytes( + oplog_len + .try_into() + .expect("slice length checked to be exactly 4"), + ) as usize; + let oplog_bytes = bytes + .get(4..4 + oplog_len) + .ok_or_else(|| LoroError::DecodeError("decode_oplog: invalid oplog length".into()))?; + let mut changes = ChangeStore::decode_snapshot_for_updates( + oplog_bytes.to_vec().into(), + &oplog.arena, + oplog.vv(), + )?; + changes.sort_unstable_by_key(|x| x.lamport); + Ok(changes) +} + +pub(crate) fn encode_updates(doc: &LoroDoc, vv: &VersionVector, w: &mut W) { + let oplog = doc.oplog().lock(); + oplog.export_blocks_from(vv, w); +} + +pub(crate) fn encode_updates_in_range( + oplog: &OpLog, + spans: &[IdSpan], + w: &mut W, +) { + oplog.export_blocks_in_range(spans, w); +} + +pub(crate) fn decode_updates(oplog: &mut OpLog, body: Bytes) -> Result, LoroError> { + let mut reader: &[u8] = body.as_ref(); + let mut index = 0; + let self_vv = oplog.vv(); + let mut changes = Vec::new(); + while !reader.is_empty() { + let old_reader_len = reader.len(); + let len = leb128::read::unsigned(&mut reader) + .map_err(|_| LoroError::DecodeError("decode_updates: invalid block length".into()))? + as usize; + index += old_reader_len - reader.len(); + let end = index.checked_add(len).ok_or_else(|| { + LoroError::DecodeError("decode_updates: block length overflow".into()) + })?; + if end > body.len() { + return Err(LoroError::DecodeError( + "decode_updates: truncated block payload".into(), + )); + } + let block_bytes = body.slice(index..end); + let new_changes = ChangeStore::decode_block_bytes(block_bytes, &oplog.arena, self_vv)?; + changes.extend(new_changes); + index = end; + reader = &reader[len..]; + } + + changes.sort_unstable_by_key(|x| x.lamport); + Ok(changes) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn decode_updates_rejects_truncated_block() { + let doc = LoroDoc::new(); + let mut oplog = doc.oplog.lock(); + let err = decode_updates(&mut oplog, Bytes::from_static(&[0x02, 0x01])) + .expect_err("truncated update block should be rejected"); + assert!(matches!(err, LoroError::DecodeError(_))); + } +} + +pub(crate) fn decode_snapshot_blob_meta( + parsed: ParsedHeaderAndBody, +) -> LoroResult { + let (oplog_bytes, is_shallow) = _decode_snapshot_meta_partial(parsed.body)?; + let mode = if is_shallow { + EncodedBlobMode::ShallowSnapshot + } else { + EncodedBlobMode::Snapshot + }; + + let doc = LoroDoc::new(); + let mut oplog = doc.oplog.lock(); + oplog.decode_change_store(oplog_bytes.to_vec().into())?; + let timestamp = oplog.get_greatest_timestamp(oplog.dag.frontiers()); + let f = oplog.dag.shallow_since_frontiers().clone(); + let start_timestamp = oplog.get_timestamp_of_version(&f); + let change_num = oplog.change_store().change_num() as u32; + + Ok(ImportBlobMetadata { + mode, + partial_start_vv: oplog.dag.shallow_since_vv().to_vv(), + partial_end_vv: oplog.vv().clone(), + start_timestamp, + start_frontiers: f, + end_timestamp: timestamp, + change_num, + }) +} + +pub(crate) fn decode_updates_blob_meta( + parsed: ParsedHeaderAndBody, +) -> LoroResult { + let doc = LoroDoc::new(); + let mut oplog = doc.oplog.lock(); + let changes = decode_updates(&mut oplog, parsed.body.to_vec().into())?; + let mut start_vv = VersionVector::new(); + let mut end_vv = VersionVector::new(); + for c in changes.iter() { + match start_vv.get(&c.id.peer).copied() { + Some(start) if start <= c.id.counter => {} + _ => { + start_vv.insert(c.id.peer, c.id.counter); + } + } + match end_vv.get(&c.id.peer).copied() { + Some(end) if end >= c.ctr_end() => {} + _ => { + end_vv.insert(c.id.peer, c.ctr_end()); + } + } + } + + let mut start_frontiers = Frontiers::new(); + for c in changes.iter() { + for dep in c.deps().iter() { + if let Some(start_counter) = start_vv.get(&dep.peer) { + if *start_counter > dep.counter { + start_frontiers.push(dep); + } + } else if end_vv.get(&dep.peer).is_none() { + start_frontiers.push(dep); + } + } + } + + Ok(ImportBlobMetadata { + mode: EncodedBlobMode::Updates, + partial_start_vv: start_vv, + partial_end_vv: end_vv, + start_timestamp: changes.first().map(|x| x.timestamp).unwrap_or(0), + start_frontiers, + end_timestamp: changes.last().map(|x| x.timestamp).unwrap_or(0), + change_num: changes.len() as u32, + }) +} diff --git a/3rdparty/loro/crates/loro-internal/src/encoding/json_schema.rs b/3rdparty/loro/crates/loro-internal/src/encoding/json_schema.rs new file mode 100644 index 0000000000..2f846c3137 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/encoding/json_schema.rs @@ -0,0 +1,1912 @@ +use super::outdated_encode_reordered::ValueRegister; +use crate::{ + arena::SharedArena, + change::{Change, ChangeRef}, + container::{ + list::list_op::{DeleteSpan, DeleteSpanWithId, InnerListOp}, + map::MapSet, + richtext::TextStyleInfoFlag, + tree::tree_op::TreeOp, + }, + op::{FutureInnerContent, InnerContent, Op, SliceRange}, + oplog::BlockChangeRef, + version::Frontiers, + OpLog, VersionVector, +}; +use either::Either; +use itertools::Itertools; +use json::{JsonChange, JsonOpContent, JsonSchema}; +use loro_common::{ + ContainerID, ContainerType, Counter, HasCounterSpan, HasId, IdLp, IdSpan, LoroError, + LoroResult, LoroValue, PeerID, TreeID, ID, +}; +use rle::{HasLength, RleVec, Sliceable}; +use std::sync::Arc; + +const SCHEMA_VERSION: u8 = 1; + +fn refine_vv(vv: &VersionVector, oplog: &OpLog) -> VersionVector { + let mut refined = VersionVector::new(); + for (&peer, &counter) in vv.iter() { + if counter <= 0 { + continue; + } + let end = oplog.vv().get(&peer).copied().unwrap_or(0); + if end <= counter { + refined.insert(peer, end); + } else { + refined.insert(peer, counter); + } + } + refined +} + +pub(crate) fn export_json<'a, 'c: 'a>( + oplog: &'c OpLog, + start_vv: &VersionVector, + end_vv: &VersionVector, + with_peer_compression: bool, +) -> JsonSchema { + let actual_start_vv = refine_vv(start_vv, oplog); + let actual_end_vv = refine_vv(end_vv, oplog); + + let frontiers = oplog.dag.vv_to_frontiers(&actual_start_vv); + + let diff_changes = init_encode(oplog, &actual_start_vv, &actual_end_vv); + if with_peer_compression { + let mut peer_register = ValueRegister::::new(); + let changes = encode_changes(&diff_changes, &oplog.arena, Some(&mut peer_register)); + JsonSchema { + changes, + schema_version: SCHEMA_VERSION, + peers: Some(peer_register.unwrap_vec()), + start_version: frontiers, + } + } else { + let changes = encode_changes(&diff_changes, &oplog.arena, None); + JsonSchema { + changes, + schema_version: SCHEMA_VERSION, + peers: None, + start_version: frontiers, + } + } +} + +pub(crate) fn export_json_in_id_span(oplog: &OpLog, mut id_span: IdSpan) -> Vec { + id_span.normalize_(); + if id_span.counter.end <= 0 { + return vec![]; + } + id_span.counter.start = id_span.counter.start.max(0); + id_span.counter.end = id_span.counter.end.max(0); + + let end = oplog.vv().get(&id_span.peer).copied().unwrap_or(0); + if id_span.counter.start >= end { + return vec![]; + } + + id_span.counter.end = id_span.counter.end.min(end); + let mut diff_changes: Vec> = Vec::new(); + while id_span.counter.start < id_span.counter.end { + let Some(change) = oplog.get_change_at(id_span.id_start()) else { + break; + }; + let ctr_end = change.ctr_end(); + if change.id.counter >= id_span.counter.start && change.ctr_end() <= id_span.counter.end { + diff_changes.push(Either::Left(change)); + } else { + let start = if change.id.counter < id_span.counter.start { + (id_span.counter.start - change.id.counter) as usize + } else { + 0 + }; + + let end = if change.ctr_end() > id_span.counter.end { + (id_span.counter.end - change.id.counter) as usize + } else { + change.atom_len() + }; + + diff_changes.push(Either::Right(change.slice(start, end))); + } + + id_span.counter.start = ctr_end; + } + + encode_changes(&diff_changes, &oplog.arena, None) +} + +pub(crate) fn decode_json_changes( + json: JsonSchema, + arena: &SharedArena, +) -> LoroResult> { + decode_changes(json, arena) +} + +fn init_encode<'s, 'a: 's>( + oplog: &'a OpLog, + start_vv: &VersionVector, + end_vv: &VersionVector, +) -> Vec> { + let mut diff_changes: Vec> = Vec::new(); + for change in oplog.iter_changes_peer_by_peer(start_vv, end_vv) { + let start_cnt = start_vv.get(&change.id.peer).copied().unwrap_or(0); + let end_cnt = end_vv.get(&change.id.peer).copied().unwrap_or(0); + if change.id.counter < start_cnt { + if change.ctr_end() <= start_cnt { + continue; + } + + let offset = start_cnt - change.id.counter; + let to = change + .atom_len() + .min((end_cnt - change.id.counter) as usize); + diff_changes.push(Either::Right(change.slice(offset as usize, to))); + } else if change.id.counter + change.atom_len() as i32 > end_cnt { + let len = end_cnt - change.id.counter; + diff_changes.push(Either::Right(change.slice(0, len as usize))); + } else { + diff_changes.push(Either::Left(change)); + } + } + diff_changes.sort_by_key(|x| match x { + Either::Left(c) => c.lamport, + Either::Right(c) => c.lamport, + }); + diff_changes +} + +fn register_id(id: &ID, peer_register: Option<&mut ValueRegister>) -> ID { + let peer = match peer_register { + Some(peer_register) => peer_register.register(&id.peer) as PeerID, + None => id.peer, + }; + ID::new(peer as PeerID, id.counter) +} + +fn register_idlp(idlp: &IdLp, peer_register: Option<&mut ValueRegister>) -> IdLp { + let peer = match peer_register { + Some(peer_register) => peer_register.register(&idlp.peer) as PeerID, + None => idlp.peer, + }; + IdLp { + peer, + lamport: idlp.lamport, + } +} + +fn register_tree_id(tree: &TreeID, peer_register: Option<&mut ValueRegister>) -> TreeID { + TreeID { + peer: match peer_register { + Some(peer_register) => peer_register.register(&tree.peer) as PeerID, + None => tree.peer, + }, + counter: tree.counter, + } +} + +fn register_container_id( + container: ContainerID, + peer_register: Option<&mut ValueRegister>, +) -> ContainerID { + match container { + ContainerID::Normal { + peer, + counter, + container_type, + } => ContainerID::Normal { + peer: match peer_register { + Some(peer_register) => peer_register.register(&peer) as PeerID, + None => peer, + }, + counter, + container_type, + }, + r => r, + } +} + +fn convert_container_id( + container: ContainerID, + peers: &Option>, +) -> LoroResult { + match container { + ContainerID::Normal { + peer, + counter, + container_type, + } => Ok(ContainerID::Normal { + peer: get_peer_from_peers(peers, peer)?, + counter, + container_type, + }), + r => Ok(r), + } +} + +pub(crate) fn get_peer_from_peers(peers: &Option>, peer: PeerID) -> LoroResult { + match peers { + Some(peers) => peers.get(peer as usize).copied().ok_or_else(|| { + LoroError::DecodeError("peer index out of range in compressed peer table".into()) + }), + None => Ok(peer), + } +} + +fn convert_id(id: &ID, peers: &Option>) -> LoroResult { + Ok(ID { + peer: get_peer_from_peers(peers, id.peer)?, + counter: id.counter, + }) +} + +fn convert_idlp(idlp: &IdLp, peers: &Option>) -> LoroResult { + Ok(IdLp { + lamport: idlp.lamport, + peer: get_peer_from_peers(peers, idlp.peer)?, + }) +} + +fn convert_tree_id(tree: &TreeID, peers: &Option>) -> LoroResult { + Ok(TreeID { + peer: get_peer_from_peers(peers, tree.peer)?, + counter: tree.counter, + }) +} +pub(crate) fn encode_change_to_json(change: ChangeRef<'_>, arena: &SharedArena) -> JsonSchema { + let f = change.deps.clone(); + let changes = vec![encode_change(change, arena, None)]; + JsonSchema { + changes, + schema_version: SCHEMA_VERSION, + peers: None, + start_version: f, + } +} + +fn encode_changes( + diff_changes: &[Either], + arena: &SharedArena, + mut peer_register: Option<&mut ValueRegister>, +) -> Vec { + let mut changes = Vec::with_capacity(diff_changes.len()); + for change in diff_changes.iter() { + let change: &Change = match change { + Either::Left(c) => c, + Either::Right(c) => c, + }; + let c = encode_change( + ChangeRef::from_change(change), + arena, + peer_register.as_deref_mut(), + ); + changes.push(c); + } + changes +} + +pub(crate) fn encode_change( + change: ChangeRef<'_, Op>, + arena: &SharedArena, + mut peer_register: Option<&mut ValueRegister>, +) -> JsonChange { + let mut ops = Vec::with_capacity(change.ops.len()); + for Op { + counter, + container, + content, + } in change.ops.iter() + { + let mut container = arena.get_container_id(*container).unwrap(); + if container.is_normal() { + container = register_container_id(container, peer_register.as_deref_mut()); + } + let op = match container.container_type() { + ContainerType::List => match content { + InnerContent::List(list) => JsonOpContent::List(match list { + InnerListOp::Insert { slice, pos } => { + let mut values = + arena.get_values(slice.0.start as usize..slice.0.end as usize); + values.iter_mut().for_each(|x| { + if let LoroValue::Container(id) = x { + if id.is_normal() { + *id = register_container_id( + id.clone(), + peer_register.as_deref_mut(), + ); + } + } + }); + json::ListOp::Insert { + pos: *pos as u32, + value: values, + } + } + InnerListOp::Delete(DeleteSpanWithId { + id_start, + span: DeleteSpan { pos, signed_len }, + }) => json::ListOp::Delete { + pos: *pos as i32, + len: *signed_len as i32, + start_id: register_id(id_start, peer_register.as_deref_mut()), + }, + _ => unreachable!(), + }), + _ => unreachable!(), + }, + ContainerType::MovableList => match content { + InnerContent::List(list) => JsonOpContent::MovableList(match list { + InnerListOp::Insert { slice, pos } => { + let mut values = + arena.get_values(slice.0.start as usize..slice.0.end as usize); + values.iter_mut().for_each(|x| { + if let LoroValue::Container(id) = x { + if id.is_normal() { + *id = register_container_id( + id.clone(), + peer_register.as_deref_mut(), + ); + } + } + }); + json::MovableListOp::Insert { + pos: *pos as u32, + value: values, + } + } + InnerListOp::Delete(DeleteSpanWithId { + id_start, + span: DeleteSpan { pos, signed_len }, + }) => json::MovableListOp::Delete { + pos: *pos as i32, + len: *signed_len as i32, + start_id: register_id(id_start, peer_register.as_deref_mut()), + }, + InnerListOp::Move { + from, + elem_id: from_id, + to, + } => json::MovableListOp::Move { + from: *from, + to: *to, + elem_id: register_idlp(from_id, peer_register.as_deref_mut()), + }, + InnerListOp::Set { elem_id, value } => { + let value = if let LoroValue::Container(id) = value { + if id.is_normal() { + LoroValue::Container(register_container_id( + id.clone(), + peer_register.as_deref_mut(), + )) + } else { + value.clone() + } + } else { + value.clone() + }; + json::MovableListOp::Set { + elem_id: register_idlp(elem_id, peer_register.as_deref_mut()), + value, + } + } + _ => unreachable!(), + }), + _ => unreachable!(), + }, + ContainerType::Text => match content { + InnerContent::List(list) => JsonOpContent::Text(match list { + InnerListOp::InsertText { + slice, + unicode_start: _, + unicode_len: _, + pos, + } => { + let text = String::from_utf8(slice.as_bytes().to_vec()).unwrap(); + json::TextOp::Insert { pos: *pos, text } + } + InnerListOp::Delete(DeleteSpanWithId { + id_start, + span: DeleteSpan { pos, signed_len }, + }) => json::TextOp::Delete { + pos: *pos as i32, + len: *signed_len as i32, + start_id: register_id(id_start, peer_register.as_deref_mut()), + }, + InnerListOp::StyleStart { + start, + end, + key, + value, + info, + } => json::TextOp::Mark { + start: *start, + end: *end, + style_key: key.to_string(), + style_value: value.clone(), + info: info.to_byte(), + }, + InnerListOp::StyleEnd => json::TextOp::MarkEnd, + _ => unreachable!(), + }), + _ => unreachable!(), + }, + ContainerType::Map => match content { + InnerContent::Map(MapSet { key, value }) => { + JsonOpContent::Map(if let Some(v) = value { + let value = if let LoroValue::Container(id) = v { + if id.is_normal() { + LoroValue::Container(register_container_id( + id.clone(), + peer_register.as_deref_mut(), + )) + } else { + v.clone() + } + } else { + v.clone() + }; + json::MapOp::Insert { + key: key.to_string(), + value, + } + } else { + json::MapOp::Delete { + key: key.to_string(), + } + }) + } + + _ => unreachable!(), + }, + + ContainerType::Tree => match content { + InnerContent::Tree(op) => JsonOpContent::Tree(match &**op { + TreeOp::Create { + target, + parent, + position, + } => json::TreeOp::Create { + target: register_tree_id(target, peer_register.as_deref_mut()), + parent: parent.map(|p| register_tree_id(&p, peer_register.as_deref_mut())), + fractional_index: position.clone(), + }, + TreeOp::Move { + target, + parent, + position, + } => json::TreeOp::Move { + target: register_tree_id(target, peer_register.as_deref_mut()), + parent: parent.map(|p| register_tree_id(&p, peer_register.as_deref_mut())), + fractional_index: position.clone(), + }, + TreeOp::Delete { target } => json::TreeOp::Delete { + target: register_tree_id(target, peer_register.as_deref_mut()), + }, + }), + _ => unreachable!(), + }, + ContainerType::Unknown(_) => { + let InnerContent::Future(FutureInnerContent::Unknown { prop, value }) = content + else { + unreachable!(); + }; + JsonOpContent::Future(json::FutureOpWrapper { + prop: *prop, + value: json::FutureOp::Unknown((**value).clone()), + }) + } + #[cfg(feature = "counter")] + ContainerType::Counter => { + let InnerContent::Future(f) = content else { + unreachable!() + }; + match f { + FutureInnerContent::Counter(x) => { + JsonOpContent::Future(json::FutureOpWrapper { + prop: 0, + value: json::FutureOp::Counter(super::OwnedValue::F64(*x)), + }) + } + _ => unreachable!(), + } + } + }; + ops.push(json::JsonOp { + counter: *counter, + container, + content: op, + }); + } + let c = json::JsonChange { + id: register_id(change.id, peer_register.as_deref_mut()), + ops, + deps: change + .deps + .iter() + // Make sure the order is deterministic + .sorted() + .map(|id| register_id(&id, peer_register.as_deref_mut())) + .collect(), + lamport: *change.lamport, + timestamp: *change.timestamp, + msg: change.commit_msg.as_deref().map(|x| x.to_string()), + }; + c +} + +fn decode_changes(json: JsonSchema, arena: &SharedArena) -> LoroResult> { + let JsonSchema { + schema_version, + start_version, + peers, + changes, + } = json; + if schema_version != SCHEMA_VERSION { + return Err(LoroError::DecodeError( + format!( + "unsupported json schema version: expected {SCHEMA_VERSION}, got {schema_version}" + ) + .into_boxed_str(), + )); + } + validate_json_frontiers(&start_version)?; + + let mut ans = Vec::with_capacity(changes.len()); + for json::JsonChange { + id, + timestamp, + deps, + lamport, + msg, + ops: json_ops, + } in changes + { + let id = convert_id(&id, &peers)?; + validate_json_id_counter(id, "change id")?; + let mut ops: RleVec<[Op; 1]> = RleVec::new(); + let mut expected_counter = id.counter; + if json_ops.is_empty() { + return Err(LoroError::DecodeError( + "invalid json change: change must contain at least one op".into(), + )); + } + + for op in json_ops { + let next_counter = validate_json_op_counter(expected_counter, &op)?; + let op = decode_op(op, arena, &peers)?; + validate_json_op_created_container_ids(id.peer, &op, arena)?; + ops.push(op); + expected_counter = next_counter; + } + + let deps = deps + .into_iter() + .map(|id| convert_id(&id, &peers)) + .collect::>>()?; + for dep in &deps { + validate_json_id_counter(*dep, "dependency id")?; + } + + let change = Change { + id, + timestamp, + deps: Frontiers::from_iter(deps), + lamport, + ops, + commit_msg: msg.map(|x| x.into()), + }; + ans.push(change); + } + Ok(ans) +} + +fn validate_json_frontiers(frontiers: &Frontiers) -> LoroResult<()> { + for id in frontiers.iter() { + validate_json_id_counter(id, "start version id")?; + } + + Ok(()) +} + +fn validate_json_id_counter(id: ID, name: &str) -> LoroResult<()> { + if id.counter < 0 { + return Err(LoroError::DecodeError( + format!("invalid json counter: {name} counter must be non-negative").into_boxed_str(), + )); + } + + Ok(()) +} + +fn validate_json_tree_id_counter(id: &TreeID, name: &str) -> LoroResult<()> { + if id.counter < 0 { + return Err(LoroError::DecodeError( + format!("invalid json counter: {name} counter must be non-negative").into_boxed_str(), + )); + } + + Ok(()) +} + +fn validate_json_container_id_counter(id: &ContainerID, name: &str) -> LoroResult<()> { + if let ContainerID::Normal { counter, .. } = id { + if *counter < 0 { + return Err(LoroError::DecodeError( + format!("invalid json counter: {name} counter must be non-negative") + .into_boxed_str(), + )); + } + } + + Ok(()) +} + +fn validate_json_op_counter(expected_counter: Counter, op: &json::JsonOp) -> LoroResult { + let op_len = op.content.op_len(); + if op_len == 0 { + return Err(LoroError::DecodeError( + "invalid json op counter: op length must be greater than zero".into(), + )); + } + + if expected_counter < 0 || op.counter < 0 { + return Err(LoroError::DecodeError( + "invalid json op counter: op counters must be non-negative".into(), + )); + } + + let op_len = Counter::try_from(op_len).map_err(|_| { + LoroError::DecodeError("invalid json op counter: op length is too large".into()) + })?; + if op.counter != expected_counter { + return Err(LoroError::DecodeError( + "invalid json op counter: op counters must be contiguous with the change id".into(), + )); + } + + expected_counter + .checked_add(op_len) + .ok_or_else(|| LoroError::DecodeError("invalid json op counter: counter overflow".into())) +} + +fn validate_json_op_created_container_ids( + peer: PeerID, + op: &Op, + arena: &SharedArena, +) -> LoroResult<()> { + match &op.content { + InnerContent::List(list) => match list { + InnerListOp::Insert { slice, .. } => { + for (offset, value) in arena.iter_value_slice(slice.to_range()).enumerate() { + let LoroValue::Container(id) = value else { + validate_json_value_has_no_container_refs(&value)?; + continue; + }; + let offset = Counter::try_from(offset).map_err(|_| { + LoroError::DecodeError( + "invalid json container id: list offset is too large".into(), + ) + })?; + let counter = op.counter.checked_add(offset).ok_or_else(|| { + LoroError::DecodeError("invalid json container id: counter overflow".into()) + })?; + validate_json_created_container_id(&id, ID::new(peer, counter))?; + } + } + InnerListOp::Set { value, .. } => { + if let LoroValue::Container(id) = value { + validate_json_created_container_id(id, ID::new(peer, op.counter))?; + } else { + validate_json_value_has_no_container_refs(value)?; + } + } + InnerListOp::Move { .. } + | InnerListOp::InsertText { .. } + | InnerListOp::Delete(_) + | InnerListOp::StyleEnd => {} + InnerListOp::StyleStart { value, .. } => { + validate_json_value_has_no_container_refs(value)?; + } + }, + InnerContent::Map(map) => { + if let Some(value) = &map.value { + if let LoroValue::Container(id) = value { + validate_json_created_container_id(id, ID::new(peer, op.counter))?; + } else { + validate_json_value_has_no_container_refs(value)?; + } + } + } + InnerContent::Tree(tree) => { + if let TreeOp::Create { target, .. } = tree.as_ref() { + validate_json_tree_create_target(target, ID::new(peer, op.counter))?; + } + } + InnerContent::Future(_) => {} + } + + Ok(()) +} + +fn validate_json_value_has_no_container_refs(value: &LoroValue) -> LoroResult<()> { + let mut stack = vec![value]; + while let Some(value) = stack.pop() { + match value { + LoroValue::Container(_) => { + return Err(LoroError::DecodeError( + "invalid json container id: container values must not be nested".into(), + )); + } + LoroValue::List(list) => { + stack.extend(list.iter()); + } + LoroValue::Map(map) => { + stack.extend(map.values()); + } + LoroValue::Null + | LoroValue::Bool(_) + | LoroValue::Double(_) + | LoroValue::I64(_) + | LoroValue::Binary(_) + | LoroValue::String(_) => {} + } + } + + Ok(()) +} + +fn validate_json_created_container_id(id: &ContainerID, expected_id: ID) -> LoroResult<()> { + let ContainerID::Normal { peer, counter, .. } = id else { + return Err(LoroError::DecodeError( + "invalid json container id: created child containers must be normal".into(), + )); + }; + + if *peer != expected_id.peer || *counter != expected_id.counter { + return Err(LoroError::DecodeError( + "invalid json container id: created child container id must match the op id".into(), + )); + } + + Ok(()) +} + +fn validate_json_tree_create_target(target: &TreeID, expected_id: ID) -> LoroResult<()> { + if target.peer != expected_id.peer || target.counter != expected_id.counter { + return Err(LoroError::DecodeError( + "invalid json tree target: tree create target must match the op id".into(), + )); + } + + Ok(()) +} + +fn decode_op(op: json::JsonOp, arena: &SharedArena, peers: &Option>) -> LoroResult { + let json::JsonOp { + counter, + container, + content, + } = op; + let container = convert_container_id(container, peers)?; + validate_json_container_id_counter(&container, "op container")?; + let idx = arena.register_container(&container); + let content = match container.container_type() { + ContainerType::Text => match content { + JsonOpContent::Text(text) => match text { + json::TextOp::Insert { pos, text } => { + let (slice, result) = arena.alloc_str_with_slice(&text); + InnerContent::List(InnerListOp::InsertText { + slice, + unicode_start: result.start as u32, + unicode_len: (result.end - result.start) as u32, + pos, + }) + } + json::TextOp::Delete { + pos, + len, + start_id: id_start, + } => { + let id_start = convert_id(&id_start, peers)?; + validate_json_id_counter(id_start, "text delete start id")?; + InnerContent::List(InnerListOp::Delete(DeleteSpanWithId { + id_start, + span: DeleteSpan { + pos: pos as isize, + signed_len: len as isize, + }, + })) + } + json::TextOp::Mark { + start, + end, + style_key, + style_value, + info, + } => InnerContent::List(InnerListOp::StyleStart { + start, + end, + key: style_key.into(), + value: style_value, + info: TextStyleInfoFlag::from_byte(info), + }), + json::TextOp::MarkEnd => InnerContent::List(InnerListOp::StyleEnd), + }, + _ => { + return Err(LoroError::DecodeError( + "invalid op content for text container".into(), + )) + } + }, + ContainerType::List => match content { + JsonOpContent::List(list) => match list { + json::ListOp::Insert { + pos, + value: mut values, + } => { + for v in values.iter_mut() { + if let LoroValue::Container(id) = v { + if id.is_normal() { + *id = convert_container_id(id.clone(), peers)?; + } + } + } + let range = arena.alloc_values(values.iter().cloned()); + InnerContent::List(InnerListOp::Insert { + slice: SliceRange::new(range.start as u32..range.end as u32), + pos: pos as usize, + }) + } + json::ListOp::Delete { pos, len, start_id } => { + let start_id = convert_id(&start_id, peers)?; + validate_json_id_counter(start_id, "list delete start id")?; + InnerContent::List(InnerListOp::Delete(DeleteSpanWithId { + id_start: start_id, + span: DeleteSpan { + pos: pos as isize, + signed_len: len as isize, + }, + })) + } + }, + _ => { + return Err(LoroError::DecodeError( + "invalid op content for list container".into(), + )) + } + }, + ContainerType::MovableList => match content { + JsonOpContent::MovableList(list) => match list { + json::MovableListOp::Insert { + pos, + value: mut values, + } => { + for v in values.iter_mut() { + if let LoroValue::Container(id) = v { + if id.is_normal() { + *id = convert_container_id(id.clone(), peers)?; + } + } + } + let range = arena.alloc_values(values.iter().cloned()); + InnerContent::List(InnerListOp::Insert { + slice: SliceRange::new(range.start as u32..range.end as u32), + pos: pos as usize, + }) + } + json::MovableListOp::Delete { pos, len, start_id } => { + let start_id = convert_id(&start_id, peers)?; + validate_json_id_counter(start_id, "movable list delete start id")?; + InnerContent::List(InnerListOp::Delete(DeleteSpanWithId { + id_start: start_id, + span: DeleteSpan { + pos: pos as isize, + signed_len: len as isize, + }, + })) + } + json::MovableListOp::Move { + from, + elem_id: from_id, + to, + } => { + let from_id = convert_idlp(&from_id, peers)?; + InnerContent::List(InnerListOp::Move { + from, + elem_id: from_id, + to, + }) + } + json::MovableListOp::Set { elem_id, mut value } => { + let elem_id = convert_idlp(&elem_id, peers)?; + if let LoroValue::Container(id) = &mut value { + *id = convert_container_id(id.clone(), peers)?; + } + InnerContent::List(InnerListOp::Set { elem_id, value }) + } + }, + _ => { + return Err(LoroError::DecodeError( + "invalid op content for movable list container".into(), + )) + } + }, + ContainerType::Map => match content { + JsonOpContent::Map(map) => match map { + json::MapOp::Insert { key, mut value } => { + if let LoroValue::Container(id) = &mut value { + *id = convert_container_id(id.clone(), peers)?; + } + InnerContent::Map(MapSet { + key: key.into(), + value: Some(value), + }) + } + json::MapOp::Delete { key } => InnerContent::Map(MapSet { + key: key.into(), + value: None, + }), + }, + _ => { + return Err(LoroError::DecodeError( + "invalid op content for map container".into(), + )) + } + }, + ContainerType::Tree => match content { + JsonOpContent::Tree(tree) => match tree { + json::TreeOp::Create { + target, + parent, + fractional_index, + } => { + let target = convert_tree_id(&target, peers)?; + validate_json_tree_id_counter(&target, "tree create target")?; + let parent = match parent { + Some(p) => { + let parent = convert_tree_id(&p, peers)?; + validate_json_tree_id_counter(&parent, "tree create parent")?; + Some(parent) + } + None => None, + }; + InnerContent::Tree(Arc::new(TreeOp::Create { + target, + parent, + position: fractional_index, + })) + } + json::TreeOp::Move { + target, + parent, + fractional_index, + } => { + let target = convert_tree_id(&target, peers)?; + validate_json_tree_id_counter(&target, "tree move target")?; + let parent = match parent { + Some(p) => { + let parent = convert_tree_id(&p, peers)?; + validate_json_tree_id_counter(&parent, "tree move parent")?; + Some(parent) + } + None => None, + }; + InnerContent::Tree(Arc::new(TreeOp::Move { + target, + parent, + position: fractional_index, + })) + } + json::TreeOp::Delete { target } => { + let target = convert_tree_id(&target, peers)?; + validate_json_tree_id_counter(&target, "tree delete target")?; + InnerContent::Tree(Arc::new(TreeOp::Delete { target })) + } + }, + _ => { + return Err(LoroError::DecodeError( + "invalid op content for tree container".into(), + )) + } + }, + ContainerType::Unknown(_) => match content { + JsonOpContent::Future(json::FutureOpWrapper { + prop, + value: json::FutureOp::Unknown(value), + }) => InnerContent::Future(FutureInnerContent::Unknown { + prop, + value: Box::new(value), + }), + _ => { + return Err(LoroError::DecodeError( + "invalid op content for unknown container".into(), + )) + } + }, + #[cfg(feature = "counter")] + ContainerType::Counter => { + let JsonOpContent::Future(json::FutureOpWrapper { prop: _, value }) = content else { + return Err(LoroError::DecodeError( + "invalid op content for counter container".into(), + )); + }; + use crate::encoding::OwnedValue; + match value { + json::FutureOp::Counter(OwnedValue::F64(c)) + | json::FutureOp::Unknown(OwnedValue::F64(c)) => { + InnerContent::Future(FutureInnerContent::Counter(c)) + } + json::FutureOp::Counter(OwnedValue::I64(c)) + | json::FutureOp::Unknown(OwnedValue::I64(c)) => { + InnerContent::Future(FutureInnerContent::Counter(c as f64)) + } + _ => { + return Err(LoroError::DecodeError( + "invalid counter op value type".into(), + )) + } + } + } // Note: The Future Type need try to parse Op from the unknown content + }; + Ok(Op { + counter, + container: idx, + content, + }) +} + +impl TryFrom<&str> for JsonSchema { + type Error = serde_json::Error; + + fn try_from(value: &str) -> Result { + serde_json::from_str(value) + } +} + +impl TryFrom<&String> for JsonSchema { + type Error = serde_json::Error; + + fn try_from(value: &String) -> Result { + serde_json::from_str(value) + } +} + +impl TryFrom for JsonSchema { + type Error = serde_json::Error; + + fn try_from(value: String) -> Result { + serde_json::from_str(&value) + } +} + +pub mod json { + use crate::{ + encoding::OwnedValue, + version::{Frontiers, VersionRange}, + }; + use fractional_index::FractionalIndex; + use loro_common::{ContainerID, Counter, IdLp, Lamport, LoroValue, PeerID, TreeID, ID}; + use serde::{Deserialize, Serialize}; + use std::ops::Range; + + use super::{get_peer_from_peers, redact_value}; + + #[derive(Debug, Clone, Serialize, Deserialize)] + pub struct JsonSchema { + pub schema_version: u8, + #[serde(with = "self::serde_impl::frontiers")] + pub start_version: Frontiers, + #[serde(with = "self::serde_impl::peer_id")] + pub peers: Option>, + pub changes: Vec, + } + + #[derive(Debug, Clone, Serialize, Deserialize)] + pub struct JsonChange { + #[serde(with = "self::serde_impl::id")] + pub id: ID, + pub timestamp: i64, + #[serde(with = "self::serde_impl::deps")] + pub deps: Vec, + pub lamport: Lamport, + pub msg: Option, + pub ops: Vec, + } + + impl JsonChange { + pub fn checked_op_len(&self) -> Option { + let Some(last_op) = self.ops.last() else { + return Some(0); + }; + if last_op.counter < self.id.counter { + return None; + } + + let counter_delta = last_op.counter.checked_sub(self.id.counter)?; + let counter_delta = usize::try_from(counter_delta).ok()?; + counter_delta.checked_add(last_op.content.op_len()) + } + + pub fn op_len(&self) -> usize { + self.checked_op_len().unwrap_or(0) + } + } + + #[derive(Debug, Clone)] + pub struct JsonOp { + pub content: JsonOpContent, + pub container: ContainerID, + pub counter: i32, + } + + #[derive(Debug, Clone, Serialize, Deserialize)] + #[serde(untagged)] + pub enum JsonOpContent { + List(ListOp), + MovableList(MovableListOp), + Map(MapOp), + Text(TextOp), + Tree(TreeOp), + // #[serde(with = "self::serde_impl::future_op")] + Future(FutureOpWrapper), + } + + impl JsonOpContent { + pub fn op_len(&self) -> usize { + match self { + JsonOpContent::List(list_op) => list_op.op_len(), + JsonOpContent::MovableList(movable_list_op) => movable_list_op.op_len(), + JsonOpContent::Map(..) => 1, + JsonOpContent::Text(text_op) => text_op.op_len(), + JsonOpContent::Tree(..) => 1, + JsonOpContent::Future(..) => 1, + } + } + } + + #[derive(Debug, Clone, Serialize, Deserialize)] + pub struct FutureOpWrapper { + #[serde(flatten)] + pub value: FutureOp, + pub prop: i32, + } + + #[derive(Debug, Clone, Serialize, Deserialize)] + #[serde(tag = "type", rename_all = "snake_case")] + pub enum ListOp { + Insert { + pos: u32, + value: Vec, + }, + Delete { + pos: i32, + len: i32, + #[serde(with = "self::serde_impl::id")] + start_id: ID, + }, + } + + impl ListOp { + fn op_len(&self) -> usize { + match self { + ListOp::Insert { value: values, .. } => values.len(), + ListOp::Delete { len, .. } => (*len).unsigned_abs() as usize, + } + } + } + + #[derive(Debug, Clone, Serialize, Deserialize)] + #[serde(tag = "type", rename_all = "snake_case")] + pub enum MovableListOp { + Insert { + pos: u32, + value: Vec, + }, + Delete { + pos: i32, + len: i32, + #[serde(with = "self::serde_impl::id")] + start_id: ID, + }, + Move { + from: u32, + to: u32, + #[serde(with = "self::serde_impl::idlp")] + elem_id: IdLp, + }, + Set { + #[serde(with = "self::serde_impl::idlp")] + elem_id: IdLp, + value: LoroValue, + }, + } + + impl MovableListOp { + fn op_len(&self) -> usize { + match self { + MovableListOp::Insert { value: values, .. } => values.len(), + MovableListOp::Delete { len, .. } => (*len).unsigned_abs() as usize, + MovableListOp::Move { .. } => 1, + MovableListOp::Set { .. } => 1, + } + } + } + + #[derive(Debug, Clone, Serialize, Deserialize)] + #[serde(tag = "type", rename_all = "snake_case")] + pub enum MapOp { + Insert { key: String, value: LoroValue }, + Delete { key: String }, + } + + #[derive(Debug, Clone, Serialize, Deserialize)] + #[serde(tag = "type", rename_all = "snake_case")] + pub enum TextOp { + Insert { + pos: u32, + text: String, + }, + Delete { + pos: i32, + len: i32, + #[serde(with = "self::serde_impl::id")] + start_id: ID, + }, + Mark { + start: u32, + end: u32, + style_key: String, + style_value: LoroValue, + info: u8, + }, + MarkEnd, + } + + impl TextOp { + fn op_len(&self) -> usize { + match self { + TextOp::Insert { text, .. } => text.chars().count(), + TextOp::Delete { len, .. } => len.unsigned_abs() as usize, + TextOp::Mark { .. } => 1, + TextOp::MarkEnd => 1, + } + } + } + + #[derive(Debug, Clone, Serialize, Deserialize)] + #[serde(tag = "type", rename_all = "snake_case")] + pub enum TreeOp { + Create { + #[serde(with = "self::serde_impl::tree_id")] + target: TreeID, + #[serde(with = "self::serde_impl::option_tree_id")] + parent: Option, + #[serde(default, with = "self::serde_impl::fractional_index")] + fractional_index: FractionalIndex, + }, + Move { + #[serde(with = "self::serde_impl::tree_id")] + target: TreeID, + #[serde(with = "self::serde_impl::option_tree_id")] + parent: Option, + #[serde(default, with = "self::serde_impl::fractional_index")] + fractional_index: FractionalIndex, + }, + Delete { + #[serde(with = "self::serde_impl::tree_id")] + target: TreeID, + }, + } + + #[derive(Debug, Clone, Serialize, Deserialize)] + #[serde(tag = "type", rename_all = "snake_case")] + pub enum FutureOp { + #[cfg(feature = "counter")] + Counter(OwnedValue), + Unknown(OwnedValue), + } + + mod serde_impl { + + use loro_common::{ContainerID, ContainerType}; + use serde::{ + de::{Error, IgnoredAny, MapAccess, Visitor}, + ser::SerializeStruct, + Deserialize, Deserializer, Serialize, Serializer, + }; + + #[allow(unused_imports)] + use crate::encoding::OwnedValue; + + impl Serialize for super::JsonOp { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut s = serializer.serialize_struct("Op", 3)?; + s.serialize_field("container", &self.container.to_string())?; + s.serialize_field("content", &self.content)?; + s.serialize_field("counter", &self.counter)?; + s.end() + } + } + + impl<'de> Deserialize<'de> for super::JsonOp { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + struct __Visitor; + + impl<'de> Visitor<'de> for __Visitor { + type Value = super::JsonOp; + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("struct Op") + } + + fn visit_map(self, mut map: A) -> Result + where + A: MapAccess<'de>, + { + let mut container = None; + let mut content = None; + let mut counter = None; + + while let Some(key) = map.next_key::()? { + match key.as_str() { + "container" => { + if container.is_some() { + return Err(A::Error::duplicate_field("container")); + } + let value = map.next_value::()?; + container = + Some(ContainerID::try_from(value.as_str()).map_err( + |_| A::Error::custom("invalid container id"), + )?); + } + "content" => { + if content.is_some() { + return Err(A::Error::duplicate_field("content")); + } + content = Some(map.next_value::()?); + } + "counter" => { + if counter.is_some() { + return Err(A::Error::duplicate_field("counter")); + } + counter = Some(map.next_value::()?); + } + _ => { + let _ = map.next_value::()?; + } + } + } + + let container = + container.ok_or_else(|| A::Error::missing_field("container"))?; + let content = content.ok_or_else(|| A::Error::missing_field("content"))?; + let op = json_op_content_from_value(&container, content)?; + let counter = counter.ok_or_else(|| A::Error::missing_field("counter"))?; + Ok(super::JsonOp { + container, + content: op, + counter, + }) + } + } + const FIELDS: &[&str] = &["container", "content", "counter"]; + deserializer.deserialize_struct("JsonOp", FIELDS, __Visitor) + } + } + + fn json_op_content_from_value( + container: &ContainerID, + value: serde_json::Value, + ) -> Result { + match container.container_type() { + ContainerType::List => serde_json::from_value(value) + .map(super::JsonOpContent::List) + .map_err(E::custom), + ContainerType::MovableList => serde_json::from_value(value) + .map(super::JsonOpContent::MovableList) + .map_err(E::custom), + ContainerType::Map => serde_json::from_value(value) + .map(super::JsonOpContent::Map) + .map_err(E::custom), + ContainerType::Text => serde_json::from_value(value) + .map(super::JsonOpContent::Text) + .map_err(E::custom), + ContainerType::Tree => serde_json::from_value(value) + .map(super::JsonOpContent::Tree) + .map_err(E::custom), + ContainerType::Unknown(_) => serde_json::from_value(value) + .map(super::JsonOpContent::Future) + .map_err(E::custom), + #[cfg(feature = "counter")] + ContainerType::Counter => { + match serde_json::from_value::(value.clone()) { + Ok(op) => Ok(super::JsonOpContent::Future(op)), + Err(_) => { + let value = + serde_json::from_value::(value).map_err(E::custom)?; + Ok(super::JsonOpContent::Future(super::FutureOpWrapper { + prop: 0, + value: super::FutureOp::Counter(value), + })) + } + } + } + } + } + + pub mod id { + use loro_common::ID; + use serde::{Deserialize, Deserializer, Serializer}; + + pub fn serialize(id: &ID, s: S) -> Result + where + S: Serializer, + { + s.serialize_str(&id.to_string()) + } + + pub fn deserialize<'de, 'a, D>(d: D) -> Result + where + D: Deserializer<'de>, + { + // NOTE: https://github.com/serde-rs/serde/issues/2467 we use String here + let str: String = Deserialize::deserialize(d)?; + let id: ID = ID::try_from(str.as_str()) + .map_err(|_| serde::de::Error::custom("invalid ID format"))?; + Ok(id) + } + } + + pub mod frontiers { + use itertools::Itertools; + use loro_common::ID; + use serde::{ser::SerializeMap, Deserializer, Serializer}; + + use crate::version::Frontiers; + + pub fn serialize(f: &Frontiers, s: S) -> Result + where + S: Serializer, + { + let mut map = s.serialize_map(Some(f.len()))?; + for id in f + .iter() + // Make sure the order is deterministic + .sorted() + { + map.serialize_entry(&id.peer.to_string(), &id.counter)?; + } + map.end() + } + + pub fn deserialize<'de, 'a, D>(d: D) -> Result + where + D: Deserializer<'de>, + { + struct __Visitor; + impl<'de> serde::de::Visitor<'de> for __Visitor { + type Value = Frontiers; + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("a Frontiers") + } + + fn visit_map(self, mut map: A) -> Result + where + A: serde::de::MapAccess<'de>, + { + let mut f = Frontiers::default(); + while let Some((k, v)) = map.next_entry::()? { + f.push(ID::new( + k.parse().map_err(|_| { + serde::de::Error::custom("invalid peer id in frontiers") + })?, + v, + )) + } + Ok(f) + } + } + d.deserialize_map(__Visitor) + } + } + + pub mod deps { + use loro_common::ID; + use serde::{Deserialize, Deserializer, Serializer}; + + pub fn serialize(deps: &[ID], s: S) -> Result + where + S: Serializer, + { + s.collect_seq(deps.iter().map(|x| x.to_string())) + } + + pub fn deserialize<'de, 'a, D>(d: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + let deps: Vec = Deserialize::deserialize(d)?; + deps.into_iter() + .map(|x| { + ID::try_from(x.as_str()) + .map_err(|_| serde::de::Error::custom("invalid ID in deps")) + }) + .collect::, _>>() + } + } + + pub mod peer_id { + use loro_common::PeerID; + use serde::{Deserialize, Deserializer, Serializer}; + + pub fn serialize(peers: &Option>, s: S) -> Result + where + S: Serializer, + { + match peers { + Some(peers) => s.collect_seq(peers.iter().map(|x| x.to_string())), + None => s.serialize_none(), + } + } + + pub fn deserialize<'de, 'a, D>(d: D) -> Result>, D::Error> + where + D: Deserializer<'de>, + { + let peers: Option> = Deserialize::deserialize(d)?; + peers + .map(|x| { + x.into_iter() + .map(|x| { + x.parse() + .map_err(|_| serde::de::Error::custom("invalid peer id")) + }) + .collect::, _>>() + }) + .transpose() + } + } + + pub mod idlp { + use loro_common::IdLp; + use serde::{Deserialize, Deserializer, Serializer}; + + pub fn serialize(idlp: &IdLp, s: S) -> Result + where + S: Serializer, + { + s.serialize_str(&idlp.to_string()) + } + + pub fn deserialize<'de, 'a, D>(d: D) -> Result + where + D: Deserializer<'de>, + { + let str: String = Deserialize::deserialize(d)?; + let id: IdLp = IdLp::try_from(str.as_str()) + .map_err(|_| serde::de::Error::custom("invalid IdLp format"))?; + Ok(id) + } + } + + pub mod tree_id { + use loro_common::TreeID; + use serde::{Deserialize, Deserializer, Serializer}; + + pub fn serialize(id: &TreeID, s: S) -> Result + where + S: Serializer, + { + s.serialize_str(&id.to_string()) + } + + pub fn deserialize<'de, 'a, D>(d: D) -> Result + where + D: Deserializer<'de>, + { + let str: String = Deserialize::deserialize(d)?; + let id: TreeID = TreeID::try_from(str.as_str()) + .map_err(|_| serde::de::Error::custom("invalid TreeID format"))?; + Ok(id) + } + } + + pub mod option_tree_id { + use loro_common::TreeID; + use serde::{Deserialize, Deserializer, Serializer}; + + pub fn serialize(id: &Option, s: S) -> Result + where + S: Serializer, + { + match id { + Some(id) => s.serialize_str(&id.to_string()), + None => s.serialize_none(), + } + } + + pub fn deserialize<'de, 'a, D>(d: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + let str: Option = Deserialize::deserialize(d)?; + match str { + Some(str) => { + let id: TreeID = TreeID::try_from(str.as_str()) + .map_err(|_| serde::de::Error::custom("invalid TreeID format"))?; + Ok(Some(id)) + } + None => Ok(None), + } + } + } + + pub mod fractional_index { + use fractional_index::FractionalIndex; + use serde::{Deserialize, Deserializer, Serializer}; + + pub fn serialize(fi: &FractionalIndex, s: S) -> Result + where + S: Serializer, + { + s.serialize_str(&fi.to_string()) + } + + pub fn deserialize<'de, 'a, D>(d: D) -> Result + where + D: Deserializer<'de>, + { + let str: String = Deserialize::deserialize(d)?; + if !str.len().is_multiple_of(2) { + return Err(serde::de::Error::custom( + "invalid fractional index hex length", + )); + } + + let mut bytes = Vec::with_capacity(str.len() / 2); + for i in 0..str.len() / 2 { + let byte = u8::from_str_radix(&str[i * 2..i * 2 + 2], 16).map_err(|e| { + serde::de::Error::custom(format!("invalid fractional index hex: {e}")) + })?; + bytes.push(byte); + } + + Ok(FractionalIndex::from_bytes(bytes)) + } + } + } + + #[derive(thiserror::Error, Debug, PartialEq, Eq)] + pub enum RedactError { + #[error("unknown operation type")] + UnknownOperationType, + #[error("invalid schema: {0}")] + InvalidSchema(String), + } + + fn invalid_schema(message: impl Into) -> RedactError { + RedactError::InvalidSchema(message.into()) + } + + fn checked_redact_op_len(op: &JsonOp) -> Result { + let len = op.content.op_len(); + if len == 0 { + return Err(invalid_schema("op length must be greater than zero")); + } + + Counter::try_from(len).map_err(|_| invalid_schema("op length is too large")) + } + + fn validate_redactable_change(change: &JsonChange) -> Result { + if change.id.counter < 0 { + return Err(invalid_schema("change id counter must be non-negative")); + } + + let mut expected_counter = change.id.counter; + for op in &change.ops { + if op.counter < 0 { + return Err(invalid_schema("op counter must be non-negative")); + } + if op.counter != expected_counter { + return Err(invalid_schema( + "op counters must be contiguous with the change id", + )); + } + + let len = checked_redact_op_len(op)?; + expected_counter = expected_counter + .checked_add(len) + .ok_or_else(|| invalid_schema("op counter overflow"))?; + } + + let len = expected_counter + .checked_sub(change.id.counter) + .ok_or_else(|| invalid_schema("change counter overflow"))?; + usize::try_from(len).map_err(|_| invalid_schema("change length is too large")) + } + + /// Redacts sensitive content within the specified range by replacing it with default values. + /// + /// This method applies the following redaction rules: + /// + /// - Preserves delete and move operations without changes + /// - Replaces text insertion content with the Unicode replacement character (U+FFFD) + /// - Substitutes list and map insert values with `LoroValue::Null` + /// - Maintains child container creation operations + /// - Replaces text mark values with `LoroValue::Null` + /// - Preserves map insertion and text annotation keys + /// - Resets counter operations to zero + /// - Leaves unknown operation types (from future Loro versions) unchanged + /// + /// This approach ensures sensitive data removal while preserving the document's overall + /// structure. Redacted documents maintain seamless collaboration capabilities with both + /// redacted and non-redacted versions. + pub fn redact(json: &mut JsonSchema, range: VersionRange) -> Result<(), RedactError> { + let peers = json.peers.clone(); + let mut errors = Vec::new(); + for change in json.changes.iter_mut() { + let real_peer = get_peer_from_peers(&peers, change.id.peer) + .map_err(|_| RedactError::InvalidSchema("peer index out of range".to_string()))?; + let real_id = ID::new(real_peer, change.id.counter); + let change_len = validate_redactable_change(change)?; + if !range.has_overlap_with(real_id.to_span(change_len)) { + continue; + } + + let redact_range = range.get(&real_peer).copied().unwrap(); + for op in change.ops.iter_mut() { + if op.counter >= redact_range.1 { + break; + } + + let len = checked_redact_op_len(op)?; + let op_end = op + .counter + .checked_add(len) + .ok_or_else(|| invalid_schema("op counter overflow"))?; + if op_end <= redact_range.0 { + continue; + } + + let result = redact_op( + &mut op.content, + redact_range.0.saturating_sub(op.counter).max(0).min(len) + ..redact_range.1.saturating_sub(op.counter).max(0).min(len), + ); + match result { + Ok(()) => {} + Err(e) => errors.push(e), + } + } + } + + if errors.is_empty() { + Ok(()) + } else { + Err(errors.pop().unwrap()) + } + } + + fn redact_op(op: &mut JsonOpContent, range: Range) -> Result<(), RedactError> { + match op { + JsonOpContent::List(list_op) => { + match list_op { + ListOp::Insert { value: values, .. } => { + for i in range { + redact_value(&mut values[i as usize]); + } + } + ListOp::Delete { .. } => { + // Delete op won't be changed + } + } + } + JsonOpContent::MovableList(movable_list_op) => { + match movable_list_op { + MovableListOp::Insert { value: values, .. } => { + for i in range { + redact_value(&mut values[i as usize]); + } + } + MovableListOp::Delete { .. } | MovableListOp::Move { .. } => { + // Delete and move ops won't be changed + } + MovableListOp::Set { value, .. } => { + assert!(range.start == 0 && range.len() == 1); + redact_value(value); + } + } + } + JsonOpContent::Map(map_op) => { + match map_op { + MapOp::Insert { value, .. } => { + assert!(range.start == 0 && range.len() == 1); + redact_value(value); + } + MapOp::Delete { .. } => { + // Delete op won't be changed + } + } + } + JsonOpContent::Text(text_op) => { + match text_op { + TextOp::Insert { text, .. } => { + let mut chars = vec![]; + for (i, c) in text.chars().enumerate() { + if i < range.start as usize || i >= range.end as usize { + chars.push(c); + } else { + chars.push("� ".chars().next().unwrap()); + } + } + *text = chars.into_iter().collect(); + } + TextOp::Delete { .. } => { + // Delete op won't be changed + } + TextOp::Mark { style_value, .. } => { + assert!(range.start == 0 && range.len() == 1); + *style_value = LoroValue::Null; + } + TextOp::MarkEnd => { + // MarkEnd won't be changed + } + } + } + JsonOpContent::Tree(..) => { + // Creation of child container won't be changed + } + JsonOpContent::Future(future_op_wrapper) => match &mut future_op_wrapper.value { + #[cfg(feature = "counter")] + FutureOp::Counter(owned_value) => { + *owned_value = OwnedValue::I64(0); + } + FutureOp::Unknown(..) => { + return Err(RedactError::UnknownOperationType); + } + }, + } + + Ok(()) + } +} + +fn redact_value(v: &mut LoroValue) { + match v { + LoroValue::Container(_) => {} + _ => *v = LoroValue::Null, + } +} + +#[cfg(test)] +mod tests { + use crate::{LoroDoc, VersionVector}; + + #[test] + fn json_range_version() { + let doc = LoroDoc::new_auto_commit(); + doc.set_peer_id(0).unwrap(); + let list = doc.get_list("list"); + list.insert(0, "a").unwrap(); + list.insert(0, "b").unwrap(); + list.insert(0, "c").unwrap(); + let json = doc.export_json_updates( + &VersionVector::from_iter(vec![(0, 1)]), + &VersionVector::from_iter(vec![(0, 2)]), + true, + ); + assert_eq!(json.changes[0].ops.len(), 1); + let json = doc.export_json_updates( + &VersionVector::from_iter(vec![(0, 0)]), + &VersionVector::from_iter(vec![(0, 2)]), + true, + ); + assert_eq!(json.changes[0].ops.len(), 2); + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/encoding/outdated_encode_reordered.rs b/3rdparty/loro/crates/loro-internal/src/encoding/outdated_encode_reordered.rs new file mode 100644 index 0000000000..703393976b --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/encoding/outdated_encode_reordered.rs @@ -0,0 +1,575 @@ +pub(crate) use encode::{encode_op, get_op_prop}; +use fractional_index::FractionalIndex; +use loro_common::{ + ContainerID, ContainerType, Counter, HasCounterSpan, HasIdSpan, IdLp, LoroError, LoroResult, + TreeID, ID, +}; +use serde_columnar::{columnar, ColumnarError}; +use std::sync::Arc; + +use crate::version::VersionRange; +use crate::{ + arena::SharedArena, + change::{Change, Lamport}, + container::{ + list::list_op::DeleteSpanWithId, richtext::TextStyleInfoFlag, tree::tree_op::TreeOp, + }, + op::{FutureInnerContent, SliceRange}, + OpLog, +}; + +use super::value::{Value, ValueDecodedArenasTrait}; + +pub(crate) use crate::encoding::value_register::ValueRegister; + +#[allow(unused_imports)] +use super::value::FutureValue; + +/// If any collection in the document is longer than this, we will not decode it. +/// It will return an data corruption error instead. +pub(super) const MAX_COLLECTION_SIZE: usize = 1 << 28; + +pub(crate) struct ImportChangesResult { + pub latest_ids: Vec, + pub pending_changes: Vec, + pub changes_that_have_deps_before_shallow_root: Vec, + pub imported: VersionRange, +} + +/// NOTE: This method expects that the remote_changes are already sorted by lamport value +pub(crate) fn import_changes_to_oplog( + changes: Vec, + oplog: &mut OpLog, +) -> ImportChangesResult { + let mut pending_changes = Vec::new(); + let mut latest_ids = Vec::new(); + let mut changes_before_shallow_root = Vec::new(); + let mut imported = VersionRange::default(); + for mut change in changes { + if change.ctr_end() <= oplog.vv().get(&change.id.peer).copied().unwrap_or(0) { + // skip included changes + continue; + } + + if oplog.dag.import_deps_before_shallow_root(&change.deps) { + changes_before_shallow_root.push(change); + continue; + } + + latest_ids.push(change.id_last()); + // calc lamport or pending if its deps are not satisfied + match oplog.dag.get_change_lamport_from_deps(&change.deps) { + Some(lamport) => change.lamport = lamport, + None => { + pending_changes.push(change); + continue; + } + } + + let Some(change) = oplog.trim_the_known_part_of_change(change) else { + continue; + }; + + imported.extends_to_include_id_span(change.id_span()); + oplog.insert_new_change(change, false); + } + + ImportChangesResult { + latest_ids, + pending_changes, + changes_that_have_deps_before_shallow_root: changes_before_shallow_root, + imported, + } +} + +mod encode { + use loro_common::ContainerType; + + use crate::{ + arena::SharedArena, + encoding::value::{MarkStart, Value, ValueEncodeRegister, ValueKind, ValueWriter}, + op::{FutureInnerContent, Op}, + }; + + use super::EncodedDeleteStartId; + + fn get_future_op_prop(op: &FutureInnerContent) -> i32 { + match op { + #[cfg(feature = "counter")] + FutureInnerContent::Counter(_) => 0, + FutureInnerContent::Unknown { prop, .. } => *prop, + } + } + + pub(crate) fn get_op_prop(op: &Op, registers: &mut dyn ValueEncodeRegister) -> i32 { + match &op.content { + crate::op::InnerContent::List(list) => match list { + crate::container::list::list_op::InnerListOp::Move { to, .. } => *to as i32, + crate::container::list::list_op::InnerListOp::Set { .. } => 0, + crate::container::list::list_op::InnerListOp::Insert { pos, .. } => *pos as i32, + crate::container::list::list_op::InnerListOp::InsertText { pos, .. } => *pos as i32, + crate::container::list::list_op::InnerListOp::Delete(span) => span.span.pos as i32, + crate::container::list::list_op::InnerListOp::StyleStart { start, .. } => { + *start as i32 + } + crate::container::list::list_op::InnerListOp::StyleEnd => 0, + }, + crate::op::InnerContent::Map(map) => { + let key = registers.key_mut().register(&map.key); + key as i32 + } + crate::op::InnerContent::Tree(_) => 0, + crate::op::InnerContent::Future(f) => get_future_op_prop(f), + } + } + + #[inline] + pub(crate) fn encode_op<'p, 'a: 'p>( + op: &'a Op, + arena: &SharedArena, + delete_start: &mut Vec, + value_writer: &mut ValueWriter, + registers: &mut dyn ValueEncodeRegister, + ) -> ValueKind { + let value = match &op.content { + crate::op::InnerContent::List(list) => match list { + crate::container::list::list_op::InnerListOp::Insert { slice, .. } => { + assert!(matches!( + op.container.get_type(), + ContainerType::List | ContainerType::MovableList + )); + let value = arena.get_values(slice.0.start as usize..slice.0.end as usize); + Value::LoroValue(value.into()) + } + crate::container::list::list_op::InnerListOp::InsertText { slice, .. } => { + Value::Str(std::str::from_utf8(slice.as_bytes()).unwrap()) + } + crate::container::list::list_op::InnerListOp::Delete(span) => { + delete_start.push(EncodedDeleteStartId { + peer_idx: registers.peer_mut().register(&span.id_start.peer), + counter: span.id_start.counter, + len: span.span.signed_len, + }); + Value::DeleteSeq + } + crate::container::list::list_op::InnerListOp::StyleStart { + start, + end, + key, + value, + info, + } => Value::MarkStart(MarkStart { + len: *end - *start, + key: key.clone(), + value: value.clone(), + info: info.to_byte(), + }), + crate::container::list::list_op::InnerListOp::Set { elem_id, value } => { + Value::ListSet { + peer_idx: registers.peer_mut().register(&elem_id.peer), + lamport: elem_id.lamport, + value: value.clone(), + } + } + crate::container::list::list_op::InnerListOp::StyleEnd => Value::Null, + crate::container::list::list_op::InnerListOp::Move { + from, + elem_id: from_id, + to: _, + } => Value::ListMove { + from: *from as usize, + from_idx: registers.peer_mut().register(&from_id.peer), + lamport: from_id.lamport as usize, + }, + }, + crate::op::InnerContent::Map(map) => { + assert_eq!(op.container.get_type(), ContainerType::Map); + match &map.value { + Some(v) => Value::LoroValue(v.clone()), + None => Value::DeleteOnce, + } + } + crate::op::InnerContent::Tree(t) => { + assert_eq!(op.container.get_type(), ContainerType::Tree); + registers.encode_tree_op(t) + } + crate::op::InnerContent::Future(f) => match f { + #[cfg(feature = "counter")] + FutureInnerContent::Counter(c) => { + let c_abs = c.abs(); + if c_abs.fract() < f64::EPSILON && (c_abs as i64) < (2 << 26) { + Value::I64(*c as i64) + } else { + Value::F64(*c) + } + } + FutureInnerContent::Unknown { value, .. } => Value::from_owned(value), + }, + }; + let (k, _) = value.encode(value_writer, registers); + k + } +} + +#[allow(clippy::too_many_arguments)] +pub(crate) fn decode_op( + cid: &ContainerID, + value: Value<'_>, + del_iter: &mut impl Iterator>, + shared_arena: &SharedArena, + arenas: &dyn ValueDecodedArenasTrait, + positions: &[Vec], + prop: i32, + op_id: ID, +) -> LoroResult { + let content = match cid.container_type() { + ContainerType::Text => match value { + Value::Str(s) => { + let (slice, result) = shared_arena.alloc_str_with_slice(s); + crate::op::InnerContent::List( + crate::container::list::list_op::InnerListOp::InsertText { + slice, + unicode_start: result.start as u32, + unicode_len: (result.end - result.start) as u32, + pos: prop as u32, + }, + ) + } + Value::DeleteSeq => { + let del_start = del_iter + .next() + .ok_or(LoroError::DecodeDataCorruptionError)??; + let peer_idx = del_start.peer_idx; + let cnt = del_start.counter; + let len = del_start.len; + crate::op::InnerContent::List(crate::container::list::list_op::InnerListOp::Delete( + DeleteSpanWithId::new( + ID::new( + *arenas + .peers() + .get(peer_idx) + .ok_or(LoroError::DecodeDataCorruptionError)?, + cnt as Counter, + ), + prop as isize, + len, + ), + )) + } + Value::MarkStart(mark) => crate::op::InnerContent::List( + crate::container::list::list_op::InnerListOp::StyleStart { + start: prop as u32, + end: prop as u32 + mark.len, + key: mark.key, + value: mark.value, + info: TextStyleInfoFlag::from_byte(mark.info), + }, + ), + Value::Null => crate::op::InnerContent::List( + crate::container::list::list_op::InnerListOp::StyleEnd, + ), + _ => return Err(LoroError::DecodeDataCorruptionError), + }, + ContainerType::Map => { + let key = arenas + .keys() + .get(prop as usize) + .ok_or(LoroError::DecodeDataCorruptionError)? + .clone(); + match value { + Value::DeleteOnce => { + crate::op::InnerContent::Map(crate::container::map::MapSet { key, value: None }) + } + Value::LoroValue(v) => { + crate::op::InnerContent::Map(crate::container::map::MapSet { + key, + value: Some(v.clone()), + }) + } + _ => return Err(LoroError::DecodeDataCorruptionError), + } + } + ContainerType::List => { + let pos = prop as usize; + match value { + Value::LoroValue(arr) => { + let list = arr + .into_list() + .map_err(|_| LoroError::DecodeDataCorruptionError)?; + let range = shared_arena.alloc_values(list.iter().cloned()); + crate::op::InnerContent::List( + crate::container::list::list_op::InnerListOp::Insert { + slice: SliceRange::new(range.start as u32..range.end as u32), + pos, + }, + ) + } + Value::DeleteSeq => { + let del_start = del_iter + .next() + .ok_or(LoroError::DecodeDataCorruptionError)??; + let peer_idx = del_start.peer_idx; + let cnt = del_start.counter; + let len = del_start.len; + crate::op::InnerContent::List( + crate::container::list::list_op::InnerListOp::Delete( + DeleteSpanWithId::new( + ID::new( + *arenas + .peers() + .get(peer_idx) + .ok_or(LoroError::DecodeDataCorruptionError)?, + cnt as Counter, + ), + pos as isize, + len, + ), + ), + ) + } + _ => return Err(LoroError::DecodeDataCorruptionError), + } + } + ContainerType::Tree => match value { + Value::TreeMove(op) => crate::op::InnerContent::Tree(Arc::new( + arenas.decode_tree_op(positions, op, op_id)?, + )), + Value::RawTreeMove(op) => { + let subject = TreeID::new( + *arenas + .peers() + .get(op.subject_peer_idx) + .ok_or(LoroError::DecodeDataCorruptionError)?, + op.subject_cnt as Counter, + ); + let parent = if op.is_parent_null { + None + } else { + let parent_id = TreeID::new( + *arenas + .peers() + .get(op.parent_peer_idx) + .ok_or(LoroError::DecodeDataCorruptionError)?, + op.parent_cnt as Counter, + ); + if parent_id.is_deleted_root() { + return Ok(crate::op::InnerContent::Tree(Arc::new(TreeOp::Delete { + target: subject, + }))); + } + + Some(parent_id) + }; + + let fi = FractionalIndex::from_bytes( + positions + .get(op.position_idx) + .ok_or(LoroError::DecodeDataCorruptionError)? + .clone(), + ); + let is_create = subject.id() == op_id; + let ans = if is_create { + TreeOp::Create { + target: subject, + parent, + position: fi, + } + } else { + TreeOp::Move { + target: subject, + parent, + position: fi, + } + }; + crate::op::InnerContent::Tree(Arc::new(ans)) + } + _ => return Err(LoroError::DecodeDataCorruptionError), + }, + ContainerType::MovableList => { + let pos = prop as usize; + match value { + Value::LoroValue(arr) => { + let list = arr + .into_list() + .map_err(|_| LoroError::DecodeDataCorruptionError)?; + let range = shared_arena.alloc_values(list.iter().cloned()); + crate::op::InnerContent::List( + crate::container::list::list_op::InnerListOp::Insert { + slice: SliceRange::new(range.start as u32..range.end as u32), + pos, + }, + ) + } + Value::DeleteSeq => { + let del_start = del_iter + .next() + .ok_or(LoroError::DecodeDataCorruptionError)??; + let peer_idx = del_start.peer_idx; + let cnt = del_start.counter; + let len = del_start.len; + crate::op::InnerContent::List( + crate::container::list::list_op::InnerListOp::Delete( + DeleteSpanWithId::new( + ID::new( + *arenas + .peers() + .get(peer_idx) + .ok_or(LoroError::DecodeDataCorruptionError)?, + cnt as Counter, + ), + pos as isize, + len, + ), + ), + ) + } + Value::ListMove { + from, + from_idx, + lamport, + } => crate::op::InnerContent::List( + crate::container::list::list_op::InnerListOp::Move { + from: from as u32, + elem_id: IdLp::new( + *arenas + .peers() + .get(from_idx) + .ok_or(LoroError::DecodeDataCorruptionError)?, + lamport as Lamport, + ), + to: prop as u32, + }, + ), + Value::ListSet { + peer_idx, + lamport, + value, + } => crate::op::InnerContent::List( + crate::container::list::list_op::InnerListOp::Set { + elem_id: IdLp::new( + *arenas + .peers() + .get(peer_idx) + .ok_or(LoroError::DecodeDataCorruptionError)?, + lamport as Lamport, + ), + value, + }, + ), + _ => return Err(LoroError::DecodeDataCorruptionError), + } + } + #[cfg(feature = "counter")] + ContainerType::Counter => match value { + Value::F64(c) => crate::op::InnerContent::Future(FutureInnerContent::Counter(c)), + Value::I64(c) => crate::op::InnerContent::Future(FutureInnerContent::Counter(c as f64)), + _ => return Err(LoroError::DecodeDataCorruptionError), + }, + // NOTE: The future container type need also try to parse the unknown type + ContainerType::Unknown(_) => crate::op::InnerContent::Future(FutureInnerContent::Unknown { + prop, + value: Box::new(value.into_owned()), + }), + }; + + Ok(content) +} + +pub type PeerIdx = usize; + +#[columnar(vec, ser, de, iterable)] +#[derive(Debug, Clone)] +pub(crate) struct EncodedDeleteStartId { + #[columnar(strategy = "DeltaRle")] + peer_idx: usize, + #[columnar(strategy = "DeltaRle")] + counter: i32, + #[columnar(strategy = "DeltaRle")] + len: isize, +} + +#[cfg(test)] +mod test { + + use loro_common::LoroValue; + use rustc_hash::FxHashSet; + + use crate::{ + encoding::{ + arena::EncodedRegisters, + value::{ValueReader, ValueWriter}, + value_register::ValueRegister, + }, + fx_map, + }; + + use super::*; + + fn test_loro_value_read_write(v: impl Into, container_id: Option) { + let v = v.into(); + let id = match &container_id { + Some(ContainerID::Root { .. }) => ID::new(u64::MAX, 0), + Some(ContainerID::Normal { peer, counter, .. }) => ID::new(*peer, *counter), + None => ID::new(u64::MAX, 0), + }; + + let mut registers = EncodedRegisters { + key: ValueRegister::new(), + peer: ValueRegister::new(), + tree_id: ValueRegister::new(), + position: either::Either::Left(FxHashSet::default()), + }; + + let mut writer = ValueWriter::new(); + let (kind, _) = writer.write_value_content(&v, &mut registers); + + let binding = writer.finish(); + let mut reader = ValueReader::new(binding.as_slice()); + + let ans = reader + .read_value_content(kind, ®isters.key.unwrap_vec(), id) + .unwrap(); + assert_eq!(v, ans) + } + + #[test] + fn test_value_read_write() { + test_loro_value_read_write(true, None); + test_loro_value_read_write(false, None); + test_loro_value_read_write(123, None); + test_loro_value_read_write(1.23, None); + test_loro_value_read_write(LoroValue::Null, None); + test_loro_value_read_write( + LoroValue::Binary((vec![123, 223, 255, 0, 1, 2, 3]).into()), + None, + ); + test_loro_value_read_write("sldk;ajfas;dlkfas测试", None); + // we won't encode root container by `value content` + // test_loro_value_read_write( + // LoroValue::Container(ContainerID::new_root("name", ContainerType::Text)), + // Some(ContainerID::new_root("name", ContainerType::Text)), + // ); + test_loro_value_read_write( + LoroValue::Container(ContainerID::new_normal( + ID::new(u64::MAX, 123), + ContainerType::Tree, + )), + Some(ContainerID::new_normal( + ID::new(u64::MAX, 123), + ContainerType::Tree, + )), + ); + test_loro_value_read_write(vec![1i32, 2, 3], None); + test_loro_value_read_write( + LoroValue::Map( + (fx_map![ + "1".into() => 123.into(), + "2".into() => "123".into(), + "3".into() => vec![true].into() + ]) + .into(), + ), + None, + ); + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/encoding/shallow_snapshot.rs b/3rdparty/loro/crates/loro-internal/src/encoding/shallow_snapshot.rs new file mode 100644 index 0000000000..03ffb4f968 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/encoding/shallow_snapshot.rs @@ -0,0 +1,462 @@ +use bytes::Bytes; +use rle::HasLength; +use std::collections::BTreeSet; + +use loro_common::{ContainerID, ContainerType, LoroEncodeError, LoroError, ID}; + +use crate::{ + container::list::list_op::InnerListOp, + dag::DagUtils, + encoding::fast_snapshot::{_encode_snapshot, Snapshot}, + state::container_store::FRONTIERS_KEY, + version::{Frontiers, VersionVector}, + LoroDoc, +}; + +#[cfg(test)] +const MAX_OPS_NUM_TO_ENCODE_WITHOUT_LATEST_STATE: usize = 16; +#[cfg(not(test))] +const MAX_OPS_NUM_TO_ENCODE_WITHOUT_LATEST_STATE: usize = 256; + +#[tracing::instrument(skip_all)] +pub(crate) fn export_shallow_snapshot( + doc: &LoroDoc, + start_from: &Frontiers, + w: &mut W, +) -> Result { + let (snapshot, start_from) = export_shallow_snapshot_inner(doc, start_from)?; + _encode_snapshot(snapshot, w); + Ok(start_from) +} + +pub(crate) fn export_shallow_snapshot_inner( + doc: &LoroDoc, + start_from: &Frontiers, +) -> Result<(Snapshot, Frontiers), LoroEncodeError> { + let oplog = doc.oplog().lock(); + let start_from = calc_shallow_doc_start(&oplog, start_from); + let mut start_vv = frontiers_to_vv_for_export(&oplog, &start_from, "export_shallow_snapshot")?; + for id in start_from.iter() { + // we need to include the ops in start_from, this can make things easier + start_vv.insert(id.peer, id.counter); + } + + #[cfg(debug_assertions)] + { + use crate::dag::Dag; + if !start_from.is_empty() { + assert!(start_from.len() == 1); + let id = start_from.as_single().unwrap(); + let node = oplog.dag.get(id).unwrap(); + if id.counter == node.cnt { + let vv = oplog.dag().frontiers_to_vv(&node.deps).unwrap(); + assert_eq!(vv, start_vv); + } else { + let vv = oplog + .dag() + .frontiers_to_vv(&Frontiers::from(id.inc(-1))) + .unwrap(); + assert_eq!(vv, start_vv); + } + } + } + + loro_common::debug!( + "start version vv={:?} frontiers={:?}", + &start_vv, + &start_from, + ); + + let latest_frontiers = oplog.frontiers().clone(); + let state_frontiers = doc.state_frontiers(); + let is_attached = !doc.is_detached(); + let oplog_bytes = oplog.export_change_store_from(&start_vv, &start_from); + let latest_vv = oplog.vv(); + let ops_num: usize = latest_vv.sub_iter(&start_vv).map(|x| x.atom_len()).sum(); + if &start_from == oplog.shallow_since_frontiers() + && state_frontiers == latest_frontiers + && ops_num <= MAX_OPS_NUM_TO_ENCODE_WITHOUT_LATEST_STATE + { + let state = doc.app_state().lock(); + if let Some(shallow_root_state_bytes) = state.store.encode_shallow_root_state() { + return Ok(( + Snapshot { + oplog_bytes, + state_bytes: None, + shallow_root_state_bytes, + }, + start_from, + )); + } + } + if &start_from == oplog.shallow_since_frontiers() + && state_frontiers == latest_frontiers + && ops_num > MAX_OPS_NUM_TO_ENCODE_WITHOUT_LATEST_STATE + { + let mut state = doc.app_state().lock(); + if let Some((shallow_root_state_bytes, shallow_root_state_kv)) = + state.store.shallow_root_state_for_export() + { + let mut alive_c_bytes = shallow_root_state_kv.keys(); + if has_unknown_container_key(alive_c_bytes.iter()) { + return Err(LoroEncodeError::UnknownContainer); + } + + state.ensure_all_alive_containers(); + state.store.flush(); + + // All the containers that are created after start_from need to be encoded. + for cid in state.store.iter_all_container_ids() { + if let ContainerID::Normal { peer, counter, .. } = cid { + let temp_id = ID::new(peer, counter); + if !start_from.contains(&temp_id) { + alive_c_bytes.insert(cid.to_bytes()); + } + } else { + alive_c_bytes.insert(cid.to_bytes()); + } + } + + let new_kv = state.store.get_kv_clone(); + new_kv.remove_same(&shallow_root_state_kv); + new_kv.retain_keys(&alive_c_bytes); + + return Ok(( + Snapshot { + oplog_bytes, + state_bytes: Some(new_kv.export()), + shallow_root_state_bytes, + }, + start_from, + )); + } + } + drop(oplog); + let result = (|| -> Result { + doc._checkout_without_emitting(&start_from, false, false) + .map_err(LoroEncodeError::from)?; + let mut state = doc.app_state().lock(); + let alive_containers = state.ensure_all_alive_containers(); + if has_unknown_container(alive_containers.iter()) { + return Err(LoroEncodeError::UnknownContainer); + } + let mut alive_c_bytes: BTreeSet> = + alive_containers.iter().map(|x| x.to_bytes()).collect(); + state.store.flush(); + let shallow_root_state_kv = state.store.get_kv_clone(); + drop(state); + doc._checkout_without_emitting(&latest_frontiers, false, false) + .map_err(LoroEncodeError::from)?; + let state_bytes = if ops_num > MAX_OPS_NUM_TO_ENCODE_WITHOUT_LATEST_STATE { + let mut state = doc.app_state().lock(); + state.ensure_all_alive_containers(); + state.store.encode(); + // All the containers that are created after start_from need to be encoded + for cid in state.store.iter_all_container_ids() { + if let ContainerID::Normal { peer, counter, .. } = cid { + let temp_id = ID::new(peer, counter); + if !start_from.contains(&temp_id) { + alive_c_bytes.insert(cid.to_bytes()); + } + } else { + alive_c_bytes.insert(cid.to_bytes()); + } + } + + let new_kv = state.store.get_kv_clone(); + new_kv.remove_same(&shallow_root_state_kv); + new_kv.retain_keys(&alive_c_bytes); + Some(new_kv.export()) + } else { + None + }; + + shallow_root_state_kv.retain_keys(&alive_c_bytes); + shallow_root_state_kv.insert(FRONTIERS_KEY, start_from.encode().into()); + let shallow_root_state_bytes = shallow_root_state_kv.export(); + + Ok(Snapshot { + oplog_bytes, + state_bytes, + shallow_root_state_bytes, + }) + })(); + + restore_export_doc_state(doc, &state_frontiers, is_attached)?; + doc.drop_pending_events(); + Ok((result?, start_from)) +} + +fn has_unknown_container<'a>(mut cids: impl Iterator) -> bool { + cids.any(|cid| matches!(cid.container_type(), ContainerType::Unknown(_))) +} + +fn has_unknown_container_key<'a>(mut keys: impl Iterator>) -> bool { + keys.any(|key| ContainerID::from_bytes(key).is_unknown()) +} + +pub(crate) fn export_state_only_snapshot( + doc: &LoroDoc, + target_frontiers: &Frontiers, + w: &mut W, +) -> Result { + let oplog = doc.oplog().lock(); + let start_from = calc_shallow_doc_start(&oplog, target_frontiers); + let mut start_vv = + frontiers_to_vv_for_export(&oplog, &start_from, "export_state_only_snapshot")?; + for id in start_from.iter() { + // we need to include the ops in start_from, this can make things easier + start_vv.insert(id.peer, id.counter); + } + + loro_common::debug!( + "start version vv={:?} frontiers={:?}", + &start_vv, + &start_from, + ); + + let to_vv = frontiers_to_vv_for_export(&oplog, target_frontiers, "export_state_only_snapshot")?; + let oplog_bytes = + oplog.export_change_store_in_range(&start_vv, &start_from, &to_vv, target_frontiers); + let state_frontiers = doc.state_frontiers(); + let is_attached = !doc.is_detached(); + drop(oplog); + let result = (|| -> Result<(), LoroEncodeError> { + doc._checkout_without_emitting(&start_from, false, false) + .map_err(LoroEncodeError::from)?; + let mut state = doc.app_state().lock(); + let alive_containers = state.ensure_all_alive_containers(); + if has_unknown_container(alive_containers.iter()) { + return Err(LoroEncodeError::UnknownContainer); + } + let mut alive_c_bytes = cids_to_bytes(alive_containers); + state.store.flush(); + let shallow_state_kv = state.store.get_kv_clone(); + drop(state); + + doc._checkout_without_emitting(target_frontiers, false, false) + .map_err(LoroEncodeError::from)?; + let mut state = doc.app_state().lock(); + state.ensure_all_alive_containers(); + state.store.encode(); + for cid in state.store.iter_all_container_ids() { + if let ContainerID::Normal { peer, counter, .. } = cid { + let temp_id = ID::new(peer, counter); + if !start_from.contains(&temp_id) { + alive_c_bytes.insert(cid.to_bytes()); + } + } else { + alive_c_bytes.insert(cid.to_bytes()); + } + } + + let target_state_kv = state.store.get_kv_clone(); + drop(state); + target_state_kv.remove_same(&shallow_state_kv); + target_state_kv.retain_keys(&alive_c_bytes); + + shallow_state_kv.retain_keys(&alive_c_bytes); + shallow_state_kv.insert(FRONTIERS_KEY, start_from.encode().into()); + let shallow_state_bytes = shallow_state_kv.export(); + let snapshot = Snapshot { + oplog_bytes, + state_bytes: Some(target_state_kv.export()), + shallow_root_state_bytes: shallow_state_bytes, + }; + _encode_snapshot(snapshot, w); + Ok(()) + })(); + + restore_export_doc_state(doc, &state_frontiers, is_attached)?; + doc.drop_pending_events(); + result?; + Ok(start_from) +} + +fn cids_to_bytes( + alive_containers: std::collections::HashSet, +) -> BTreeSet> { + let alive_c_bytes: BTreeSet> = alive_containers.iter().map(|x| x.to_bytes()).collect(); + alive_c_bytes +} + +fn frontiers_to_vv_for_export( + oplog: &crate::OpLog, + frontiers: &Frontiers, + context: &str, +) -> Result { + oplog.dag().frontiers_to_vv(frontiers).ok_or_else(|| { + LoroEncodeError::FrontiersNotFound(format!( + "{context}: unreachable frontiers {frontiers:?}" + )) + }) +} + +fn restore_export_doc_state( + doc: &LoroDoc, + state_frontiers: &Frontiers, + was_attached: bool, +) -> Result<(), LoroEncodeError> { + if &doc.state_frontiers() != state_frontiers { + doc._checkout_without_emitting(state_frontiers, false, false) + .map_err(LoroEncodeError::from)?; + } + + if was_attached { + doc.set_detached(false); + } + + Ok(()) +} + +/// Calculates optimal starting version for the shallow doc +/// +/// It should be the LCA of the user given version and the latest version. +/// Otherwise, users cannot replay the history from the initial version till the latest version. +fn calc_shallow_doc_start(oplog: &crate::OpLog, frontiers: &Frontiers) -> Frontiers { + // Find the LCA of the given frontiers by iteratively pairwise GCA. + // This converges to a single frontier or empty if there is no common ancestor. + let mut current = frontiers.clone(); + while current.len() > 1 { + let ids: Vec = current.iter().collect(); + let mut next = Frontiers::new(); + let mut i = 0; + while i < ids.len() { + if i + 1 < ids.len() { + let (gca, _) = oplog + .dag() + .find_common_ancestor(&Frontiers::from(ids[i]), &Frontiers::from(ids[i + 1])); + for id in gca.iter() { + next.push(id); + } + } else { + next.push(ids[i]); + } + i += 2; + } + if next == current { + // Cannot converge further (pairwise GCAs are the nodes themselves). + // Fall back to empty frontiers, meaning export full history. + return clamp_to_shallow_root(oplog, Frontiers::default()); + } + current = next; + } + + let mut ans = Frontiers::new(); + for id in current.iter() { + let mut processed = false; + if let Some(op) = oplog.get_op_that_includes(id) { + if let crate::op::InnerContent::List(InnerListOp::StyleStart { .. }) = &op.content { + // StyleStart and StyleEnd operations must be kept together in the GC snapshot. + // Splitting them could lead to an weird document state that cannot be + // properly encoded. To ensure they stay together, we advance the frontier by + // one step to include both operations. + + // > Id.counter + 1 is guaranteed to be the StyleEnd Op + ans.push(id.inc(1)); + processed = true; + } + } + + if !processed { + ans.push(id); + } + } + + clamp_to_shallow_root(oplog, ans) +} + +fn clamp_to_shallow_root(oplog: &crate::OpLog, frontiers: Frontiers) -> Frontiers { + if oplog.shallow_since_vv().is_empty() { + return frontiers; + } + + let Some(vv) = oplog.dag().frontiers_to_vv(&frontiers) else { + return oplog.shallow_since_frontiers().clone(); + }; + + if vv.includes_vv(&oplog.shallow_since_vv().to_vv()) { + frontiers + } else { + oplog.shallow_since_frontiers().clone() + } +} + +pub(crate) fn encode_snapshot_at( + doc: &LoroDoc, + frontiers: &Frontiers, + w: &mut W, +) -> Result<(), LoroEncodeError> { + let was_detached = doc.is_detached(); + let version_before_start = doc.state_frontiers().clone(); + doc._checkout_without_emitting(frontiers, true, false) + .map_err(LoroEncodeError::from)?; + let result = 'block: { + let oplog = doc.oplog().lock(); + let mut state = doc.app_state().lock(); + let is_shallow = state.store.shallow_root_store().is_some(); + if is_shallow { + break 'block Err(LoroEncodeError::from(LoroError::NotImplemented( + "fork_at on shallow docs", + ))); + } + + if state.is_in_txn() { + break 'block Err(LoroEncodeError::internal( + "encode_snapshot_at: state is unexpectedly still in a transaction", + )); + } + let Some(oplog_bytes) = oplog.fork_changes_up_to(frontiers) else { + break 'block Err(LoroEncodeError::FrontiersNotFound(format!( + "frontiers: {:?} when export in SnapshotAt mode", + frontiers + ))); + }; + + if oplog.is_shallow() { + let Some(shallow_root_frontiers) = state.store.shallow_root_frontiers() else { + break 'block Err(LoroEncodeError::internal( + "encode_snapshot_at: shallow oplog is missing shallow root frontiers", + )); + }; + if oplog.shallow_since_frontiers() != shallow_root_frontiers { + break 'block Err(LoroEncodeError::internal( + "encode_snapshot_at: shallow root frontiers are inconsistent", + )); + } + } + + let alive_containers = state.ensure_all_alive_containers(); + if has_unknown_container(alive_containers.iter()) { + break 'block Err(LoroEncodeError::UnknownContainer); + } + + let alive_c_bytes = cids_to_bytes(alive_containers); + state.store.flush(); + let state_kv = state.store.get_kv_clone(); + state_kv.retain_keys(&alive_c_bytes); + let bytes = state_kv.export(); + _encode_snapshot( + Snapshot { + oplog_bytes, + state_bytes: Some(bytes), + shallow_root_state_bytes: Bytes::new(), + }, + w, + ); + + Ok(()) + }; + let restore_result = doc + ._checkout_without_emitting(&version_before_start, false, false) + .map_err(LoroEncodeError::from); + if !was_detached { + doc.set_detached(false); + } + doc.app_state().lock().take_events(); + + match result { + Err(err) => Err(err), + Ok(()) => restore_result, + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/encoding/value.rs b/3rdparty/loro/crates/loro-internal/src/encoding/value.rs new file mode 100644 index 0000000000..908ef5c982 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/encoding/value.rs @@ -0,0 +1,1515 @@ +use enum_as_inner::EnumAsInner; +use fractional_index::FractionalIndex; +use loro_common::{ + ContainerID, ContainerType, Counter, InternalString, LoroError, LoroResult, LoroValue, PeerID, + TreeID, ID, +}; +use rustc_hash::FxHashMap; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; + +use crate::{ + change::Lamport, container::tree::tree_op::TreeOp, + encoding::outdated_encode_reordered::MAX_COLLECTION_SIZE, +}; + +use super::{ + arena::{EncodedRegisters, EncodedTreeID}, + value_register::ValueRegister, +}; + +pub trait ValueDecodedArenasTrait { + fn keys(&self) -> &[InternalString]; + fn peers(&self) -> &[PeerID]; + fn decode_tree_op( + &self, + positions: &[Vec], + op: super::value::EncodedTreeMove, + id: ID, + ) -> LoroResult; +} + +pub trait ValueEncodeRegister { + fn key_mut(&mut self) -> &mut ValueRegister; + fn peer_mut(&mut self) -> &mut ValueRegister; + fn encode_tree_op(&mut self, op: &TreeOp) -> Value<'static>; +} + +#[derive(Debug)] +pub enum ValueKind { + Null, // 0 + True, // 1 + False, // 2 + I64, // 3 + F64, // 4 + Str, // 5 + Binary, // 6 + ContainerType, // 7 + DeleteOnce, // 8 + DeleteSeq, // 9 + DeltaInt, // 10 + LoroValue, // 11 + MarkStart, // 12 + TreeMove, // 13 + ListMove, // 14 + ListSet, // 15 + Future(FutureValueKind), + // 16 Counter + RawTreeMove, // 17 +} + +#[derive(Debug)] +pub enum LoroValueKind { + Null, + True, + False, + I64, + F64, + Binary, + Str, + List, + Map, + ContainerType, +} +impl LoroValueKind { + fn from_u8(kind: u8) -> Option { + match kind { + 0 => Some(LoroValueKind::Null), + 1 => Some(LoroValueKind::True), + 2 => Some(LoroValueKind::False), + 3 => Some(LoroValueKind::I64), + 4 => Some(LoroValueKind::F64), + 5 => Some(LoroValueKind::Str), + 6 => Some(LoroValueKind::Binary), + 7 => Some(LoroValueKind::List), + 8 => Some(LoroValueKind::Map), + 9 => Some(LoroValueKind::ContainerType), + _ => None, + } + } + + fn to_u8(&self) -> u8 { + match self { + LoroValueKind::Null => 0, + LoroValueKind::True => 1, + LoroValueKind::False => 2, + LoroValueKind::I64 => 3, + LoroValueKind::F64 => 4, + LoroValueKind::Str => 5, + LoroValueKind::Binary => 6, + LoroValueKind::List => 7, + LoroValueKind::Map => 8, + LoroValueKind::ContainerType => 9, + } + } +} + +#[derive(Debug)] +pub enum FutureValueKind { + Unknown(u8), +} + +impl ValueKind { + pub(crate) fn to_u8(&self) -> u8 { + match self { + ValueKind::Null => 0, + ValueKind::True => 1, + ValueKind::False => 2, + ValueKind::I64 => 3, + ValueKind::F64 => 4, + ValueKind::Str => 5, + ValueKind::Binary => 6, + ValueKind::ContainerType => 7, + ValueKind::DeleteOnce => 8, + ValueKind::DeleteSeq => 9, + ValueKind::DeltaInt => 10, + ValueKind::LoroValue => 11, + ValueKind::MarkStart => 12, + ValueKind::TreeMove => 13, + ValueKind::ListMove => 14, + ValueKind::ListSet => 15, + ValueKind::Future(future_value_kind) => match future_value_kind { + FutureValueKind::Unknown(u8) => *u8 | 0x80, + }, + ValueKind::RawTreeMove => 16, + } + } + + pub(crate) fn from_u8(mut kind: u8) -> Self { + kind &= 0x7F; + match kind { + 0 => ValueKind::Null, + 1 => ValueKind::True, + 2 => ValueKind::False, + 3 => ValueKind::I64, + 4 => ValueKind::F64, + 5 => ValueKind::Str, + 6 => ValueKind::Binary, + 7 => ValueKind::ContainerType, + 8 => ValueKind::DeleteOnce, + 9 => ValueKind::DeleteSeq, + 10 => ValueKind::DeltaInt, + 11 => ValueKind::LoroValue, + 12 => ValueKind::MarkStart, + 13 => ValueKind::TreeMove, + 14 => ValueKind::ListMove, + 15 => ValueKind::ListSet, + 16 => ValueKind::RawTreeMove, + _ => ValueKind::Future(FutureValueKind::Unknown(kind)), + } + } +} + +#[derive(Debug, EnumAsInner)] +pub enum Value<'a> { + Null, + True, + False, + I64(i64), + F64(f64), + Str(&'a str), + Binary(&'a [u8]), + ContainerIdx(usize), + DeleteOnce, + DeleteSeq, + DeltaInt(i32), + #[allow(clippy::enum_variant_names)] + LoroValue(LoroValue), + MarkStart(MarkStart), + TreeMove(EncodedTreeMove), + RawTreeMove(RawTreeMove), + ListMove { + from: usize, + from_idx: usize, + lamport: usize, + }, + ListSet { + peer_idx: usize, + lamport: Lamport, + value: LoroValue, + }, + Future(FutureValue<'a>), +} + +#[derive(Debug)] +pub enum FutureValue<'a> { + // The future value cannot depend on the arena for encoding. + Unknown { kind: u8, data: &'a [u8] }, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "value_type", content = "value", rename_all = "snake_case")] +pub enum OwnedValue { + Null, + True, + False, + I64(i64), + F64(f64), + Str(Arc), + Binary(Arc>), + ContainerIdx(usize), + DeleteOnce, + DeleteSeq, + DeltaInt(i32), + LoroValue(LoroValue), + MarkStart(MarkStart), + TreeMove(EncodedTreeMove), + RawTreeMove(RawTreeMove), + ListMove { + from: usize, + from_idx: usize, + lamport: usize, + }, + ListSet { + peer_idx: usize, + lamport: Lamport, + value: LoroValue, + }, + #[serde(untagged)] + Future(OwnedFutureValue), +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "value_type", content = "value")] +pub enum OwnedFutureValue { + // The future value cannot depend on the arena for encoding. + Unknown { kind: u8, data: Arc> }, +} + +impl<'a> Value<'a> { + pub fn from_owned(owned_value: &'a OwnedValue) -> Self { + match owned_value { + OwnedValue::Null => Value::Null, + OwnedValue::True => Value::True, + OwnedValue::False => Value::False, + OwnedValue::DeleteOnce => Value::DeleteOnce, + OwnedValue::I64(x) => Value::I64(*x), + OwnedValue::ContainerIdx(x) => Value::ContainerIdx(*x), + OwnedValue::F64(x) => Value::F64(*x), + OwnedValue::Str(x) => Value::Str(x.as_str()), + OwnedValue::DeleteSeq => Value::DeleteSeq, + OwnedValue::DeltaInt(x) => Value::DeltaInt(*x), + OwnedValue::LoroValue(x) => Value::LoroValue(x.clone()), + OwnedValue::MarkStart(x) => Value::MarkStart(x.clone()), + OwnedValue::Binary(x) => Value::Binary(x.as_slice()), + OwnedValue::TreeMove(x) => Value::TreeMove(x.clone()), + OwnedValue::ListMove { + from, + from_idx, + lamport, + } => Value::ListMove { + from: *from, + from_idx: *from_idx, + lamport: *lamport, + }, + OwnedValue::ListSet { + peer_idx, + lamport, + value, + } => Value::ListSet { + peer_idx: *peer_idx, + lamport: *lamport, + value: value.clone(), + }, + OwnedValue::Future(value) => match value { + OwnedFutureValue::Unknown { kind, data } => Value::Future(FutureValue::Unknown { + kind: *kind, + data: data.as_slice(), + }), + }, + OwnedValue::RawTreeMove(t) => Value::RawTreeMove(t.clone()), + } + } + + pub fn into_owned(self) -> OwnedValue { + match self { + Value::Null => OwnedValue::Null, + Value::True => OwnedValue::True, + Value::False => OwnedValue::False, + Value::DeleteOnce => OwnedValue::DeleteOnce, + Value::I64(x) => OwnedValue::I64(x), + Value::ContainerIdx(x) => OwnedValue::ContainerIdx(x), + Value::F64(x) => OwnedValue::F64(x), + Value::Str(x) => OwnedValue::Str(Arc::new(x.to_owned())), + Value::DeleteSeq => OwnedValue::DeleteSeq, + Value::DeltaInt(x) => OwnedValue::DeltaInt(x), + Value::LoroValue(x) => OwnedValue::LoroValue(x), + Value::MarkStart(x) => OwnedValue::MarkStart(x), + Value::Binary(x) => OwnedValue::Binary(Arc::new(x.to_owned())), + Value::TreeMove(x) => OwnedValue::TreeMove(x), + Value::ListMove { + from, + from_idx, + lamport, + } => OwnedValue::ListMove { + from, + from_idx, + lamport, + }, + Value::ListSet { + peer_idx, + lamport, + value, + } => OwnedValue::ListSet { + peer_idx, + lamport, + value, + }, + Value::Future(value) => match value { + FutureValue::Unknown { kind, data } => { + OwnedValue::Future(OwnedFutureValue::Unknown { + kind, + data: Arc::new(data.to_owned()), + }) + } + }, + Value::RawTreeMove(t) => OwnedValue::RawTreeMove(t), + } + } + + fn decode_without_arena<'r: 'a>( + future_kind: FutureValueKind, + value_reader: &'r mut ValueReader, + ) -> LoroResult { + let bytes = value_reader.read_binary()?; + let value = match future_kind { + FutureValueKind::Unknown(kind) => { + Value::Future(FutureValue::Unknown { kind, data: bytes }) + } + }; + Ok(value) + } + + pub(crate) fn decode<'r: 'a>( + kind: ValueKind, + value_reader: &'r mut ValueReader, + arenas: &'a dyn ValueDecodedArenasTrait, + id: ID, + ) -> LoroResult { + Ok(match kind { + ValueKind::Null => Value::Null, + ValueKind::True => Value::True, + ValueKind::False => Value::False, + ValueKind::I64 => Value::I64(value_reader.read_i64()?), + ValueKind::F64 => Value::F64(value_reader.read_f64()?), + ValueKind::Str => Value::Str(value_reader.read_str()?), + ValueKind::Binary => Value::Binary(value_reader.read_binary()?), + ValueKind::ContainerType => Value::ContainerIdx(value_reader.read_usize()?), + ValueKind::DeleteOnce => Value::DeleteOnce, + ValueKind::DeleteSeq => Value::DeleteSeq, + ValueKind::DeltaInt => Value::DeltaInt(value_reader.read_i32()?), + ValueKind::LoroValue => { + Value::LoroValue(value_reader.read_value_type_and_content(arenas.keys(), id)?) + } + ValueKind::MarkStart => Value::MarkStart(value_reader.read_mark(arenas.keys(), id)?), + ValueKind::TreeMove => Value::TreeMove(value_reader.read_tree_move()?), + ValueKind::RawTreeMove => Value::RawTreeMove(value_reader.read_raw_tree_move()?), + ValueKind::ListMove => { + let from = value_reader.read_usize()?; + let from_idx = value_reader.read_usize()?; + let lamport = value_reader.read_usize()?; + Value::ListMove { + from, + from_idx, + lamport, + } + } + ValueKind::ListSet => { + let peer_idx = value_reader.read_usize()?; + let lamport = value_reader.read_usize()? as u32; + let value = value_reader.read_value_type_and_content(arenas.keys(), id)?; + Value::ListSet { + peer_idx, + lamport, + value, + } + } + ValueKind::Future(future_kind) => { + Self::decode_without_arena(future_kind, value_reader)? + } + }) + } + + fn encode_without_registers( + value: FutureValue, + value_writer: &mut ValueWriter, + ) -> (FutureValueKind, usize) { + // Note: we should encode FutureValue as binary data. + // [binary data length, binary data] + // when decoding, we will use reader.read_binary() to read the binary data. + // So such as FutureValue::Counter, we should write 0 as the length of binary data first. + match value { + FutureValue::Unknown { kind, data } => ( + FutureValueKind::Unknown(kind), + value_writer.write_binary(data), + ), + } + } + + pub(super) fn encode( + self, + value_writer: &mut ValueWriter, + registers: &mut dyn ValueEncodeRegister, + ) -> (ValueKind, usize) { + match self { + Value::Null => (ValueKind::Null, 0), + Value::True => (ValueKind::True, 0), + Value::False => (ValueKind::False, 0), + Value::I64(x) => (ValueKind::I64, value_writer.write_i64(x)), + Value::F64(x) => (ValueKind::F64, value_writer.write_f64(x)), + Value::Str(x) => (ValueKind::Str, value_writer.write_str(x)), + Value::Binary(x) => (ValueKind::Binary, value_writer.write_binary(x)), + Value::ContainerIdx(x) => (ValueKind::ContainerType, value_writer.write_usize(x)), + Value::DeleteOnce => (ValueKind::DeleteOnce, 0), + Value::DeleteSeq => (ValueKind::DeleteSeq, 0), + Value::DeltaInt(x) => (ValueKind::DeltaInt, value_writer.write_i32(x)), + Value::LoroValue(x) => ( + ValueKind::LoroValue, + value_writer.write_value_type_and_content(&x, registers), + ), + Value::MarkStart(x) => (ValueKind::MarkStart, value_writer.write_mark(x, registers)), + Value::TreeMove(tree) => (ValueKind::TreeMove, value_writer.write_tree_move(&tree)), + Value::ListMove { + from, + from_idx, + lamport, + } => ( + ValueKind::ListMove, + value_writer.write_usize(from) + + value_writer.write_usize(from_idx) + + value_writer.write_usize(lamport), + ), + Value::ListSet { + peer_idx, + lamport, + value, + } => ( + ValueKind::ListSet, + value_writer.write_usize(peer_idx) + + value_writer.write_usize(lamport as usize) + + value_writer.write_value_type_and_content(&value, registers), + ), + Value::Future(value) => { + let (k, i) = Self::encode_without_registers(value, value_writer); + (ValueKind::Future(k), i) + } + Value::RawTreeMove(x) => (ValueKind::RawTreeMove, value_writer.write_raw_tree_move(x)), + } + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct MarkStart { + pub len: u32, + pub key: InternalString, + pub value: LoroValue, + pub info: u8, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct RawTreeMove { + pub subject_peer_idx: usize, + pub subject_cnt: Counter, + pub is_parent_null: bool, + pub parent_peer_idx: usize, + pub parent_cnt: Counter, + pub position_idx: usize, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct EncodedTreeMove { + pub target_idx: usize, + pub is_parent_null: bool, + pub parent_idx: usize, + pub position: usize, +} + +impl EncodedTreeMove { + pub fn as_tree_op( + &self, + peer_ids: &[u64], + positions: &[Vec], + tree_ids: &[EncodedTreeID], + op_id: ID, + ) -> LoroResult { + let parent = if self.is_parent_null { + None + } else { + let EncodedTreeID { peer_idx, counter } = tree_ids[self.parent_idx]; + Some(TreeID::new( + *(peer_ids + .get(peer_idx) + .ok_or(LoroError::DecodeDataCorruptionError)?), + counter as Counter, + )) + }; + let is_delete = parent.is_some_and(|p| TreeID::is_deleted_root(&p)); + let position = if is_delete { + None + } else { + let bytes = &positions[self.position]; + Some(FractionalIndex::from_bytes(bytes.clone())) + }; + let EncodedTreeID { peer_idx, counter } = tree_ids[self.target_idx]; + let target = TreeID::new( + *(peer_ids + .get(peer_idx) + .ok_or(LoroError::DecodeDataCorruptionError)?), + counter as Counter, + ); + + let is_create = target.id() == op_id; + let op = if is_delete { + TreeOp::Delete { target } + } else if is_create { + TreeOp::Create { + target, + parent, + position: position.unwrap(), + } + } else { + TreeOp::Move { + target, + parent, + position: position.unwrap(), + } + }; + Ok(op) + } + + pub fn from_tree_op<'p, 'a: 'p>(op: &'a TreeOp, registers: &mut EncodedRegisters) -> Self { + match op { + TreeOp::Create { + target, + parent, + position, + } + | TreeOp::Move { + target, + parent, + position, + } => { + let bytes = position.as_bytes(); + let either::Right(position_register) = &mut registers.position else { + unreachable!() + }; + let position = position_register.get(&bytes).unwrap(); + let target_idx = registers.tree_id.register(&EncodedTreeID { + peer_idx: registers.peer.register(&target.peer), + counter: target.counter, + }); + + let parent_idx = parent.map(|x| { + registers.tree_id.register(&EncodedTreeID { + peer_idx: registers.peer.register(&x.peer), + counter: x.counter, + }) + }); + EncodedTreeMove { + target_idx, + is_parent_null: parent.is_none(), + parent_idx: parent_idx.unwrap_or(0), + position, + } + } + TreeOp::Delete { target } => { + let target_idx = registers.tree_id.register(&EncodedTreeID { + peer_idx: registers.peer.register(&target.peer), + counter: target.counter, + }); + let parent_idx = registers.tree_id.register(&EncodedTreeID { + peer_idx: registers.peer.register(&TreeID::delete_root().peer), + counter: TreeID::delete_root().counter, + }); + EncodedTreeMove { + target_idx, + is_parent_null: false, + parent_idx, + position: 0, + } + } + } + } +} + +pub struct ValueWriter { + buffer: Vec, +} + +pub struct ValueReader<'a> { + raw: &'a [u8], +} + +impl<'a> ValueReader<'a> { + pub fn new(raw: &'a [u8]) -> Self { + ValueReader { raw } + } + + pub fn read_value_type_and_content( + &mut self, + keys: &[InternalString], + id: ID, + ) -> LoroResult { + let kind = self.read_u8()?; + let kind = LoroValueKind::from_u8(kind).ok_or(LoroError::DecodeDataCorruptionError)?; + self.read_value_content(kind, keys, id) + } + + pub fn read_value_content( + &mut self, + kind: LoroValueKind, + keys: &[InternalString], + id: ID, + ) -> LoroResult { + Ok(match kind { + LoroValueKind::Null => LoroValue::Null, + LoroValueKind::True => LoroValue::Bool(true), + LoroValueKind::False => LoroValue::Bool(false), + LoroValueKind::I64 => LoroValue::I64(self.read_i64()?), + LoroValueKind::F64 => LoroValue::Double(self.read_f64()?), + LoroValueKind::Str => LoroValue::String((self.read_str()?.to_owned()).into()), + LoroValueKind::List => { + let len = self.read_usize()?; + if len > MAX_COLLECTION_SIZE { + return Err(LoroError::DecodeDataCorruptionError); + } + let mut ans = Vec::with_capacity(len); + for i in 0..len { + ans.push(self.recursive_read_value_type_and_content(keys, id.inc(i as i32))?); + } + ans.into() + } + LoroValueKind::Map => { + let len = self.read_usize()?; + if len > MAX_COLLECTION_SIZE { + return Err(LoroError::DecodeDataCorruptionError); + } + let mut ans = FxHashMap::with_capacity_and_hasher(len, Default::default()); + for _ in 0..len { + let key_idx = self.read_usize()?; + let key = keys + .get(key_idx) + .ok_or(LoroError::DecodeDataCorruptionError)? + .to_string(); + let value = self.recursive_read_value_type_and_content(keys, id)?; + ans.insert(key, value); + } + ans.into() + } + LoroValueKind::Binary => LoroValue::Binary(self.read_binary()?.to_owned().into()), + LoroValueKind::ContainerType => { + let u8 = self.read_u8()?; + let container_id = ContainerID::new_normal( + id, + ContainerType::try_from_u8(u8).unwrap_or(ContainerType::Unknown(u8)), + ); + + LoroValue::Container(container_id) + } + }) + } + + /// Read a value that may be very deep efficiently. + /// + /// This method avoids using recursive calls to read deeply nested values. + /// Otherwise, it may cause stack overflow. + fn recursive_read_value_type_and_content( + &mut self, + keys: &[InternalString], + id: ID, + ) -> LoroResult { + #[derive(Debug)] + enum Task { + Init, + ReadList { + left: usize, + vec: Vec, + key_idx_in_parent: usize, + }, + ReadMap { + left: usize, + map: FxHashMap, + key_idx_in_parent: usize, + }, + } + impl Task { + fn should_read(&self) -> bool { + !matches!( + self, + Self::ReadList { left: 0, .. } | Self::ReadMap { left: 0, .. } + ) + } + + fn key_idx(&self) -> usize { + match self { + Self::ReadList { + key_idx_in_parent, .. + } => *key_idx_in_parent, + Self::ReadMap { + key_idx_in_parent, .. + } => *key_idx_in_parent, + _ => unreachable!(), + } + } + + fn into_value(self) -> LoroValue { + match self { + Self::ReadList { vec, .. } => vec.into(), + Self::ReadMap { map, .. } => map.into(), + _ => unreachable!(), + } + } + } + let mut stack = vec![Task::Init]; + while let Some(mut task) = stack.pop() { + if task.should_read() { + let key_idx = if matches!(task, Task::ReadMap { .. }) { + self.read_usize()? + } else { + 0 + }; + let kind = self.read_u8()?; + let kind = + LoroValueKind::from_u8(kind).ok_or(LoroError::DecodeDataCorruptionError)?; + let value = match kind { + LoroValueKind::Null => LoroValue::Null, + LoroValueKind::True => LoroValue::Bool(true), + LoroValueKind::False => LoroValue::Bool(false), + LoroValueKind::I64 => LoroValue::I64(self.read_i64()?), + LoroValueKind::F64 => LoroValue::Double(self.read_f64()?), + LoroValueKind::Str => LoroValue::String((self.read_str()?.to_owned()).into()), + LoroValueKind::List => { + let len = self.read_usize()?; + if len > MAX_COLLECTION_SIZE { + return Err(LoroError::DecodeDataCorruptionError); + } + let ans = Vec::with_capacity(len); + stack.push(task); + stack.push(Task::ReadList { + left: len, + vec: ans, + key_idx_in_parent: key_idx, + }); + continue; + } + LoroValueKind::Map => { + let len = self.read_usize()?; + if len > MAX_COLLECTION_SIZE { + return Err(LoroError::DecodeDataCorruptionError); + } + + let ans = FxHashMap::with_capacity_and_hasher(len, Default::default()); + stack.push(task); + stack.push(Task::ReadMap { + left: len, + map: ans, + key_idx_in_parent: key_idx, + }); + continue; + } + LoroValueKind::Binary => { + LoroValue::Binary(self.read_binary()?.to_owned().into()) + } + LoroValueKind::ContainerType => { + let u8 = self.read_u8()?; + let container_id = ContainerID::new_normal( + id, + ContainerType::try_from_u8(u8).unwrap_or(ContainerType::Unknown(u8)), + ); + + LoroValue::Container(container_id) + } + }; + + task = match task { + Task::Init => { + return Ok(value); + } + Task::ReadList { + mut left, + mut vec, + key_idx_in_parent, + } => { + left -= 1; + vec.push(value); + let task = Task::ReadList { + left, + vec, + key_idx_in_parent, + }; + if left != 0 { + stack.push(task); + continue; + } + + task + } + Task::ReadMap { + mut left, + mut map, + key_idx_in_parent, + } => { + left -= 1; + let key = keys + .get(key_idx) + .ok_or(LoroError::DecodeDataCorruptionError)? + .to_string(); + map.insert(key, value); + let task = Task::ReadMap { + left, + map, + key_idx_in_parent, + }; + if left != 0 { + stack.push(task); + continue; + } + task + } + }; + } + + let key_index = task.key_idx(); + let value = task.into_value(); + if let Some(last) = stack.last_mut() { + match last { + Task::Init => { + return Ok(value); + } + Task::ReadList { left, vec, .. } => { + *left -= 1; + vec.push(value); + } + Task::ReadMap { left, map, .. } => { + *left -= 1; + let key = keys + .get(key_index) + .ok_or(LoroError::DecodeDataCorruptionError)? + .to_string(); + map.insert(key, value); + } + } + } else { + return Ok(value); + } + } + + unreachable!(); + } + + pub fn read_i64(&mut self) -> LoroResult { + leb128::read::signed(&mut self.raw).map_err(|_| LoroError::DecodeDataCorruptionError) + } + + #[allow(unused)] + pub fn read_u64(&mut self) -> LoroResult { + leb128::read::unsigned(&mut self.raw).map_err(|_| LoroError::DecodeDataCorruptionError) + } + + #[allow(unused)] + pub fn read_i32(&mut self) -> LoroResult { + leb128::read::signed(&mut self.raw) + .map(|x| x as i32) + .map_err(|_| LoroError::DecodeDataCorruptionError) + } + + pub fn read_f64(&mut self) -> LoroResult { + if self.raw.len() < 8 { + return Err(LoroError::DecodeDataCorruptionError); + } + + let mut bytes = [0; 8]; + bytes.copy_from_slice(&self.raw[..8]); + self.raw = &self.raw[8..]; + Ok(f64::from_be_bytes(bytes)) + } + + pub fn read_usize(&mut self) -> LoroResult { + Ok(leb128::read::unsigned(&mut self.raw) + .map_err(|_| LoroError::DecodeDataCorruptionError)? as usize) + } + + pub fn read_str(&mut self) -> LoroResult<&'a str> { + let len = self.read_usize()?; + if self.raw.len() < len { + return Err(LoroError::DecodeDataCorruptionError); + } + + let ans = std::str::from_utf8(&self.raw[..len]) + .map_err(|_| LoroError::DecodeDataCorruptionError)?; + self.raw = &self.raw[len..]; + Ok(ans) + } + + #[allow(unused)] + fn read_binary_vec(&mut self) -> LoroResult> { + let len = self.read_usize()?; + if self.raw.len() < len { + return Err(LoroError::DecodeDataCorruptionError); + } + let ans = &self.raw[..len]; + self.raw = &self.raw[len..]; + Ok(ans.to_vec()) + } + + fn read_u8(&mut self) -> LoroResult { + if self.raw.is_empty() { + return Err(LoroError::DecodeDataCorruptionError); + } + + let ans = self.raw[0]; + self.raw = &self.raw[1..]; + Ok(ans) + } + + pub fn read_binary(&mut self) -> LoroResult<&'a [u8]> { + let len = self.read_usize()?; + if self.raw.len() < len { + return Err(LoroError::DecodeDataCorruptionError); + } + + let ans = &self.raw[..len]; + self.raw = &self.raw[len..]; + Ok(ans) + } + + pub fn read_mark<'s: 'm, 'm>( + &mut self, + keys: &'s [InternalString], + id: ID, + ) -> LoroResult { + let info = self.read_u8()?; + let len = self.read_usize()?; + let key_idx = self.read_usize()?; + let value = self.read_value_type_and_content(keys, id)?; + Ok(MarkStart { + len: len as u32, + key: keys + .get(key_idx) + .ok_or(LoroError::DecodeDataCorruptionError)? + .clone(), + value, + info, + }) + } + + pub fn read_tree_move(&mut self) -> LoroResult { + let subject_idx = self.read_usize()?; + let is_parent_null = self.read_u8()? != 0; + let position = self.read_usize()?; + let mut parent_idx = 0; + if !is_parent_null { + parent_idx = self.read_usize()?; + } + Ok(EncodedTreeMove { + target_idx: subject_idx, + is_parent_null, + parent_idx, + position, + }) + } + + fn read_raw_tree_move(&mut self) -> LoroResult { + let subject_peer_idx = self.read_usize()?; + let subject_cnt = self.read_usize()?; + let position_idx = self.read_usize()?; + let is_parent_null = self.read_u8()? != 0; + let mut parent_peer_idx = 0; + let mut parent_cnt = 0; + if !is_parent_null { + parent_peer_idx = self.read_usize()?; + parent_cnt = self.read_usize()?; + } + + Ok(RawTreeMove { + subject_peer_idx, + subject_cnt: subject_cnt as i32, + position_idx, + is_parent_null, + parent_peer_idx, + parent_cnt: parent_cnt as i32, + }) + } +} + +impl ValueWriter { + pub fn new() -> Self { + ValueWriter { buffer: Vec::new() } + } + + pub fn write_value_type_and_content( + &mut self, + value: &LoroValue, + registers: &mut dyn ValueEncodeRegister, + ) -> usize { + let len = self.write_u8(get_loro_value_kind(value).to_u8()); + let (_, l) = self.write_value_content(value, registers); + len + l + } + + pub fn write_value_content( + &mut self, + value: &LoroValue, + registers: &mut dyn ValueEncodeRegister, + ) -> (LoroValueKind, usize) { + match value { + LoroValue::Null => (LoroValueKind::Null, 0), + LoroValue::Bool(true) => (LoroValueKind::True, 0), + LoroValue::Bool(false) => (LoroValueKind::False, 0), + LoroValue::I64(value) => (LoroValueKind::I64, self.write_i64(*value)), + LoroValue::Double(value) => (LoroValueKind::F64, self.write_f64(*value)), + LoroValue::String(value) => (LoroValueKind::Str, self.write_str(value)), + LoroValue::List(value) => { + let mut len = self.write_usize(value.len()); + for value in value.iter() { + let l = self.write_value_type_and_content(value, registers); + len += l; + } + (LoroValueKind::List, len) + } + LoroValue::Map(value) => { + let mut len = self.write_usize(value.len()); + for (key, value) in value.iter() { + let key_idx = registers.key_mut().register(&key.as_str().into()); + len += self.write_usize(key_idx); + let l = self.write_value_type_and_content(value, registers); + len += l; + } + (LoroValueKind::Map, len) + } + LoroValue::Binary(value) => (LoroValueKind::Binary, self.write_binary(value)), + LoroValue::Container(c) => ( + LoroValueKind::ContainerType, + self.write_u8(c.container_type().to_u8()), + ), + } + } + + pub fn write_i64(&mut self, value: i64) -> usize { + let len = self.buffer.len(); + leb128::write::signed(&mut self.buffer, value).unwrap(); + self.buffer.len() - len + } + + fn write_i32(&mut self, value: i32) -> usize { + let len = self.buffer.len(); + leb128::write::signed(&mut self.buffer, value as i64).unwrap(); + self.buffer.len() - len + } + + #[allow(unused)] + fn write_u64(&mut self, value: u64) -> usize { + let len = self.buffer.len(); + leb128::write::unsigned(&mut self.buffer, value).unwrap(); + self.buffer.len() - len + } + + fn write_usize(&mut self, value: usize) -> usize { + let len = self.buffer.len(); + leb128::write::unsigned(&mut self.buffer, value as u64).unwrap(); + self.buffer.len() - len + } + + fn write_f64(&mut self, value: f64) -> usize { + let len = self.buffer.len(); + self.buffer.extend_from_slice(&value.to_be_bytes()); + self.buffer.len() - len + } + + fn write_str(&mut self, value: &str) -> usize { + let len = self.buffer.len(); + self.write_usize(value.len()); + self.buffer.extend_from_slice(value.as_bytes()); + self.buffer.len() - len + } + + fn write_u8(&mut self, value: u8) -> usize { + let len = self.buffer.len(); + self.buffer.push(value); + self.buffer.len() - len + } + + fn write_binary(&mut self, value: &[u8]) -> usize { + let len = self.buffer.len(); + self.write_usize(value.len()); + self.buffer.extend_from_slice(value); + self.buffer.len() - len + } + + fn write_mark(&mut self, mark: MarkStart, registers: &mut dyn ValueEncodeRegister) -> usize { + let key_idx = registers.key_mut().register(&mark.key); + let len = self.buffer.len(); + self.write_u8(mark.info); + self.write_usize(mark.len as usize); + self.write_usize(key_idx); + self.write_value_type_and_content(&mark.value, registers); + self.buffer.len() - len + } + + fn write_tree_move(&mut self, op: &EncodedTreeMove) -> usize { + let len = self.buffer.len(); + self.write_usize(op.target_idx); + self.write_u8(op.is_parent_null as u8); + self.write_usize(op.position); + if op.is_parent_null { + return self.buffer.len() - len; + } + self.write_usize(op.parent_idx); + self.buffer.len() - len + } + + pub(crate) fn finish(self) -> Vec { + self.buffer + } + + fn write_raw_tree_move(&mut self, op: RawTreeMove) -> usize { + let len = self.buffer.len(); + self.write_usize(op.subject_peer_idx); + self.write_usize(op.subject_cnt as usize); + self.write_usize(op.position_idx); + self.write_u8(op.is_parent_null as u8); + if op.is_parent_null { + return self.buffer.len() - len; + } + self.write_usize(op.parent_peer_idx); + self.write_usize(op.parent_cnt as usize); + self.buffer.len() - len + } +} + +fn get_loro_value_kind(value: &LoroValue) -> LoroValueKind { + match value { + LoroValue::Null => LoroValueKind::Null, + LoroValue::Bool(true) => LoroValueKind::True, + LoroValue::Bool(false) => LoroValueKind::False, + LoroValue::I64(_) => LoroValueKind::I64, + LoroValue::Double(_) => LoroValueKind::F64, + LoroValue::String(_) => LoroValueKind::Str, + LoroValue::List(_) => LoroValueKind::List, + LoroValue::Map(_) => LoroValueKind::Map, + LoroValue::Binary(_) => LoroValueKind::Binary, + LoroValue::Container(_) => LoroValueKind::ContainerType, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{container::tree::tree_op::TreeOp, fx_map}; + + #[derive(Default)] + struct TestRegisters { + key: ValueRegister, + peer: ValueRegister, + } + + impl ValueEncodeRegister for TestRegisters { + fn key_mut(&mut self) -> &mut ValueRegister { + &mut self.key + } + + fn peer_mut(&mut self) -> &mut ValueRegister { + &mut self.peer + } + + fn encode_tree_op(&mut self, op: &TreeOp) -> Value<'static> { + let target = match op { + TreeOp::Create { target, .. } + | TreeOp::Move { target, .. } + | TreeOp::Delete { target } => *target, + }; + + Value::TreeMove(EncodedTreeMove { + target_idx: self.peer.register(&target.peer), + is_parent_null: true, + parent_idx: 0, + position: 0, + }) + } + } + + struct TestArenas { + keys: Vec, + peers: Vec, + } + + impl ValueDecodedArenasTrait for TestArenas { + fn keys(&self) -> &[InternalString] { + &self.keys + } + + fn peers(&self) -> &[PeerID] { + &self.peers + } + + fn decode_tree_op( + &self, + positions: &[Vec], + op: EncodedTreeMove, + id: ID, + ) -> LoroResult { + op.as_tree_op(&self.peers, positions, &[], id) + } + } + + fn encode_then_decode_owned(value: Value<'_>) -> OwnedValue { + let mut writer = ValueWriter::new(); + let mut registers = TestRegisters::default(); + let (kind, _) = value.encode(&mut writer, &mut registers); + let bytes = writer.finish(); + let arenas = TestArenas { + keys: registers.key.unwrap_vec(), + peers: registers.peer.unwrap_vec(), + }; + let mut reader = ValueReader::new(&bytes); + Value::decode(kind, &mut reader, &arenas, ID::new(9, 1)) + .unwrap() + .into_owned() + } + + #[test] + fn value_kind_tags_roundtrip_including_future_kinds() { + let kinds = [ + ValueKind::Null, + ValueKind::True, + ValueKind::False, + ValueKind::I64, + ValueKind::F64, + ValueKind::Str, + ValueKind::Binary, + ValueKind::ContainerType, + ValueKind::DeleteOnce, + ValueKind::DeleteSeq, + ValueKind::DeltaInt, + ValueKind::LoroValue, + ValueKind::MarkStart, + ValueKind::TreeMove, + ValueKind::ListMove, + ValueKind::ListSet, + ValueKind::RawTreeMove, + ]; + + for kind in kinds { + let tag = kind.to_u8(); + assert_eq!(ValueKind::from_u8(tag).to_u8(), tag); + } + + let future = ValueKind::Future(FutureValueKind::Unknown(42)); + assert_eq!(future.to_u8(), 42 | 0x80); + match ValueKind::from_u8(future.to_u8()) { + ValueKind::Future(FutureValueKind::Unknown(kind)) => assert_eq!(kind, 42), + other => panic!("expected future kind, got {other:?}"), + } + + let loro_kinds = [ + LoroValueKind::Null, + LoroValueKind::True, + LoroValueKind::False, + LoroValueKind::I64, + LoroValueKind::F64, + LoroValueKind::Str, + LoroValueKind::Binary, + LoroValueKind::List, + LoroValueKind::Map, + LoroValueKind::ContainerType, + ]; + for kind in loro_kinds { + assert_eq!( + LoroValueKind::from_u8(kind.to_u8()) + .expect("known value kind tag should decode") + .to_u8(), + kind.to_u8() + ); + } + } + + #[test] + fn owned_value_conversion_is_lossless_for_all_current_variants() { + let mark = MarkStart { + len: 3, + key: "bold".into(), + value: true.into(), + info: 7, + }; + let tree_move = EncodedTreeMove { + target_idx: 2, + is_parent_null: false, + parent_idx: 1, + position: 4, + }; + let raw_tree_move = RawTreeMove { + subject_peer_idx: 3, + subject_cnt: 9, + is_parent_null: false, + parent_peer_idx: 4, + parent_cnt: 8, + position_idx: 6, + }; + let values = vec![ + OwnedValue::Null, + OwnedValue::True, + OwnedValue::False, + OwnedValue::I64(-123), + OwnedValue::F64(1.25), + OwnedValue::Str(Arc::new("hello".to_string())), + OwnedValue::Binary(Arc::new(vec![1, 2, 3])), + OwnedValue::ContainerIdx(42), + OwnedValue::DeleteOnce, + OwnedValue::DeleteSeq, + OwnedValue::DeltaInt(-9), + OwnedValue::LoroValue( + fx_map!( + "k".to_string() => + LoroValue::from(vec![LoroValue::from(1_i64), LoroValue::from("v")]) + ) + .into(), + ), + OwnedValue::MarkStart(mark), + OwnedValue::TreeMove(tree_move), + OwnedValue::RawTreeMove(raw_tree_move), + OwnedValue::ListMove { + from: 1, + from_idx: 2, + lamport: 3, + }, + OwnedValue::ListSet { + peer_idx: 4, + lamport: 5, + value: "set".into(), + }, + OwnedValue::Future(OwnedFutureValue::Unknown { + kind: 99, + data: Arc::new(vec![4, 5, 6]), + }), + ]; + + for owned in values { + assert_eq!(Value::from_owned(&owned).into_owned(), owned); + } + } + + #[test] + fn value_encode_decode_roundtrips_non_loro_value_variants() { + let cases = vec![ + (Value::Null, OwnedValue::Null), + (Value::True, OwnedValue::True), + (Value::False, OwnedValue::False), + (Value::I64(-42), OwnedValue::I64(-42)), + (Value::F64(3.5), OwnedValue::F64(3.5)), + ( + Value::Str("encoded"), + OwnedValue::Str(Arc::new("encoded".to_string())), + ), + ( + Value::Binary(&[9, 8, 7]), + OwnedValue::Binary(Arc::new(vec![9, 8, 7])), + ), + (Value::ContainerIdx(17), OwnedValue::ContainerIdx(17)), + (Value::DeleteOnce, OwnedValue::DeleteOnce), + (Value::DeleteSeq, OwnedValue::DeleteSeq), + (Value::DeltaInt(-17), OwnedValue::DeltaInt(-17)), + ( + Value::LoroValue(fx_map!("nested".to_string() => LoroValue::from(1_i64)).into()), + OwnedValue::LoroValue( + fx_map!("nested".to_string() => LoroValue::from(1_i64)).into(), + ), + ), + ( + Value::MarkStart(MarkStart { + len: 2, + key: "italic".into(), + value: "yes".into(), + info: 3, + }), + OwnedValue::MarkStart(MarkStart { + len: 2, + key: "italic".into(), + value: "yes".into(), + info: 3, + }), + ), + ( + Value::TreeMove(EncodedTreeMove { + target_idx: 5, + is_parent_null: true, + parent_idx: 0, + position: 7, + }), + OwnedValue::TreeMove(EncodedTreeMove { + target_idx: 5, + is_parent_null: true, + parent_idx: 0, + position: 7, + }), + ), + ( + Value::RawTreeMove(RawTreeMove { + subject_peer_idx: 1, + subject_cnt: 2, + is_parent_null: true, + parent_peer_idx: 0, + parent_cnt: 0, + position_idx: 3, + }), + OwnedValue::RawTreeMove(RawTreeMove { + subject_peer_idx: 1, + subject_cnt: 2, + is_parent_null: true, + parent_peer_idx: 0, + parent_cnt: 0, + position_idx: 3, + }), + ), + ( + Value::ListMove { + from: 3, + from_idx: 4, + lamport: 5, + }, + OwnedValue::ListMove { + from: 3, + from_idx: 4, + lamport: 5, + }, + ), + ( + Value::ListSet { + peer_idx: 1, + lamport: 2, + value: vec![LoroValue::from(1_i64), LoroValue::from(2_i64)].into(), + }, + OwnedValue::ListSet { + peer_idx: 1, + lamport: 2, + value: vec![LoroValue::from(1_i64), LoroValue::from(2_i64)].into(), + }, + ), + ( + Value::Future(FutureValue::Unknown { + kind: 100, + data: &[1, 3, 5], + }), + OwnedValue::Future(OwnedFutureValue::Unknown { + kind: 100, + data: Arc::new(vec![1, 3, 5]), + }), + ), + ]; + + for (value, expected) in cases { + assert_eq!(encode_then_decode_owned(value), expected); + } + } + + #[test] + fn loro_value_writer_reader_roundtrips_nested_scalars_collections_and_containers() { + let container = ContainerID::new_normal(ID::new(7, 2), ContainerType::Tree); + let value: LoroValue = fx_map!( + "null".to_string() => LoroValue::Null, + "true".to_string() => true.into(), + "false".to_string() => false.into(), + "int".to_string() => (-9_i64).into(), + "float".to_string() => 2.25.into(), + "string".to_string() => "text".into(), + "binary".to_string() => vec![1_u8, 2, 3].into(), + "container".to_string() => container.into(), + "list".to_string() => vec![ + LoroValue::Null, + true.into(), + fx_map!("child".to_string() => "value".into()).into(), + ].into() + ) + .into(); + + let mut writer = ValueWriter::new(); + let mut registers = TestRegisters::default(); + writer.write_value_type_and_content(&value, &mut registers); + let keys = registers.key.unwrap_vec(); + assert!(keys.iter().any(|key| key.as_ref() == "child")); + + let bytes = writer.finish(); + let mut reader = ValueReader::new(&bytes); + let decoded = reader + .read_value_type_and_content(&keys, ID::new(7, 2)) + .unwrap(); + assert_eq!(decoded, value); + } + + #[test] + fn reader_reports_corruption_for_truncated_or_invalid_value_payloads() { + assert!(ValueReader::new(&[]).read_i64().is_err()); + assert!(ValueReader::new(&[1, 2, 3]).read_f64().is_err()); + assert!(ValueReader::new(&[5, b'a']).read_str().is_err()); + assert!(ValueReader::new(&[5, 1, 2]).read_binary().is_err()); + + let mut writer = ValueWriter::new(); + writer.write_usize(1); + writer.write_usize(9); + writer.write_value_type_and_content(&"value".into(), &mut TestRegisters::default()); + let bytes = writer.finish(); + let mut reader = ValueReader::new(&bytes); + assert!(reader + .read_value_content(LoroValueKind::Map, &[], ID::new(1, 0)) + .is_err()); + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/encoding/value_register.rs b/3rdparty/loro/crates/loro-internal/src/encoding/value_register.rs new file mode 100644 index 0000000000..97ff593393 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/encoding/value_register.rs @@ -0,0 +1,51 @@ +use rustc_hash::FxHashMap; + +#[derive(Debug)] +pub struct ValueRegister { + map_value_to_index: FxHashMap, + vec: Vec, +} + +impl Default for ValueRegister { + fn default() -> Self { + Self::new() + } +} + +impl ValueRegister { + pub fn new() -> Self { + Self { + map_value_to_index: FxHashMap::default(), + vec: Vec::new(), + } + } + + /// Return the index of the given value. If it does not exist, + /// insert it and return the new index. + pub fn register(&mut self, key: &T) -> usize { + if let Some(index) = self.map_value_to_index.get(key) { + *index + } else { + let idx = self.vec.len(); + self.vec.push(key.clone()); + self.map_value_to_index.insert(key.clone(), idx); + idx + } + } + + pub fn get(&self, key: &T) -> Option { + self.map_value_to_index.get(key).copied() + } + + pub fn unwrap_vec(self) -> Vec { + self.vec + } + + pub fn get_value(&self, index: usize) -> Option<&T> { + self.vec.get(index) + } + + pub fn vec(&self) -> &Vec { + &self.vec + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/error.rs b/3rdparty/loro/crates/loro-internal/src/error.rs new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/error.rs @@ -0,0 +1 @@ + diff --git a/3rdparty/loro/crates/loro-internal/src/estimated_size.rs b/3rdparty/loro/crates/loro-internal/src/estimated_size.rs new file mode 100644 index 0000000000..64718f56bc --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/estimated_size.rs @@ -0,0 +1,27 @@ +use loro_common::LoroValue; + +pub trait EstimatedSize { + /// Estimate the storage size of the object in bytes + fn estimate_storage_size(&self) -> usize; +} + +impl EstimatedSize for LoroValue { + fn estimate_storage_size(&self) -> usize { + match self { + LoroValue::Null => 1, + LoroValue::Bool(_) => 1, + LoroValue::Double(_) => 8, + LoroValue::I64(i) => 1 + (64 - i.leading_zeros()) as usize / 7, + LoroValue::Binary(b) => b.len() + 1, + LoroValue::String(s) => s.len() + 1, + LoroValue::List(l) => 3 + l.iter().map(|x| x.estimate_storage_size()).sum::(), + LoroValue::Map(m) => { + 3 + m + .iter() + .map(|(k, v)| k.len() + 3 + v.estimate_storage_size()) + .sum::() + } + LoroValue::Container(_) => 6, + } + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/event.rs b/3rdparty/loro/crates/loro-internal/src/event.rs new file mode 100644 index 0000000000..ee9abab7af --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/event.rs @@ -0,0 +1,522 @@ +use enum_as_inner::EnumAsInner; +use itertools::Itertools; +use loro_delta::{array_vec::ArrayVec, delta_trait::DeltaAttr, DeltaItem, DeltaRope}; +use rustc_hash::FxHashMap; +use serde::{Deserialize, Serialize}; +use smallvec::SmallVec; + +use crate::{ + container::richtext::richtext_state::RichtextStateChunk, + delta::{Delta, MapDelta, Meta, MovableListInnerDelta, ResolvedMapDelta, TreeDelta, TreeDiff}, + diff_calc::DiffMode, + handler::ValueOrHandler, + op::SliceWithId, + utils::string_slice::StringSlice, + InternalString, +}; + +use std::{borrow::Cow, hash::Hash}; + +use loro_common::{ContainerID, LoroValue, TreeID}; + +use crate::{container::idx::ContainerIdx, version::Frontiers}; + +#[derive(Debug, Clone)] +pub struct ContainerDiff { + pub id: ContainerID, + pub path: Vec<(ContainerID, Index)>, + pub(crate) idx: ContainerIdx, + pub is_unknown: bool, + pub diff: Diff, +} + +/// The kind of the event trigger. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum EventTriggerKind { + /// The event is triggered by a local transaction. + Local, + /// The event is triggered by importing + Import, + /// The event is triggered by checkout + Checkout, +} + +impl std::fmt::Display for EventTriggerKind { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + EventTriggerKind::Local => write!(f, "local"), + EventTriggerKind::Import => write!(f, "import"), + EventTriggerKind::Checkout => write!(f, "checkout"), + } + } +} + +impl EventTriggerKind { + #[inline] + pub fn is_local(&self) -> bool { + matches!(self, EventTriggerKind::Local) + } + + #[inline] + pub fn is_import(&self) -> bool { + matches!(self, EventTriggerKind::Import) + } + + #[inline] + pub fn is_checkout(&self) -> bool { + matches!(self, EventTriggerKind::Checkout) + } +} + +#[derive(Debug, Clone)] +pub struct DiffEvent<'a> { + /// The receiver of the event. + pub current_target: Option, + /// A list of events that should be received by the current target. + pub events: &'a [&'a ContainerDiff], + pub event_meta: &'a DocDiff, +} + +/// It's the exposed event type. +/// It's exposed to the user. The user can use this to apply the diff to their local state. +/// +/// [DocDiff] may include the diff that calculated from several transactions and imports. +/// They all should have the same origin and local flag. +#[derive(Debug, Clone)] +pub struct DocDiff { + pub from: Frontiers, + pub to: Frontiers, + pub origin: InternalString, + pub by: EventTriggerKind, + pub diff: Vec, +} + +#[derive(Debug, Clone)] +pub(crate) struct InternalContainerDiff { + pub(crate) idx: ContainerIdx, + /// Indicates whether this event was generated by a container that has been resurrected during a checkout operation. + pub(crate) bring_back: bool, + pub(crate) diff: DiffVariant, + /// This mode decides how should we apply the diff. + pub(crate) diff_mode: DiffMode, +} + +#[derive(Default, Debug, Clone, EnumAsInner)] +pub(crate) enum DiffVariant { + #[default] + None, + Internal(InternalDiff), + External(Diff), +} + +/// It's used for transmitting and recording the diff internally. +/// +/// It can be convert into a [DocDiff]. +// Internally, we need to batch the diff then calculate the event. Because +// we need to sort the diff by containers' created time, to make sure the +// the path to each container is up-to-date. +#[derive(Debug, Clone)] +pub(crate) struct InternalDocDiff<'a> { + pub(crate) origin: InternalString, + pub(crate) by: EventTriggerKind, + /// The values inside this array is in random order + pub(crate) diff: Cow<'a, [InternalContainerDiff]>, + pub(crate) new_version: Cow<'a, Frontiers>, +} + +impl InternalDocDiff<'_> { + pub fn into_owned(self) -> InternalDocDiff<'static> { + InternalDocDiff { + origin: self.origin, + by: self.by, + diff: Cow::Owned((*self.diff).to_owned()), + new_version: Cow::Owned((*self.new_version).to_owned()), + } + } + + pub fn can_merge(&self, other: &Self) -> bool { + self.by == other.by + } +} + +pub type Path = SmallVec<[Index; 4]>; + +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize, enum_as_inner::EnumAsInner)] +pub enum Index { + Key(InternalString), + Seq(usize), + Node(TreeID), +} + +impl std::fmt::Debug for Index { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Key(arg0) => write!(f, "Index::Key(\"{}\")", arg0), + Self::Seq(arg0) => write!(f, "Index::Seq({})", arg0), + Self::Node(arg0) => write!(f, "Index::Node({})", arg0), + } + } +} + +impl std::fmt::Display for Index { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Index::Key(key) => write!(f, "{}", key), + Index::Seq(s) => write!(f, "{}", s), + Index::Node(id) => write!(f, "{}@{}", id.peer, id.counter), + } + } +} + +impl From for Index { + fn from(s: usize) -> Self { + Index::Seq(s) + } +} + +impl TryFrom<&str> for Index { + type Error = &'static str; + fn try_from(s: &str) -> Result { + if s.is_empty() { + return Ok(Index::Key(InternalString::default())); + } + + let c = s.chars().next().unwrap(); + if c.is_ascii_digit() { + if let Ok(seq) = s.parse::() { + Ok(Index::Seq(seq)) + } else if let Ok(id) = s.try_into() { + Ok(Index::Node(id)) + } else { + Ok(Index::Key(InternalString::from(s))) + } + } else { + Ok(Index::Key(InternalString::from(s))) + } + } +} + +impl DiffVariant { + pub fn compose(self, other: Self) -> Result { + match (self, other) { + (DiffVariant::Internal(a), DiffVariant::Internal(b)) => { + Ok(DiffVariant::Internal(a.compose(b)?)) + } + (DiffVariant::External(a), DiffVariant::External(b)) => { + Ok(DiffVariant::External(a.compose(b)?)) + } + (a, _) => Err(a), + } + } + + pub fn is_empty(&self) -> bool { + match self { + DiffVariant::Internal(diff) => diff.is_empty(), + DiffVariant::External(diff) => diff.is_empty(), + DiffVariant::None => true, + } + } +} + +#[non_exhaustive] +#[derive(Clone, Debug, EnumAsInner)] +pub(crate) enum InternalDiff { + ListRaw(Delta), + /// This always uses entity indexes. + RichtextRaw(DeltaRope), + Map(MapDelta), + Tree(TreeDelta), + MovableList(MovableListInnerDelta), + #[cfg(feature = "counter")] + Counter(f64), + Unknown, +} + +impl From for DiffVariant { + fn from(diff: InternalDiff) -> Self { + DiffVariant::Internal(diff) + } +} + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub struct ListDeltaMeta { + /// Whether the content of the insert is moved from + /// a deletion in the same delta and **the value is not changed**. + /// + /// If true, this op must be a move op under the hood. + /// But an insert created by a move op doesn't necessarily + /// have this flag, because the insert content may not + /// be moved from a deletion in the same delta. + pub from_move: bool, +} + +impl Meta for ListDeltaMeta { + fn is_empty(&self) -> bool { + !self.from_move + } + + fn compose( + &mut self, + other: &Self, + type_pair: (crate::delta::DeltaType, crate::delta::DeltaType), + ) { + // We can't have two Some because we don't have `move_from` for Retain. + // And this function is only called when composing a insert/retain with a retain. + if let (crate::delta::DeltaType::Insert, crate::delta::DeltaType::Insert) = type_pair { + unreachable!() + } + + self.from_move = self.from_move || other.from_move; + } + + fn is_mergeable(&self, other: &Self) -> bool { + self.from_move == other.from_move + } + + fn merge(&mut self, _other: &Self) {} +} + +impl DeltaAttr for ListDeltaMeta { + fn compose(&mut self, other: &Self) { + self.from_move = self.from_move || other.from_move; + } + + fn attr_is_empty(&self) -> bool { + !self.from_move + } +} + +pub type ListDiffInsertItem = ArrayVec; +pub type ListDiffItem = DeltaItem; +pub type ListDiff = DeltaRope; + +#[derive(Default, Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TextMeta(pub FxHashMap); + +impl From for FxHashMap { + fn from(value: TextMeta) -> Self { + value.0 + } +} + +impl From> for TextMeta { + fn from(value: FxHashMap) -> Self { + TextMeta(value) + } +} + +pub type TextDiffItem = DeltaItem; +pub type TextDiff = DeltaRope; + +impl DeltaAttr for TextMeta { + fn compose(&mut self, other: &Self) { + for (key, value) in other.0.iter() { + self.0.insert(key.clone(), value.clone()); + } + } + + fn attr_is_empty(&self) -> bool { + self.0.is_empty() + } +} + +/// Diff is the diff between two versions of a container. +/// It's used to describe the change of a container and the events. +/// +/// # Internal +/// +/// Text index variants: +/// +/// - When `wasm` is enabled, it should use utf16 indexes. +/// - When `wasm` is disabled, it should use unicode indexes. +#[non_exhaustive] +#[derive(Clone, Debug, EnumAsInner)] +pub enum Diff { + List(ListDiff), + /// - When feature `wasm` is enabled, it should use utf16 indexes. + /// - When feature `wasm` is disabled, it should use unicode indexes. + Text(TextDiff), + Map(ResolvedMapDelta), + Tree(TreeDiff), + #[cfg(feature = "counter")] + Counter(f64), + Unknown, +} + +impl From for DiffVariant { + fn from(diff: Diff) -> Self { + DiffVariant::External(diff) + } +} + +impl InternalDiff { + pub(crate) fn is_empty(&self) -> bool { + match self { + InternalDiff::ListRaw(s) => s.is_empty(), + InternalDiff::RichtextRaw(t) => t.is_empty(), + InternalDiff::Map(m) => m.updated.is_empty(), + InternalDiff::Tree(t) => t.is_empty(), + InternalDiff::MovableList(t) => t.is_empty(), + #[cfg(feature = "counter")] + InternalDiff::Counter(c) => c.abs() < f64::EPSILON, + InternalDiff::Unknown => true, + } + } + + pub(crate) fn compose(self, diff: InternalDiff) -> Result { + // PERF: avoid clone + match (self, diff) { + (InternalDiff::ListRaw(a), InternalDiff::ListRaw(b)) => { + Ok(InternalDiff::ListRaw(a.compose(b))) + } + (InternalDiff::RichtextRaw(mut a), InternalDiff::RichtextRaw(b)) => { + a.compose(&b); + Ok(InternalDiff::RichtextRaw(a)) + } + (InternalDiff::Map(a), InternalDiff::Map(b)) => Ok(InternalDiff::Map(a.compose(b))), + (InternalDiff::Tree(a), InternalDiff::Tree(b)) => Ok(InternalDiff::Tree(a.compose(b))), + (a, _) => Err(a), + } + } +} + +impl Diff { + pub fn compose_ref(&mut self, diff: &Diff) { + match (self, diff) { + (Diff::List(a), Diff::List(b)) => { + a.compose(b); + } + (Diff::Text(a), Diff::Text(b)) => { + a.compose(b); + } + (Diff::Map(a), Diff::Map(b)) => { + // Move the accumulator out instead of cloning it, so composing + // a long run of fragments stays linear rather than O(n^2). + *a = std::mem::take(a).compose(b.clone()); + } + (Diff::Tree(a), Diff::Tree(b)) => { + *a = std::mem::take(a).compose(b.clone()); + } + #[cfg(feature = "counter")] + (Diff::Counter(a), Diff::Counter(b)) => *a += b, + (_, _) => unreachable!(), + } + } + + pub fn compose(self, diff: Diff) -> Result { + // PERF: avoid clone + match (self, diff) { + (Diff::List(mut a), Diff::List(b)) => { + a.compose(&b); + Ok(Diff::List(a)) + } + (Diff::Text(mut a), Diff::Text(b)) => { + a.compose(&b); + Ok(Diff::Text(a)) + } + (Diff::Map(a), Diff::Map(b)) => Ok(Diff::Map(a.compose(b))), + + (Diff::Tree(a), Diff::Tree(b)) => Ok(Diff::Tree(a.compose(b))), + #[cfg(feature = "counter")] + (Diff::Counter(a), Diff::Counter(b)) => Ok(Diff::Counter(a + b)), + (a, _) => Err(a), + } + } + + // Transform this diff based on the other diff + pub fn transform(&mut self, other: &Self, left_prior: bool) { + match (self, other) { + (Diff::List(a), Diff::List(b)) => a.transform_(b, left_prior), + (Diff::Text(a), Diff::Text(b)) => a.transform_(b, left_prior), + (Diff::Map(a), Diff::Map(b)) => a.transform(b, left_prior), + (Diff::Tree(a), Diff::Tree(b)) => a.transform(b, left_prior), + #[cfg(feature = "counter")] + (Diff::Counter(_a), Diff::Counter(_b)) => { + // Counter operations commute; no transformation is needed. + } + _ => {} + } + } + + pub fn is_empty(&self) -> bool { + match self { + Diff::List(s) => s.is_empty(), + Diff::Text(t) => t.is_empty(), + Diff::Map(m) => m.updated.is_empty(), + Diff::Tree(t) => t.diff.is_empty(), + #[cfg(feature = "counter")] + Diff::Counter(c) => c.abs() < f64::EPSILON, + Diff::Unknown => true, + } + } + + #[allow(unused)] + pub(crate) fn concat(self, diff: Diff) -> Diff { + match (self, diff) { + (Diff::List(mut a), Diff::List(b)) => { + a.compose(&b); + Diff::List(a) + } + (Diff::Text(mut a), Diff::Text(b)) => { + a.compose(&b); + Diff::Text(a) + } + (Diff::Map(a), Diff::Map(b)) => { + let mut a = a; + for (k, v) in b.updated { + a = a.with_entry(k, v); + } + Diff::Map(a) + } + + (Diff::Tree(a), Diff::Tree(b)) => Diff::Tree(a.extend(b.diff)), + #[cfg(feature = "counter")] + (Diff::Counter(a), Diff::Counter(b)) => Diff::Counter(a + b), + _ => unreachable!(), + } + } + + /// Transform the cursor based on this diff + pub(crate) fn transform_cursor(&self, pos: usize, left_prior: bool) -> usize { + match self { + Diff::List(list) => list.transform_pos(pos, left_prior), + Diff::Text(text) => text.transform_pos(pos, left_prior), + _ => pos, + } + } +} + +pub fn str_to_path(s: &str) -> Option> { + s.split('/').map(|x| x.try_into()).try_collect().ok() +} + +pub fn path_to_str(path: &[Index]) -> String { + path.iter().map(|x| x.to_string()).join("/") +} + +#[cfg(test)] +mod test { + use std::sync::Arc; + + use itertools::Itertools; + use loro_common::LoroValue; + + use crate::{cursor::PosType, ApplyDiff, LoroDoc}; + + #[test] + fn test_text_event() { + let loro = LoroDoc::new(); + let _g = loro.subscribe_root(Arc::new(|event| { + let mut value = LoroValue::String(Default::default()); + value.apply_diff(&event.events.iter().map(|x| x.diff.clone()).collect_vec()); + assert_eq!(value, "h223ello".into()); + })); + let mut txn = loro.txn().unwrap(); + let text = loro.get_text("id"); + text.insert_with_txn(&mut txn, 0, "hello", PosType::Unicode) + .unwrap(); + text.insert_with_txn(&mut txn, 1, "223", PosType::Unicode) + .unwrap(); + txn.commit().unwrap(); + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/fork.rs b/3rdparty/loro/crates/loro-internal/src/fork.rs new file mode 100644 index 0000000000..371a82c0ca --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/fork.rs @@ -0,0 +1,52 @@ +//! The fork module provides functionality to create a new LoroDoc instance at a specified version +//! (Frontiers) with minimal overhead. +//! +//! # Implementation Overview +//! +//! The `fork_at` function in this module allows for the creation of a new document that reflects +//! the state of the original document at a given version. The function achieves this by: +//! +//! ## Exporting Necessary Data: +//! +//! - **Change Store Data**: Collects all changes up to the specified version from the change +//! store's key-value (kv) data store. It includes the version vector and frontiers for accurate +//! identification of the version. +//! +//! - **Container Store Data**: Exports the container store's kv data representing the document's +//! state at the specified version. This involves checking out to the desired version, exporting +//! the state, and efficiently checking back to the latest version. +//! +//! - **GC Store Data**: If applicable, exports the gc store's kv data, ensuring that version +//! identifiers are included. +//! +//! ## Reconstructing the New Document: +//! +//! Imports the exported data into a new LoroDoc instance using optimized import mechanisms +//! similar to those used in fast snapshot imports. +//! +//! By focusing on exporting only the necessary data and optimizing state transitions during +//! version checkout, the `fork_at` function minimizes overhead and efficiently creates new +//! document instances representing past versions. +//! +use std::borrow::Cow; + +use crate::{version::Frontiers, LoroDoc}; +use loro_common::{LoroError, LoroResult}; + +impl LoroDoc { + /// Creates a new LoroDoc at a specified version (Frontiers) + pub fn fork_at(&self, frontiers: &Frontiers) -> LoroResult { + let bytes = self + .export(crate::loro::ExportMode::SnapshotAt { + version: Cow::Borrowed(frontiers), + }) + .map_err(LoroError::from)?; + let doc = LoroDoc::new(); + doc.set_config(&self.config); + if self.auto_commit.load(std::sync::atomic::Ordering::Relaxed) { + doc.start_auto_commit(); + } + doc.import(&bytes)?; + Ok(doc) + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/fuzz.rs b/3rdparty/loro/crates/loro-internal/src/fuzz.rs new file mode 100644 index 0000000000..6c22cdddd8 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/fuzz.rs @@ -0,0 +1,89 @@ +use std::{fmt::Debug, time::Instant}; + +pub fn minify_error(site_num: u8, actions: Vec, f: F, normalize: N) +where + F: Fn(u8, &mut [T]), + N: Fn(u8, &mut [T]) -> Vec, + T: Clone + Debug, +{ + std::panic::set_hook(Box::new(|_info| { + // ignore panic output + // println!("{:?}", _info); + })); + + let f_ref: *const _ = &f; + let f_ref: usize = f_ref as usize; + #[allow(clippy::redundant_clone)] + let mut actions_clone = actions.clone(); + let action_ref: usize = (&mut actions_clone) as *mut _ as usize; + #[allow(clippy::blocks_in_conditions)] + if std::panic::catch_unwind(|| { + // SAFETY: test + let f = unsafe { &*(f_ref as *const F) }; + // SAFETY: test + let actions_ref = unsafe { &mut *(action_ref as *mut Vec) }; + f(site_num, actions_ref); + }) + .is_ok() + { + println!("No Error Found"); + return; + } + + let mut minified = actions.clone(); + let mut candidates = Vec::new(); + for i in 0..actions.len() { + let mut new = actions.clone(); + new.remove(i); + candidates.push(new); + } + + println!("Minifying..."); + let start = Instant::now(); + while let Some(candidate) = candidates.pop() { + let f_ref: *const _ = &f; + let f_ref: usize = f_ref as usize; + let mut actions_clone = candidate.clone(); + let action_ref: usize = (&mut actions_clone) as *mut _ as usize; + #[allow(clippy::blocks_in_conditions)] + if std::panic::catch_unwind(|| { + // SAFETY: test + let f = unsafe { &*(f_ref as *const F) }; + // SAFETY: test + let actions_ref = unsafe { &mut *(action_ref as *mut Vec) }; + f(site_num, actions_ref); + }) + .is_err() + { + for i in 0..candidate.len() { + let mut new = candidate.clone(); + new.remove(i); + candidates.push(new); + } + if candidate.len() < minified.len() { + minified = candidate; + println!("New min len={}", minified.len()); + } + if candidates.len() > 40 { + candidates.drain(0..30); + } + } + if start.elapsed().as_secs() > 10 && minified.len() <= 4 { + break; + } + if start.elapsed().as_secs() > 60 { + break; + } + } + + let minified = normalize(site_num, &mut minified); + println!( + "Old Length {}, New Length {}", + actions.len(), + minified.len() + ); + dbg!(&minified); + if actions.len() > minified.len() { + minify_error(site_num, minified, f, normalize); + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/handler.rs b/3rdparty/loro/crates/loro-internal/src/handler.rs new file mode 100644 index 0000000000..a86e9bb2d4 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/handler.rs @@ -0,0 +1,5738 @@ +use super::{state::DocState, txn::Transaction}; +use crate::sync::Mutex; +use crate::{ + container::{ + idx::ContainerIdx, + list::list_op::{DeleteSpan, DeleteSpanWithId, ListOp}, + richtext::{richtext_state::PosType, RichtextState, StyleKey, StyleOp, TextStyleInfoFlag}, + }, + cursor::{Cursor, Side}, + delta::{DeltaItem, Meta, StyleMeta, TreeExternalDiff}, + diff::{diff, diff_impl::UpdateTimeoutError, OperateProxy}, + event::{Diff, TextDiff, TextDiffItem, TextMeta}, + op::ListSlice, + state::{IndexType, State, TreeParentId}, + txn::EventHint, + utils::{string_slice::StringSlice, utf16::count_utf16_len}, + LoroDoc, LoroDocInner, +}; +use append_only_bytes::BytesSlice; +use enum_as_inner::EnumAsInner; +use generic_btree::rle::HasLength; +use loro_common::{ + ContainerID, ContainerType, IdFull, InternalString, LoroError, LoroResult, LoroValue, PeerID, + TreeID, ID, +}; +use rustc_hash::FxHashMap; +use serde::{Deserialize, Serialize}; +use std::{borrow::Cow, cmp::Reverse, collections::BinaryHeap, fmt::Debug, ops::Deref, sync::Arc}; +use tracing::{error, instrument}; + +pub use crate::diff::diff_impl::UpdateOptions; +pub use tree::TreeHandler; +mod movable_list_apply_delta; +mod tree; + +const REGULAR_CONTAINER_VALUE_ARG_ERROR: &str = + "Cannot use a LoroValue::Container as a regular value. To create a child container, use insert_container/set_container, or ensure_mergeable_* on maps for mergeable children"; + +mod text_update; + +fn ensure_no_regular_container_value(value: &LoroValue) -> LoroResult<()> { + // Fast path: scalar values can never transitively hold a container, so we + // skip the heap allocation + traversal below. This is the common case on + // the per-op insert hot path (inserting numbers/strings/bools), where the + // previous unconditional `vec![value]` allocation showed up as a measurable + // regression. + if !matches!( + value, + LoroValue::Container(_) | LoroValue::List(_) | LoroValue::Map(_) + ) { + return Ok(()); + } + + let mut stack = vec![value]; + while let Some(value) = stack.pop() { + match value { + LoroValue::Container(_) => { + return Err(LoroError::ArgErr( + REGULAR_CONTAINER_VALUE_ARG_ERROR + .to_string() + .into_boxed_str(), + )); + } + LoroValue::List(list) => { + stack.extend(list.iter()); + } + LoroValue::Map(map) => { + stack.extend(map.values()); + } + LoroValue::Null + | LoroValue::Bool(_) + | LoroValue::Double(_) + | LoroValue::I64(_) + | LoroValue::Binary(_) + | LoroValue::String(_) => {} + } + } + + Ok(()) +} + +fn checked_range_end( + pos: usize, + len: usize, + container_len: usize, + // Lazily built: this is on the per-op edit hot path, so the position-context + // string must only be allocated when a bound check actually fails. + info: impl Fn() -> Box, +) -> LoroResult { + let end = pos.checked_add(len).ok_or_else(|| LoroError::OutOfBound { + pos: usize::MAX, + len: container_len, + info: info(), + })?; + if end > container_len { + return Err(LoroError::OutOfBound { + pos: end, + len: container_len, + info: info(), + }); + } + + Ok(end) +} + +fn checked_delta_index_end(pos: usize, len: usize, container_len: usize) -> LoroResult { + pos.checked_add(len).ok_or_else(|| LoroError::OutOfBound { + pos: usize::MAX, + len: container_len, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + }) +} + +pub trait HandlerTrait: Clone + Sized { + fn is_attached(&self) -> bool; + fn attached_handler(&self) -> Option<&BasicHandler>; + fn get_value(&self) -> LoroValue; + fn get_deep_value(&self) -> LoroValue; + fn kind(&self) -> ContainerType; + fn to_handler(&self) -> Handler; + fn from_handler(h: Handler) -> Option; + fn doc(&self) -> Option; + /// This method returns an attached handler. + fn attach( + &self, + txn: &mut Transaction, + parent: &BasicHandler, + self_id: ContainerID, + ) -> LoroResult; + /// If a detached container is attached, this method will return its corresponding attached handler. + fn get_attached(&self) -> Option; + + fn parent(&self) -> Option { + self.attached_handler().and_then(|x| x.parent()) + } + + fn idx(&self) -> ContainerIdx { + self.attached_handler() + .map(|x| x.container_idx) + .unwrap_or_else(|| { + ContainerIdx::from_index_and_type(ContainerIdx::INDEX_MASK, self.kind()) + }) + } + + fn id(&self) -> ContainerID { + self.attached_handler() + .map(|x| x.id.clone()) + .unwrap_or_else(|| ContainerID::new_normal(ID::NONE_ID, self.kind())) + } + + fn with_state(&self, f: impl FnOnce(&mut State) -> LoroResult) -> LoroResult { + let inner = self + .attached_handler() + .ok_or(LoroError::MisuseDetachedContainer { + method: "with_state", + })?; + let state = inner.doc.state.clone(); + let mut guard = state.lock(); + guard.with_state_mut(inner.container_idx, f) + } +} + +fn create_handler(inner: &BasicHandler, id: ContainerID) -> Handler { + Handler::new_attached(id, inner.doc.clone()) +} + +fn value_to_value_or_handler(inner: &BasicHandler, value: LoroValue) -> ValueOrHandler { + match value { + LoroValue::Container(container_id) => { + ValueOrHandler::Handler(create_handler(inner, container_id)) + } + value => ValueOrHandler::Value(value), + } +} + +/// Flatten attributes that allow overlap +#[derive(Clone, Debug)] +pub struct BasicHandler { + id: ContainerID, + container_idx: ContainerIdx, + doc: LoroDoc, +} + +struct DetachedInner { + value: T, + /// If the handler attached later, this field will be filled. + attached: Option, +} + +impl DetachedInner { + fn new(v: T) -> Self { + Self { + value: v, + attached: None, + } + } +} + +enum MaybeDetached { + Detached(Arc>>), + Attached(BasicHandler), +} + +impl Clone for MaybeDetached { + fn clone(&self) -> Self { + match self { + MaybeDetached::Detached(a) => MaybeDetached::Detached(Arc::clone(a)), + MaybeDetached::Attached(a) => MaybeDetached::Attached(a.clone()), + } + } +} + +impl MaybeDetached { + fn new_detached(v: T) -> Self { + MaybeDetached::Detached(Arc::new(Mutex::new(DetachedInner::new(v)))) + } + + fn is_attached(&self) -> bool { + match self { + MaybeDetached::Detached(_) => false, + MaybeDetached::Attached(_) => true, + } + } + + fn attached_handler(&self) -> Option<&BasicHandler> { + match self { + MaybeDetached::Detached(_) => None, + MaybeDetached::Attached(a) => Some(a), + } + } + + fn try_attached_state(&self) -> LoroResult<&BasicHandler> { + match self { + MaybeDetached::Detached(_) => Err(LoroError::MisuseDetachedContainer { + method: "inner_state", + }), + MaybeDetached::Attached(a) => Ok(a), + } + } +} + +impl From for MaybeDetached { + fn from(a: BasicHandler) -> Self { + MaybeDetached::Attached(a) + } +} + +impl BasicHandler { + pub(crate) fn doc(&self) -> LoroDoc { + self.doc.clone() + } + + #[inline] + fn with_doc_state(&self, f: impl FnOnce(&mut DocState) -> R) -> R { + let state = self.doc.state.clone(); + let mut guard = state.lock(); + f(&mut guard) + } + + fn with_txn( + &self, + f: impl FnOnce(&mut Transaction) -> Result, + ) -> Result { + with_txn(&self.doc, f) + } + + fn get_parent(&self) -> Option { + let parent_idx = self.doc.arena.get_parent(self.container_idx)?; + let parent_id = self.doc.arena.get_container_id(parent_idx).unwrap(); + { + let kind = parent_id.container_type(); + let handler = BasicHandler { + container_idx: parent_idx, + id: parent_id, + doc: self.doc.clone(), + }; + + Some(match kind { + ContainerType::Map => Handler::Map(MapHandler { + inner: handler.into(), + }), + ContainerType::List => Handler::List(ListHandler { + inner: handler.into(), + }), + ContainerType::Tree => Handler::Tree(TreeHandler { + inner: handler.into(), + }), + ContainerType::Text => Handler::Text(TextHandler { + inner: handler.into(), + }), + ContainerType::MovableList => Handler::MovableList(MovableListHandler { + inner: handler.into(), + }), + #[cfg(feature = "counter")] + ContainerType::Counter => Handler::Counter(counter::CounterHandler { + inner: handler.into(), + }), + ContainerType::Unknown(_) => unreachable!(), + }) + } + } + + pub fn get_value(&self) -> LoroValue { + self.doc.state.lock().get_value_by_idx(self.container_idx) + } + + pub fn get_deep_value(&self) -> LoroValue { + self.doc + .state + .lock() + .get_container_deep_value(self.container_idx) + } + + fn with_state(&self, f: impl FnOnce(&mut State) -> R) -> R { + let mut guard = self.doc.state.lock(); + guard.with_state_mut(self.container_idx, f) + } + + pub fn parent(&self) -> Option { + self.get_parent() + } + + fn is_deleted(&self) -> bool { + self.doc.state.lock().is_deleted(self.container_idx) + } + + fn has_decoded_state(&self) -> bool { + self.with_doc_state(|state| state.has_decoded_container_state(self.container_idx)) + } +} + +/// Flatten attributes that allow overlap +#[derive(Clone)] +pub struct TextHandler { + inner: MaybeDetached, +} + +impl HandlerTrait for TextHandler { + fn to_handler(&self) -> Handler { + Handler::Text(self.clone()) + } + + fn attach( + &self, + txn: &mut Transaction, + parent: &BasicHandler, + self_id: ContainerID, + ) -> LoroResult { + match &self.inner { + MaybeDetached::Detached(t) => { + let mut t = t.lock(); + let inner = create_handler(parent, self_id); + let text = inner.into_text().unwrap(); + let mut delta: Vec = Vec::new(); + for span in t.value.iter() { + delta.push(TextDelta::Insert { + insert: span.text.to_string(), + attributes: span.attributes.to_option_map(), + }); + } + + text.apply_delta_with_txn(txn, &delta)?; + t.attached = text.attached_handler().cloned(); + Ok(text) + } + MaybeDetached::Attached(a) => { + let new_inner = create_handler(a, self_id); + let ans = new_inner.into_text().unwrap(); + + let delta = self.get_delta(); + ans.apply_delta_with_txn(txn, &delta)?; + Ok(ans) + } + } + } + + fn attached_handler(&self) -> Option<&BasicHandler> { + self.inner.attached_handler() + } + + fn get_value(&self) -> LoroValue { + match &self.inner { + MaybeDetached::Detached(t) => { + let t = t.lock(); + LoroValue::String((t.value.to_string()).into()) + } + MaybeDetached::Attached(a) => a.get_value(), + } + } + + fn get_deep_value(&self) -> LoroValue { + self.get_value() + } + + fn is_attached(&self) -> bool { + matches!(&self.inner, MaybeDetached::Attached(..)) + } + + fn kind(&self) -> ContainerType { + ContainerType::Text + } + + fn get_attached(&self) -> Option { + match &self.inner { + MaybeDetached::Detached(d) => d.lock().attached.clone().map(|x| Self { + inner: MaybeDetached::Attached(x), + }), + MaybeDetached::Attached(_a) => Some(self.clone()), + } + } + + fn from_handler(h: Handler) -> Option { + match h { + Handler::Text(x) => Some(x), + _ => None, + } + } + + fn doc(&self) -> Option { + match &self.inner { + MaybeDetached::Detached(_) => None, + MaybeDetached::Attached(a) => Some(a.doc()), + } + } +} + +impl std::fmt::Debug for TextHandler { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match &self.inner { + MaybeDetached::Detached(_) => { + write!(f, "TextHandler(Unattached)") + } + MaybeDetached::Attached(a) => { + write!(f, "TextHandler({:?})", &a.id) + } + } + } +} + +#[derive(Debug, Clone, EnumAsInner, Deserialize, Serialize, PartialEq)] +#[serde(untagged)] +pub enum TextDelta { + Retain { + retain: usize, + attributes: Option>, + }, + Insert { + insert: String, + attributes: Option>, + }, + Delete { + delete: usize, + }, +} + +impl TextDelta { + pub fn from_text_diff<'a>(diff: impl Iterator) -> Vec { + let mut ans = Vec::with_capacity(diff.size_hint().0); + for iter in diff { + match iter { + loro_delta::DeltaItem::Retain { len, attr } => { + ans.push(TextDelta::Retain { + retain: *len, + attributes: if attr.0.is_empty() { + None + } else { + Some(attr.0.clone()) + }, + }); + } + loro_delta::DeltaItem::Replace { + value, + attr, + delete, + } => { + if value.rle_len() > 0 { + ans.push(TextDelta::Insert { + insert: value.to_string(), + attributes: if attr.0.is_empty() { + None + } else { + Some(attr.0.clone()) + }, + }); + } + if *delete > 0 { + ans.push(TextDelta::Delete { delete: *delete }); + } + } + } + } + + ans + } + + pub fn into_text_diff(vec: impl Iterator) -> TextDiff { + let mut delta = TextDiff::new(); + for item in vec { + match item { + TextDelta::Retain { retain, attributes } => { + delta.push_retain(retain, TextMeta(attributes.unwrap_or_default().clone())); + } + TextDelta::Insert { insert, attributes } => { + delta.push_insert( + StringSlice::from(insert.as_str()), + TextMeta(attributes.unwrap_or_default()), + ); + } + TextDelta::Delete { delete } => { + delta.push_delete(delete); + } + } + } + + delta + } +} + +impl From<&DeltaItem> for TextDelta { + fn from(value: &DeltaItem) -> Self { + match value { + crate::delta::DeltaItem::Retain { retain, attributes } => TextDelta::Retain { + retain: *retain, + attributes: attributes.to_option_map(), + }, + crate::delta::DeltaItem::Insert { insert, attributes } => TextDelta::Insert { + insert: insert.to_string(), + attributes: attributes.to_option_map(), + }, + crate::delta::DeltaItem::Delete { + delete, + attributes: _, + } => TextDelta::Delete { delete: *delete }, + } + } +} + +#[derive(Clone)] +pub struct MapHandler { + inner: MaybeDetached>, +} + +impl HandlerTrait for MapHandler { + fn is_attached(&self) -> bool { + matches!(&self.inner, MaybeDetached::Attached(..)) + } + + fn attached_handler(&self) -> Option<&BasicHandler> { + match &self.inner { + MaybeDetached::Detached(_) => None, + MaybeDetached::Attached(a) => Some(a), + } + } + + fn get_value(&self) -> LoroValue { + match &self.inner { + MaybeDetached::Detached(m) => { + let m = m.lock(); + let mut map = FxHashMap::default(); + for (k, v) in m.value.iter() { + map.insert(k.to_string(), v.to_value()); + } + LoroValue::Map(map.into()) + } + MaybeDetached::Attached(a) => a.get_value(), + } + } + + fn get_deep_value(&self) -> LoroValue { + match &self.inner { + MaybeDetached::Detached(m) => { + let m = m.lock(); + let mut map = FxHashMap::default(); + for (k, v) in m.value.iter() { + map.insert(k.to_string(), v.to_deep_value()); + } + LoroValue::Map(map.into()) + } + MaybeDetached::Attached(a) => a.get_deep_value(), + } + } + + fn kind(&self) -> ContainerType { + ContainerType::Map + } + + fn to_handler(&self) -> Handler { + Handler::Map(self.clone()) + } + + fn attach( + &self, + txn: &mut Transaction, + parent: &BasicHandler, + self_id: ContainerID, + ) -> LoroResult { + match &self.inner { + MaybeDetached::Detached(m) => { + let mut m = m.lock(); + let inner = create_handler(parent, self_id); + let map = inner.into_map().unwrap(); + for (k, v) in m.value.iter() { + match v { + ValueOrHandler::Value(v) => { + map.insert_with_txn(txn, k, v.clone())?; + } + ValueOrHandler::Handler(h) => { + map.insert_container_with_txn(txn, k, h.clone())?; + } + } + } + m.attached = map.attached_handler().cloned(); + Ok(map) + } + MaybeDetached::Attached(a) => { + let new_inner = create_handler(a, self_id); + let ans = new_inner.into_map().unwrap(); + + for (k, v) in self.get_value().into_map().unwrap().iter() { + if let LoroValue::Container(id) = v { + ans.insert_container_with_txn(txn, k, create_handler(a, id.clone()))?; + } else { + ans.insert_with_txn(txn, k, v.clone())?; + } + } + + Ok(ans) + } + } + } + + fn get_attached(&self) -> Option { + match &self.inner { + MaybeDetached::Detached(d) => d.lock().attached.clone().map(|x| Self { + inner: MaybeDetached::Attached(x), + }), + MaybeDetached::Attached(_a) => Some(self.clone()), + } + } + + fn from_handler(h: Handler) -> Option { + match h { + Handler::Map(x) => Some(x), + _ => None, + } + } + + fn doc(&self) -> Option { + match &self.inner { + MaybeDetached::Detached(_) => None, + MaybeDetached::Attached(a) => Some(a.doc()), + } + } +} + +impl std::fmt::Debug for MapHandler { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match &self.inner { + MaybeDetached::Detached(_) => write!(f, "MapHandler Detached"), + MaybeDetached::Attached(a) => write!(f, "MapHandler {}", a.id), + } + } +} + +#[derive(Clone)] +pub struct ListHandler { + inner: MaybeDetached>, +} + +#[derive(Clone)] +pub struct MovableListHandler { + inner: MaybeDetached>, +} + +impl HandlerTrait for MovableListHandler { + fn is_attached(&self) -> bool { + matches!(&self.inner, MaybeDetached::Attached(..)) + } + + fn attached_handler(&self) -> Option<&BasicHandler> { + match &self.inner { + MaybeDetached::Detached(_) => None, + MaybeDetached::Attached(a) => Some(a), + } + } + + fn get_value(&self) -> LoroValue { + match &self.inner { + MaybeDetached::Detached(a) => { + let a = a.lock(); + LoroValue::List(a.value.iter().map(|v| v.to_value()).collect()) + } + MaybeDetached::Attached(a) => a.get_value(), + } + } + + fn get_deep_value(&self) -> LoroValue { + match &self.inner { + MaybeDetached::Detached(a) => { + let a = a.lock(); + LoroValue::List(a.value.iter().map(|v| v.to_deep_value()).collect()) + } + MaybeDetached::Attached(a) => a.get_deep_value(), + } + } + + fn kind(&self) -> ContainerType { + ContainerType::MovableList + } + + fn to_handler(&self) -> Handler { + Handler::MovableList(self.clone()) + } + + fn from_handler(h: Handler) -> Option { + match h { + Handler::MovableList(x) => Some(x), + _ => None, + } + } + + fn attach( + &self, + txn: &mut Transaction, + parent: &BasicHandler, + self_id: ContainerID, + ) -> LoroResult { + match &self.inner { + MaybeDetached::Detached(l) => { + let mut l = l.lock(); + let inner = create_handler(parent, self_id); + let list = inner.into_movable_list().unwrap(); + for (index, v) in l.value.iter().enumerate() { + match v { + ValueOrHandler::Value(v) => { + list.insert_with_txn(txn, index, v.clone())?; + } + ValueOrHandler::Handler(h) => { + list.insert_container_with_txn(txn, index, h.clone())?; + } + } + } + l.attached = list.attached_handler().cloned(); + Ok(list) + } + MaybeDetached::Attached(a) => { + let new_inner = create_handler(a, self_id); + let ans = new_inner.into_movable_list().unwrap(); + + for (i, v) in self.get_value().into_list().unwrap().iter().enumerate() { + if let LoroValue::Container(id) = v { + ans.insert_container_with_txn(txn, i, create_handler(a, id.clone()))?; + } else { + ans.insert_with_txn(txn, i, v.clone())?; + } + } + + Ok(ans) + } + } + } + + fn get_attached(&self) -> Option { + match &self.inner { + MaybeDetached::Detached(d) => d.lock().attached.clone().map(|x| Self { + inner: MaybeDetached::Attached(x), + }), + MaybeDetached::Attached(_a) => Some(self.clone()), + } + } + + fn doc(&self) -> Option { + match &self.inner { + MaybeDetached::Detached(_) => None, + MaybeDetached::Attached(a) => Some(a.doc()), + } + } +} + +impl std::fmt::Debug for MovableListHandler { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "MovableListHandler {}", self.id()) + } +} + +impl std::fmt::Debug for ListHandler { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match &self.inner { + MaybeDetached::Detached(_) => write!(f, "ListHandler Detached"), + MaybeDetached::Attached(a) => write!(f, "ListHandler {}", a.id), + } + } +} + +impl HandlerTrait for ListHandler { + fn is_attached(&self) -> bool { + self.inner.is_attached() + } + + fn attached_handler(&self) -> Option<&BasicHandler> { + self.inner.attached_handler() + } + + fn get_value(&self) -> LoroValue { + match &self.inner { + MaybeDetached::Detached(a) => { + let a = a.lock(); + LoroValue::List(a.value.iter().map(|v| v.to_value()).collect()) + } + MaybeDetached::Attached(a) => a.get_value(), + } + } + + fn get_deep_value(&self) -> LoroValue { + match &self.inner { + MaybeDetached::Detached(a) => { + let a = a.lock(); + LoroValue::List(a.value.iter().map(|v| v.to_deep_value()).collect()) + } + MaybeDetached::Attached(a) => a.get_deep_value(), + } + } + + fn kind(&self) -> ContainerType { + ContainerType::List + } + + fn to_handler(&self) -> Handler { + Handler::List(self.clone()) + } + + fn attach( + &self, + txn: &mut Transaction, + parent: &BasicHandler, + self_id: ContainerID, + ) -> LoroResult { + match &self.inner { + MaybeDetached::Detached(l) => { + let mut l = l.lock(); + let inner = create_handler(parent, self_id); + let list = inner.into_list().unwrap(); + for (index, v) in l.value.iter().enumerate() { + match v { + ValueOrHandler::Value(v) => { + list.insert_with_txn(txn, index, v.clone())?; + } + ValueOrHandler::Handler(h) => { + list.insert_container_with_txn(txn, index, h.clone())?; + } + } + } + l.attached = list.attached_handler().cloned(); + Ok(list) + } + MaybeDetached::Attached(a) => { + let new_inner = create_handler(a, self_id); + let ans = new_inner.into_list().unwrap(); + + for (i, v) in self.get_value().into_list().unwrap().iter().enumerate() { + if let LoroValue::Container(id) = v { + ans.insert_container_with_txn(txn, i, create_handler(a, id.clone()))?; + } else { + ans.insert_with_txn(txn, i, v.clone())?; + } + } + + Ok(ans) + } + } + } + + fn get_attached(&self) -> Option { + match &self.inner { + MaybeDetached::Detached(d) => d.lock().attached.clone().map(|x| Self { + inner: MaybeDetached::Attached(x), + }), + MaybeDetached::Attached(_a) => Some(self.clone()), + } + } + + fn from_handler(h: Handler) -> Option { + match h { + Handler::List(x) => Some(x), + _ => None, + } + } + + fn doc(&self) -> Option { + match &self.inner { + MaybeDetached::Detached(_) => None, + MaybeDetached::Attached(a) => Some(a.doc()), + } + } +} + +#[derive(Clone)] +pub struct UnknownHandler { + inner: BasicHandler, +} + +impl Debug for UnknownHandler { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "UnknownHandler") + } +} + +impl UnknownHandler { + pub fn is_deleted(&self) -> bool { + self.inner.is_deleted() + } +} + +impl HandlerTrait for UnknownHandler { + fn is_attached(&self) -> bool { + true + } + + fn attached_handler(&self) -> Option<&BasicHandler> { + Some(&self.inner) + } + + fn get_value(&self) -> LoroValue { + todo!() + } + + fn get_deep_value(&self) -> LoroValue { + todo!() + } + + fn kind(&self) -> ContainerType { + self.inner.id.container_type() + } + + fn to_handler(&self) -> Handler { + Handler::Unknown(self.clone()) + } + + fn from_handler(h: Handler) -> Option { + match h { + Handler::Unknown(x) => Some(x), + _ => None, + } + } + + fn attach( + &self, + _txn: &mut Transaction, + _parent: &BasicHandler, + self_id: ContainerID, + ) -> LoroResult { + let new_inner = create_handler(&self.inner, self_id); + let ans = new_inner.into_unknown().unwrap(); + Ok(ans) + } + + fn get_attached(&self) -> Option { + Some(self.clone()) + } + + fn doc(&self) -> Option { + Some(self.inner.doc()) + } +} + +#[derive(Clone, EnumAsInner, Debug)] +pub enum Handler { + Text(TextHandler), + Map(MapHandler), + List(ListHandler), + MovableList(MovableListHandler), + Tree(TreeHandler), + #[cfg(feature = "counter")] + Counter(counter::CounterHandler), + Unknown(UnknownHandler), +} + +impl HandlerTrait for Handler { + fn is_attached(&self) -> bool { + match self { + Self::Text(x) => x.is_attached(), + Self::Map(x) => x.is_attached(), + Self::List(x) => x.is_attached(), + Self::Tree(x) => x.is_attached(), + Self::MovableList(x) => x.is_attached(), + #[cfg(feature = "counter")] + Self::Counter(x) => x.is_attached(), + Self::Unknown(x) => x.is_attached(), + } + } + + fn attached_handler(&self) -> Option<&BasicHandler> { + match self { + Self::Text(x) => x.attached_handler(), + Self::Map(x) => x.attached_handler(), + Self::List(x) => x.attached_handler(), + Self::MovableList(x) => x.attached_handler(), + Self::Tree(x) => x.attached_handler(), + #[cfg(feature = "counter")] + Self::Counter(x) => x.attached_handler(), + Self::Unknown(x) => x.attached_handler(), + } + } + + fn get_value(&self) -> LoroValue { + match self { + Self::Text(x) => x.get_value(), + Self::Map(x) => x.get_value(), + Self::List(x) => x.get_value(), + Self::MovableList(x) => x.get_value(), + Self::Tree(x) => x.get_value(), + #[cfg(feature = "counter")] + Self::Counter(x) => x.get_value(), + Self::Unknown(x) => x.get_value(), + } + } + + fn get_deep_value(&self) -> LoroValue { + match self { + Self::Text(x) => x.get_deep_value(), + Self::Map(x) => x.get_deep_value(), + Self::List(x) => x.get_deep_value(), + Self::MovableList(x) => x.get_deep_value(), + Self::Tree(x) => x.get_deep_value(), + #[cfg(feature = "counter")] + Self::Counter(x) => x.get_deep_value(), + Self::Unknown(x) => x.get_deep_value(), + } + } + + fn kind(&self) -> ContainerType { + match self { + Self::Text(x) => x.kind(), + Self::Map(x) => x.kind(), + Self::List(x) => x.kind(), + Self::MovableList(x) => x.kind(), + Self::Tree(x) => x.kind(), + #[cfg(feature = "counter")] + Self::Counter(x) => x.kind(), + Self::Unknown(x) => x.kind(), + } + } + + fn to_handler(&self) -> Handler { + match self { + Self::Text(x) => x.to_handler(), + Self::Map(x) => x.to_handler(), + Self::List(x) => x.to_handler(), + Self::MovableList(x) => x.to_handler(), + Self::Tree(x) => x.to_handler(), + #[cfg(feature = "counter")] + Self::Counter(x) => x.to_handler(), + Self::Unknown(x) => x.to_handler(), + } + } + + fn attach( + &self, + txn: &mut Transaction, + parent: &BasicHandler, + self_id: ContainerID, + ) -> LoroResult { + match self { + Self::Text(x) => Ok(Handler::Text(x.attach(txn, parent, self_id)?)), + Self::Map(x) => Ok(Handler::Map(x.attach(txn, parent, self_id)?)), + Self::List(x) => Ok(Handler::List(x.attach(txn, parent, self_id)?)), + Self::MovableList(x) => Ok(Handler::MovableList(x.attach(txn, parent, self_id)?)), + Self::Tree(x) => Ok(Handler::Tree(x.attach(txn, parent, self_id)?)), + #[cfg(feature = "counter")] + Self::Counter(x) => Ok(Handler::Counter(x.attach(txn, parent, self_id)?)), + Self::Unknown(x) => Ok(Handler::Unknown(x.attach(txn, parent, self_id)?)), + } + } + + fn get_attached(&self) -> Option { + match self { + Self::Text(x) => x.get_attached().map(Handler::Text), + Self::Map(x) => x.get_attached().map(Handler::Map), + Self::List(x) => x.get_attached().map(Handler::List), + Self::MovableList(x) => x.get_attached().map(Handler::MovableList), + Self::Tree(x) => x.get_attached().map(Handler::Tree), + #[cfg(feature = "counter")] + Self::Counter(x) => x.get_attached().map(Handler::Counter), + Self::Unknown(x) => x.get_attached().map(Handler::Unknown), + } + } + + fn from_handler(h: Handler) -> Option { + Some(h) + } + + fn doc(&self) -> Option { + match self { + Self::Text(x) => x.doc(), + Self::Map(x) => x.doc(), + Self::List(x) => x.doc(), + Self::MovableList(x) => x.doc(), + Self::Tree(x) => x.doc(), + #[cfg(feature = "counter")] + Self::Counter(x) => x.doc(), + Self::Unknown(x) => x.doc(), + } + } +} + +impl Handler { + fn apply_map_container_diff_value( + map: &MapHandler, + key: &str, + old_id: ContainerID, + on_container_remap: &mut dyn FnMut(ContainerID, ContainerID), + ) -> LoroResult<()> { + if old_id.is_mergeable() { + let parent_id = map.id(); + let kind = old_id.container_type(); + let new_id = ContainerID::new_mergeable(&parent_id, key, kind); + let marker = loro_common::mergeable_marker(&parent_id, key, kind); + map.insert_without_skipping(key, marker)?; + on_container_remap(old_id, new_id); + return Ok(()); + } + + let new_h = map.insert_container(key, Handler::new_unattached(old_id.container_type()))?; + let new_id = new_h.id(); + on_container_remap(old_id, new_id); + Ok(()) + } + + pub(crate) fn new_attached(id: ContainerID, doc: LoroDoc) -> Self { + let kind = id.container_type(); + let handler = BasicHandler { + container_idx: doc.arena.register_container(&id), + id, + doc, + }; + + match kind { + ContainerType::Map => Self::Map(MapHandler { + inner: handler.into(), + }), + ContainerType::List => Self::List(ListHandler { + inner: handler.into(), + }), + ContainerType::Tree => Self::Tree(TreeHandler { + inner: handler.into(), + }), + ContainerType::Text => Self::Text(TextHandler { + inner: handler.into(), + }), + ContainerType::MovableList => Self::MovableList(MovableListHandler { + inner: handler.into(), + }), + #[cfg(feature = "counter")] + ContainerType::Counter => Self::Counter(counter::CounterHandler { + inner: handler.into(), + }), + ContainerType::Unknown(_) => Self::Unknown(UnknownHandler { inner: handler }), + } + } + + #[allow(unused)] + pub(crate) fn new_unattached(kind: ContainerType) -> Self { + match kind { + ContainerType::Text => Self::Text(TextHandler::new_detached()), + ContainerType::Map => Self::Map(MapHandler::new_detached()), + ContainerType::List => Self::List(ListHandler::new_detached()), + ContainerType::Tree => Self::Tree(TreeHandler::new_detached()), + ContainerType::MovableList => Self::MovableList(MovableListHandler::new_detached()), + #[cfg(feature = "counter")] + ContainerType::Counter => Self::Counter(counter::CounterHandler::new_detached()), + ContainerType::Unknown(_) => unreachable!(), + } + } + + pub fn id(&self) -> ContainerID { + match self { + Self::Map(x) => x.id(), + Self::List(x) => x.id(), + Self::Text(x) => x.id(), + Self::Tree(x) => x.id(), + Self::MovableList(x) => x.id(), + #[cfg(feature = "counter")] + Self::Counter(x) => x.id(), + Self::Unknown(x) => x.id(), + } + } + + pub(crate) fn container_idx(&self) -> ContainerIdx { + match self { + Self::Map(x) => x.idx(), + Self::List(x) => x.idx(), + Self::Text(x) => x.idx(), + Self::Tree(x) => x.idx(), + Self::MovableList(x) => x.idx(), + #[cfg(feature = "counter")] + Self::Counter(x) => x.idx(), + Self::Unknown(x) => x.idx(), + } + } + + pub fn c_type(&self) -> ContainerType { + match self { + Self::Map(_) => ContainerType::Map, + Self::List(_) => ContainerType::List, + Self::Text(_) => ContainerType::Text, + Self::Tree(_) => ContainerType::Tree, + Self::MovableList(_) => ContainerType::MovableList, + #[cfg(feature = "counter")] + Self::Counter(_) => ContainerType::Counter, + Self::Unknown(x) => x.id().container_type(), + } + } + + fn get_deep_value(&self) -> LoroValue { + match self { + Self::Map(x) => x.get_deep_value(), + Self::List(x) => x.get_deep_value(), + Self::MovableList(x) => x.get_deep_value(), + Self::Text(x) => x.get_deep_value(), + Self::Tree(x) => x.get_deep_value(), + #[cfg(feature = "counter")] + Self::Counter(x) => x.get_deep_value(), + Self::Unknown(x) => x.get_deep_value(), + } + } + + pub(crate) fn apply_diff( + &self, + diff: Diff, + container_remap: &mut FxHashMap, + ) -> LoroResult<()> { + // In this method we will not clone the values of the containers if + // they are remapped. It's the caller's duty to do so + let on_container_remap = &mut |old_id, new_id| { + if old_id != new_id { + container_remap.insert(old_id, new_id); + } + }; + match self { + Self::Map(x) => { + let diff = match diff { + crate::event::Diff::Map(d) => d, + _ => { + return Err(LoroError::DecodeError( + "Invalid diff type for map container".into(), + )); + } + }; + for (key, value) in diff.updated.into_iter() { + match value.value { + Some(ValueOrHandler::Handler(h)) => { + Self::apply_map_container_diff_value( + x, + &key, + h.id(), + on_container_remap, + )?; + } + Some(ValueOrHandler::Value(LoroValue::Container(old_id))) => { + Self::apply_map_container_diff_value( + x, + &key, + old_id, + on_container_remap, + )?; + } + Some(ValueOrHandler::Value(v)) => { + x.insert_without_skipping(&key, v)?; + } + None => { + x.delete(&key)?; + } + } + } + } + Self::Text(x) => { + let delta = match diff { + crate::event::Diff::Text(d) => d, + _ => { + return Err(LoroError::DecodeError( + "Invalid diff type for text container".into(), + )); + } + }; + x.apply_delta(&TextDelta::from_text_diff(delta.iter()))?; + } + Self::List(x) => { + let delta = match diff { + crate::event::Diff::List(d) => d, + _ => { + return Err(LoroError::DecodeError( + "Invalid diff type for list container".into(), + )); + } + }; + x.apply_delta(delta, on_container_remap)?; + } + Self::MovableList(x) => { + let delta = match diff { + crate::event::Diff::List(d) => d, + _ => { + return Err(LoroError::DecodeError( + "Invalid diff type for movable list container".into(), + )); + } + }; + x.apply_delta(delta, container_remap)?; + } + Self::Tree(x) => { + fn remap_tree_id( + id: &mut TreeID, + container_remap: &FxHashMap, + ) { + let mut remapped = false; + let mut map_id = id.associated_meta_container(); + while let Some(rid) = container_remap.get(&map_id) { + remapped = true; + map_id = rid.clone(); + } + if remapped { + *id = TreeID::new( + *map_id.as_normal().unwrap().0, + *map_id.as_normal().unwrap().1, + ) + } + } + let tree_diff = match diff { + crate::event::Diff::Tree(d) => d, + _ => { + return Err(LoroError::DecodeError( + "Invalid diff type for tree container".into(), + )); + } + }; + for diff in tree_diff.diff { + let mut target = diff.target; + match diff.action { + TreeExternalDiff::Create { + mut parent, + index: _, + position, + } => { + if let TreeParentId::Node(p) = &mut parent { + remap_tree_id(p, container_remap) + } + remap_tree_id(&mut target, container_remap); + if !x.is_node_unexist(&target) && !x.is_node_deleted(&target)? { + // 1@0 is the parent of 2@1 + // ┌────┐ ┌───────────────┐ + // │xxxx│◀───│Move 2@1 to 0@0◀┐ + // └────┘ └───────────────┘│ + // ┌───────┐ │ ┌────────┐ + // │Del 1@0│◀─────────────────┴─│Meta 2@1│ ◀─── undo 2 ops redo 2 ops + // └───────┘ └────────┘ + // + // When we undo the delete operation, we should not create a new tree node and its child. + // However, the concurrent operation has moved the child to another parent. It's still alive. + // So when we redo the delete operation, we should check if the target is still alive. + // If it's alive, we should move it back instead of creating new one. + x.move_at_with_target_for_apply_diff(parent, position, target)?; + } else { + let new_target = x.__internal__next_tree_id(); + if x.create_at_with_target_for_apply_diff( + parent, position, new_target, + )? { + container_remap.insert( + target.associated_meta_container(), + new_target.associated_meta_container(), + ); + } + } + } + TreeExternalDiff::Move { + mut parent, + index: _, + position, + old_parent: _, + old_index: _, + } => { + if let TreeParentId::Node(p) = &mut parent { + remap_tree_id(p, container_remap) + } + remap_tree_id(&mut target, container_remap); + // determine if the target is deleted + if x.is_node_unexist(&target) || x.is_node_deleted(&target)? { + // create the target node, we should use the new target id + let new_target = x.__internal__next_tree_id(); + if x.create_at_with_target_for_apply_diff( + parent, position, new_target, + )? { + container_remap.insert( + target.associated_meta_container(), + new_target.associated_meta_container(), + ); + } + } else { + x.move_at_with_target_for_apply_diff(parent, position, target)?; + } + } + TreeExternalDiff::Delete { .. } => { + remap_tree_id(&mut target, container_remap); + if !x.is_node_deleted(&target)? { + x.delete(target)?; + } + } + } + } + } + #[cfg(feature = "counter")] + Self::Counter(x) => { + let delta = match diff { + crate::event::Diff::Counter(d) => d, + _ => { + return Err(LoroError::DecodeError( + "Invalid diff type for counter container".into(), + )); + } + }; + x.increment(delta)?; + } + Self::Unknown(_) => { + // do nothing + } + } + + Ok(()) + } + + pub fn clear(&self) -> LoroResult<()> { + match self { + Handler::Text(text_handler) => text_handler.clear(), + Handler::Map(map_handler) => map_handler.clear(), + Handler::List(list_handler) => list_handler.clear(), + Handler::MovableList(movable_list_handler) => movable_list_handler.clear(), + Handler::Tree(tree_handler) => tree_handler.clear(), + #[cfg(feature = "counter")] + Handler::Counter(counter_handler) => counter_handler.clear(), + Handler::Unknown(_unknown_handler) => Ok(()), + } + } +} + +#[derive(Clone, EnumAsInner, Debug)] +pub enum ValueOrHandler { + Value(LoroValue), + Handler(Handler), +} + +impl ValueOrHandler { + pub(crate) fn from_value(value: LoroValue, doc: &Arc) -> Self { + if let LoroValue::Container(c) = value { + ValueOrHandler::Handler(Handler::new_attached(c, LoroDoc::from_inner(doc.clone()))) + } else { + ValueOrHandler::Value(value) + } + } + + pub(crate) fn to_value(&self) -> LoroValue { + match self { + Self::Value(v) => v.clone(), + Self::Handler(h) => LoroValue::Container(h.id().clone()), + } + } + + pub(crate) fn to_deep_value(&self) -> LoroValue { + match self { + Self::Value(v) => v.clone(), + Self::Handler(h) => h.get_deep_value(), + } + } +} + +impl From for ValueOrHandler { + fn from(value: LoroValue) -> Self { + ValueOrHandler::Value(value) + } +} + +impl TextHandler { + /// Create a new container that is detached from the document. + /// + /// The edits on a detached container will not be persisted. + /// To attach the container to the document, please insert it into an attached container. + pub fn new_detached() -> Self { + Self { + inner: MaybeDetached::new_detached(RichtextState::default()), + } + } + + /// Get the version id of the richtext + /// + /// This can be used to detect whether the richtext is changed + pub fn version_id(&self) -> Option { + match &self.inner { + MaybeDetached::Detached(_) => None, + MaybeDetached::Attached(a) => { + Some(a.with_state(|state| state.as_richtext_state_mut().unwrap().get_version_id())) + } + } + } + + pub fn get_richtext_value(&self) -> LoroValue { + match &self.inner { + MaybeDetached::Detached(t) => { + let t = t.lock(); + t.value.get_richtext_value() + } + MaybeDetached::Attached(a) => { + a.with_state(|state| state.as_richtext_state_mut().unwrap().get_richtext_value()) + } + } + } + + pub fn is_empty(&self) -> bool { + match &self.inner { + MaybeDetached::Detached(t) => t.lock().value.is_empty(), + MaybeDetached::Attached(a) if a.has_decoded_state() => { + a.with_state(|state| state.as_richtext_state_mut().unwrap().is_empty()) + } + MaybeDetached::Attached(a) => a.get_value().as_string().unwrap().is_empty(), + } + } + + pub fn len_utf8(&self) -> usize { + match &self.inner { + MaybeDetached::Detached(t) => { + let t = t.lock(); + t.value.len_utf8() + } + MaybeDetached::Attached(a) => { + a.with_doc_state(|state| state.get_text_len(a.container_idx, PosType::Bytes)) + } + } + } + + pub fn len_utf16(&self) -> usize { + match &self.inner { + MaybeDetached::Detached(t) => { + let t = t.lock(); + t.value.len_utf16() + } + MaybeDetached::Attached(a) => { + a.with_doc_state(|state| state.get_text_len(a.container_idx, PosType::Utf16)) + } + } + } + + pub fn len_unicode(&self) -> usize { + match &self.inner { + MaybeDetached::Detached(t) => { + let t = t.lock(); + t.value.len_unicode() + } + MaybeDetached::Attached(a) => { + a.with_doc_state(|state| state.get_text_len(a.container_idx, PosType::Unicode)) + } + } + } + + /// if `wasm` feature is enabled, it is a UTF-16 length + /// otherwise, it is a Unicode length + pub fn len_event(&self) -> usize { + if cfg!(feature = "wasm") { + self.len_utf16() + } else { + self.len_unicode() + } + } + + fn len(&self, pos_type: PosType) -> usize { + match &self.inner { + MaybeDetached::Detached(t) => t.lock().value.len(pos_type), + MaybeDetached::Attached(a) => { + a.with_doc_state(|state| state.get_text_len(a.container_idx, pos_type)) + } + } + } + + fn validate_text_boundary(&self, pos: usize, pos_type: PosType) -> LoroResult<()> { + let err = match pos_type { + PosType::Bytes => Some(LoroError::UTF8InUnicodeCodePoint { pos }), + PosType::Utf16 => Some(LoroError::UTF16InUnicodeCodePoint { pos }), + PosType::Event if cfg!(feature = "wasm") => { + Some(LoroError::UTF16InUnicodeCodePoint { pos }) + } + _ => None, + }; + + let Some(err) = err else { + return Ok(()); + }; + + let len = self.len(pos_type); + if pos > len { + return Ok(()); + } + + if self.all_text_positions_are_boundaries(pos_type, len) { + return Ok(()); + } + + let Some(unicode_pos) = self.convert_pos(pos, pos_type, PosType::Unicode) else { + return Err(err); + }; + if self.convert_pos(unicode_pos, PosType::Unicode, pos_type) != Some(pos) { + return Err(err); + } + + Ok(()) + } + + fn all_text_positions_are_boundaries(&self, pos_type: PosType, len: usize) -> bool { + match pos_type { + PosType::Bytes => len == self.len_unicode(), + PosType::Utf16 => len == self.len_unicode(), + PosType::Event if cfg!(feature = "wasm") => len == self.len_unicode(), + _ => false, + } + } + + pub fn diagnose(&self) { + match &self.inner { + MaybeDetached::Detached(t) => { + let t = t.lock(); + t.value.diagnose(); + } + MaybeDetached::Attached(a) => { + a.with_state(|state| state.as_richtext_state_mut().unwrap().diagnose()); + } + } + } + + pub fn iter(&self, mut callback: impl FnMut(&str) -> bool) { + // Do not call user callbacks while holding the state lock; callbacks may re-enter Loro. + let spans: Vec = match &self.inner { + MaybeDetached::Detached(t) => { + let t = t.lock(); + t.value + .iter() + .map(|span| span.text.as_str().to_owned()) + .collect() + } + MaybeDetached::Attached(a) => a.with_state(|state| { + let mut spans = Vec::new(); + state.as_richtext_state_mut().unwrap().iter(|span| { + spans.push(span.to_owned()); + true + }); + spans + }), + }; + + for span in spans { + if !callback(span.as_str()) { + return; + } + } + } + + /// Get a character at `pos` in the coordinate system specified by `pos_type`. + pub fn char_at(&self, pos: usize, pos_type: PosType) -> LoroResult { + let len = self.len(pos_type); + if pos >= len { + return Err(LoroError::OutOfBound { + pos, + len, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + }); + } + if let Ok(c) = match &self.inner { + MaybeDetached::Detached(t) => { + let t = t.lock(); + let event_pos = match pos_type { + PosType::Event => pos, + _ => t.value.index_to_event_index(pos, pos_type), + }; + t.value.get_char_by_event_index(event_pos) + } + MaybeDetached::Attached(a) if a.has_decoded_state() || pos_type == PosType::Entity => a + .with_state(|state| { + let state = state.as_richtext_state_mut().unwrap(); + let event_pos = match pos_type { + PosType::Event => pos, + _ => state.index_to_event_index(pos, pos_type), + }; + state.get_char_by_event_index(event_pos) + }), + MaybeDetached::Attached(a) => { + return text_char_at(a.get_value().as_string().unwrap(), pos, pos_type); + } + } { + Ok(c) + } else { + Err(LoroError::OutOfBound { + pos, + len, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + }) + } + } + + /// `start_index` and `end_index` are Event Index: + /// + /// - if feature="wasm", pos is a UTF-16 index + /// - if feature!="wasm", pos is a Unicode index + /// + pub fn slice( + &self, + start_index: usize, + end_index: usize, + pos_type: PosType, + ) -> LoroResult { + self.slice_with_pos_type(start_index, end_index, pos_type) + } + + pub fn slice_utf16(&self, start_index: usize, end_index: usize) -> LoroResult { + self.slice(start_index, end_index, PosType::Utf16) + } + + fn slice_with_pos_type( + &self, + start_index: usize, + end_index: usize, + pos_type: PosType, + ) -> LoroResult { + if end_index < start_index { + return Err(LoroError::EndIndexLessThanStartIndex { + start: start_index, + end: end_index, + }); + } + if start_index == end_index { + return Ok(String::new()); + } + + let info = || format!("Position: {}:{}", file!(), line!()).into_boxed_str(); + match &self.inner { + MaybeDetached::Detached(t) => { + let t = t.lock(); + let len = t.value.len(pos_type); + if end_index > len { + return Err(LoroError::OutOfBound { + pos: end_index, + len, + info: info(), + }); + } + let (start, end) = match pos_type { + PosType::Event => (start_index, end_index), + _ => ( + t.value.index_to_event_index(start_index, pos_type), + t.value.index_to_event_index(end_index, pos_type), + ), + }; + t.value.get_text_slice_by_event_index(start, end - start) + } + MaybeDetached::Attached(a) if a.has_decoded_state() || pos_type == PosType::Entity => a + .with_state(|state| { + let state = state.as_richtext_state_mut().unwrap(); + let len = state.len(pos_type); + if end_index > len { + return Err(LoroError::OutOfBound { + pos: end_index, + len, + info: info(), + }); + } + let (start, end) = match pos_type { + PosType::Event => (start_index, end_index), + _ => ( + state.index_to_event_index(start_index, pos_type), + state.index_to_event_index(end_index, pos_type), + ), + }; + state.get_text_slice_by_event_index(start, end - start) + }), + MaybeDetached::Attached(a) => text_slice( + a.get_value().as_string().unwrap(), + start_index, + end_index, + pos_type, + ) + .map_err(|err| match err { + LoroError::OutOfBound { pos, len, .. } => LoroError::OutOfBound { + pos, + len, + info: info(), + }, + err => err, + }), + } + } + + pub fn slice_delta( + &self, + start_index: usize, + end_index: usize, + pos_type: PosType, + ) -> LoroResult> { + if end_index < start_index { + return Err(LoroError::EndIndexLessThanStartIndex { + start: start_index, + end: end_index, + }); + } + if start_index == end_index { + return Ok(Vec::new()); + } + + let len = self.len(pos_type); + if end_index > len { + return Err(LoroError::OutOfBound { + pos: end_index, + len, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + }); + } + self.validate_text_boundary(start_index, pos_type)?; + self.validate_text_boundary(end_index, pos_type)?; + + match &self.inner { + MaybeDetached::Detached(t) => { + let t = t.lock(); + let ans = t.value.slice_delta(start_index, end_index, pos_type)?; + Ok(ans + .into_iter() + .map(|(s, a)| TextDelta::Insert { + insert: s, + attributes: a.to_option_map_without_null_value(), + }) + .collect()) + } + MaybeDetached::Attached(a) => a.with_state(|state| { + let ans = state.as_richtext_state_mut().unwrap().slice_delta( + start_index, + end_index, + pos_type, + )?; + Ok(ans + .into_iter() + .map(|(s, a)| TextDelta::Insert { + insert: s, + attributes: a.to_option_map_without_null_value(), + }) + .collect()) + }), + } + } + + /// `pos` is a Event Index: + /// + /// - if feature="wasm", pos is a UTF-16 index + /// - if feature!="wasm", pos is a Unicode index + /// + /// This method requires auto_commit to be enabled. + pub fn splice(&self, pos: usize, len: usize, s: &str, pos_type: PosType) -> LoroResult { + let end = checked_range_end( + pos, + len, + self.len(pos_type), + || format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + )?; + let x = self.slice(pos, end, pos_type)?; + self.delete(pos, len, pos_type)?; + self.insert(pos, s, pos_type)?; + Ok(x) + } + + pub fn splice_utf8(&self, pos: usize, len: usize, s: &str) -> LoroResult<()> { + // let x = self.slice(pos, pos + len)?; + self.delete_utf8(pos, len)?; + self.insert_utf8(pos, s)?; + Ok(()) + } + + pub fn splice_utf16(&self, pos: usize, len: usize, s: &str) -> LoroResult<()> { + self.delete(pos, len, PosType::Utf16)?; + self.insert(pos, s, PosType::Utf16)?; + Ok(()) + } + + /// Insert text at `pos` using the given `pos_type` coordinate system. + /// + /// This method requires auto_commit to be enabled. + pub fn insert(&self, pos: usize, s: &str, pos_type: PosType) -> LoroResult<()> { + match &self.inner { + MaybeDetached::Detached(t) => { + let len = self.len(pos_type); + if pos > len { + return Err(LoroError::OutOfBound { + pos, + len, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + }); + } + self.validate_text_boundary(pos, pos_type)?; + + let mut t = t.lock(); + let (index, _) = t + .value + .get_entity_index_for_text_insert(pos, pos_type) + .unwrap(); + t.value.insert_at_entity_index( + index, + BytesSlice::from_bytes(s.as_bytes()), + IdFull::NONE_ID, + ); + Ok(()) + } + MaybeDetached::Attached(a) => { + if s.is_empty() { + let len = self.len(pos_type); + if pos > len { + return Err(LoroError::OutOfBound { + pos, + len, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + }); + } + self.validate_text_boundary(pos, pos_type)?; + return Ok(()); + } + + a.with_txn(|txn| self.insert_with_txn(txn, pos, s, pos_type)) + } + } + } + + pub fn insert_utf8(&self, pos: usize, s: &str) -> LoroResult<()> { + self.insert(pos, s, PosType::Bytes) + } + + pub fn insert_utf16(&self, pos: usize, s: &str) -> LoroResult<()> { + self.insert(pos, s, PosType::Utf16) + } + + pub fn insert_unicode(&self, pos: usize, s: &str) -> LoroResult<()> { + self.insert(pos, s, PosType::Unicode) + } + + /// Insert text within an existing transaction using the provided `pos_type`. + pub fn insert_with_txn( + &self, + txn: &mut Transaction, + pos: usize, + s: &str, + pos_type: PosType, + ) -> LoroResult<()> { + self.insert_with_txn_and_attr(txn, pos, s, None, pos_type)?; + Ok(()) + } + + pub fn insert_with_txn_utf8( + &self, + txn: &mut Transaction, + pos: usize, + s: &str, + ) -> LoroResult<()> { + self.insert_with_txn(txn, pos, s, PosType::Bytes) + } + + /// Delete a span using the coordinate system described by `pos_type`. + /// + /// This method requires auto_commit to be enabled. + pub fn delete(&self, pos: usize, len: usize, pos_type: PosType) -> LoroResult<()> { + if len == 0 { + return Ok(()); + } + + let text_len = self.len(pos_type); + let end = checked_range_end( + pos, + len, + text_len, + || format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + )?; + self.validate_text_boundary(pos, pos_type)?; + self.validate_text_boundary(end, pos_type)?; + + match &self.inner { + MaybeDetached::Detached(t) => { + let mut t = t.lock(); + let ranges = t.value.get_text_entity_ranges(pos, len, pos_type)?; + for range in ranges.iter().rev() { + t.value + .drain_by_entity_index(range.entity_start, range.entity_len(), None); + } + Ok(()) + } + MaybeDetached::Attached(a) => { + a.with_txn(|txn| self.delete_with_txn(txn, pos, len, pos_type)) + } + } + } + + pub fn delete_utf8(&self, pos: usize, len: usize) -> LoroResult<()> { + self.delete(pos, len, PosType::Bytes) + } + + pub fn delete_utf16(&self, pos: usize, len: usize) -> LoroResult<()> { + self.delete(pos, len, PosType::Utf16) + } + + pub fn delete_unicode(&self, pos: usize, len: usize) -> LoroResult<()> { + self.delete(pos, len, PosType::Unicode) + } + + /// If attr is specified, it will be used as the attribute of the inserted text. + /// It will override the existing attribute of the text. + fn insert_with_txn_and_attr( + &self, + txn: &mut Transaction, + pos: usize, + s: &str, + attr: Option<&FxHashMap>, + pos_type: PosType, + ) -> Result, LoroError> { + if s.is_empty() { + return Ok(Vec::new()); + } + + // Fast path: plain-text insert into a style-free document (non-wasm). + // With no style anchors, entity_index == unicode pos and the event index + // equals the unicode index, so bounds + no-styles are checked in a single + // state access and the entire read phase (cursor location + two + // visit_previous_caches walks + styles lookup) is skipped; apply_local_op + // then locates the cursor exactly once. + #[cfg(not(feature = "wasm"))] + if attr.is_none() && pos_type == PosType::Unicode { + let inner = self.inner.try_attached_state()?; + let fast = inner.with_state(|state| { + let rt = state.as_richtext_state_mut().unwrap(); + if rt.has_styles() { + return Ok(false); + } + let len = rt.len_unicode(); + if pos > len { + return Err(LoroError::OutOfBound { + pos, + len, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + }); + } + Ok(true) + })?; + if fast { + let unicode_len = s.chars().count(); + txn.apply_local_op( + inner.container_idx, + crate::op::RawOpContent::List( + crate::container::list::list_op::ListOp::Insert { + slice: ListSlice::RawStr { + str: Cow::Borrowed(s), + unicode_len, + }, + // entity_index == unicode pos (no style anchors) + pos, + }, + ), + EventHint::InsertText { + // event index == unicode index (non-wasm) + pos: pos as u32, + styles: StyleMeta::empty(), + unicode_len: unicode_len as u32, + event_len: unicode_len as u32, + }, + &inner.doc, + )?; + return Ok(Vec::new()); + } + } + + match pos_type { + PosType::Event => { + if pos > self.len_event() { + return Err(LoroError::OutOfBound { + pos, + len: self.len_event(), + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + }); + } + } + PosType::Bytes => { + if pos > self.len_utf8() { + return Err(LoroError::OutOfBound { + pos, + len: self.len_utf8(), + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + }); + } + } + PosType::Unicode => { + if pos > self.len_unicode() { + return Err(LoroError::OutOfBound { + pos, + len: self.len_unicode(), + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + }); + } + } + PosType::Entity => {} + PosType::Utf16 => { + if pos > self.len_utf16() { + return Err(LoroError::OutOfBound { + pos, + len: self.len_utf16(), + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + }); + } + } + } + self.validate_text_boundary(pos, pos_type)?; + + let inner = self.inner.try_attached_state()?; + let (entity_index, event_index, styles) = inner.with_state(|state| { + let richtext_state = state.as_richtext_state_mut().unwrap(); + let ret = richtext_state.get_entity_index_for_text_insert(pos, pos_type); + let (entity_index, cursor) = match ret { + Err(_) => match pos_type { + PosType::Bytes => { + return ( + Err(LoroError::UTF8InUnicodeCodePoint { pos }), + 0, + StyleMeta::empty(), + ); + } + PosType::Utf16 | PosType::Event => { + return ( + Err(LoroError::UTF16InUnicodeCodePoint { pos }), + 0, + StyleMeta::empty(), + ); + } + _ => unreachable!(), + }, + Ok(x) => x, + }; + let event_index = if let Some(cursor) = cursor { + if pos_type == PosType::Event { + debug_assert_eq!( + richtext_state.get_event_index_by_cursor(cursor), + pos, + "pos={} cursor={:?} state={:#?}", + pos, + cursor, + &richtext_state + ); + pos + } else { + richtext_state.get_event_index_by_cursor(cursor) + } + } else { + assert_eq!(entity_index, 0); + 0 + }; + let styles = richtext_state.get_styles_at_entity_index(entity_index); + (Ok(entity_index), event_index, styles) + }); + + let entity_index = match entity_index { + Err(x) => return Err(x), + _ => entity_index.unwrap(), + }; + + let mut override_styles = Vec::new(); + if let Some(attr) = attr { + // current styles + let map: FxHashMap<_, _> = styles.iter().map(|x| (x.0.clone(), x.1.data)).collect(); + for (key, style) in map.iter() { + match attr.get(key.deref()) { + Some(v) if v == style => {} + new_style_value => { + // need to override + let new_style_value = new_style_value.cloned().unwrap_or(LoroValue::Null); + override_styles.push((key.clone(), new_style_value)); + } + } + } + + for (key, style) in attr.iter() { + let key = key.as_str().into(); + if !map.contains_key(&key) { + override_styles.push((key, style.clone())); + } + } + } + + let unicode_len = s.chars().count(); + let event_len = if cfg!(feature = "wasm") { + count_utf16_len(s.as_bytes()) + } else { + unicode_len + }; + + txn.apply_local_op( + inner.container_idx, + crate::op::RawOpContent::List(crate::container::list::list_op::ListOp::Insert { + slice: ListSlice::RawStr { + str: Cow::Borrowed(s), + unicode_len, + }, + pos: entity_index, + }), + EventHint::InsertText { + pos: event_index as u32, + styles, + unicode_len: unicode_len as u32, + event_len: event_len as u32, + }, + &inner.doc, + )?; + + Ok(override_styles) + } + + /// Delete text within a transaction using the specified `pos_type`. + pub fn delete_with_txn( + &self, + txn: &mut Transaction, + pos: usize, + len: usize, + pos_type: PosType, + ) -> LoroResult<()> { + self.delete_with_txn_inline(txn, pos, len, pos_type) + } + + fn delete_with_txn_inline( + &self, + txn: &mut Transaction, + pos: usize, + len: usize, + pos_type: PosType, + ) -> LoroResult<()> { + if len == 0 { + return Ok(()); + } + + let text_len = self.len(pos_type); + let end = checked_range_end( + pos, + len, + text_len, + || format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + ) + .inspect_err(|_| error!("pos={} len={} len={}", pos, len, text_len))?; + self.validate_text_boundary(pos, pos_type)?; + self.validate_text_boundary(end, pos_type)?; + + let inner = self.inner.try_attached_state()?; + let s = tracing::span!(tracing::Level::INFO, "delete", "pos={} len={}", pos, len); + let _e = s.enter(); + let mut event_pos = 0; + let mut event_len = 0; + let ranges = inner.with_state(|state| { + let richtext_state = state.as_richtext_state_mut().unwrap(); + // Fast path: with no style anchors (non-wasm), the event index equals + // the unicode index, so the two index_to_event_index walks collapse to + // identity. + let fast = cfg!(not(feature = "wasm")) + && pos_type == PosType::Unicode + && !richtext_state.has_styles(); + if fast { + event_pos = pos; + event_len = len; + } else { + event_pos = richtext_state.index_to_event_index(pos, pos_type); + let event_end = richtext_state.index_to_event_index(end, pos_type); + event_len = event_end - event_pos; + } + + richtext_state.get_text_entity_ranges_in_event_index_range(event_pos, event_len) + })?; + + //debug_assert_eq!(ranges.iter().map(|x| x.event_len).sum::(), len); + let pos = event_pos as isize; + let len = event_len as isize; + let mut event_end = pos + len; + for range in ranges.iter().rev() { + let event_start = event_end - range.event_len as isize; + txn.apply_local_op( + inner.container_idx, + crate::op::RawOpContent::List(ListOp::Delete(DeleteSpanWithId::new( + range.id_start, + range.entity_start as isize, + range.entity_len() as isize, + ))), + EventHint::DeleteText { + span: DeleteSpan { + pos: event_start, + signed_len: range.event_len as isize, + }, + unicode_len: range.entity_len(), + }, + &inner.doc, + )?; + event_end = event_start; + } + + Ok(()) + } + + /// `start` and `end` are interpreted using `pos_type`. + /// + /// This method requires auto_commit to be enabled. + pub fn mark( + &self, + start: usize, + end: usize, + key: impl Into, + value: LoroValue, + pos_type: PosType, + ) -> LoroResult<()> { + match &self.inner { + MaybeDetached::Detached(t) => { + let mut g = t.lock(); + self.mark_for_detached(&mut g.value, key, &value, start, end, pos_type) + } + MaybeDetached::Attached(a) => { + a.with_txn(|txn| self.mark_with_txn(txn, start, end, key, value, pos_type)) + } + } + } + + fn mark_for_detached( + &self, + state: &mut RichtextState, + key: impl Into, + value: &LoroValue, + start: usize, + end: usize, + pos_type: PosType, + ) -> Result<(), LoroError> { + let key: InternalString = key.into(); + let is_delete = matches!(value, &LoroValue::Null); + if start >= end { + return Err(loro_common::LoroError::ArgErr( + "Start must be less than end".to_string().into_boxed_str(), + )); + } + ensure_no_regular_container_value(value)?; + + let len = state.len(pos_type); + if end > len { + return Err(LoroError::OutOfBound { + pos: end, + len, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + }); + } + let (entity_range, styles) = + state.get_entity_range_and_text_styles_at_range(start..end, pos_type); + if let Some(styles) = styles { + if styles.has_key_value(&key, value) { + return Ok(()); + } + } + + let has_target_style = + state.range_has_style_key(entity_range.clone(), &StyleKey::Key(key.clone())); + let missing_style_key = is_delete && !has_target_style; + + if missing_style_key { + return Ok(()); + } + + let style_op = Arc::new(StyleOp { + lamport: 0, + peer: 0, + cnt: 0, + key, + value: value.clone(), + // TODO: describe this behavior in the document + info: if is_delete { + TextStyleInfoFlag::BOLD.to_delete() + } else { + TextStyleInfoFlag::BOLD + }, + }); + state.mark_with_entity_index(entity_range, style_op); + Ok(()) + } + + /// `start` and `end` are interpreted using `pos_type`. + pub fn unmark( + &self, + start: usize, + end: usize, + key: impl Into, + pos_type: PosType, + ) -> LoroResult<()> { + match &self.inner { + MaybeDetached::Detached(t) => self.mark_for_detached( + &mut t.lock().value, + key, + &LoroValue::Null, + start, + end, + pos_type, + ), + MaybeDetached::Attached(a) => a.with_txn(|txn| { + self.mark_with_txn(txn, start, end, key, LoroValue::Null, pos_type) + }), + } + } + + /// `start` and `end` are interpreted using `pos_type`. + pub fn mark_with_txn( + &self, + txn: &mut Transaction, + start: usize, + end: usize, + key: impl Into, + value: LoroValue, + pos_type: PosType, + ) -> LoroResult<()> { + if start >= end { + return Err(loro_common::LoroError::ArgErr( + "Start must be less than end".to_string().into_boxed_str(), + )); + } + ensure_no_regular_container_value(&value)?; + + let inner = self.inner.try_attached_state()?; + let key: InternalString = key.into(); + let is_delete = matches!(&value, &LoroValue::Null); + + let mut doc_state = inner.doc.state.lock(); + let len = doc_state.with_state_mut(inner.container_idx, |state| { + state.as_richtext_state_mut().unwrap().len(pos_type) + }); + + if end > len { + return Err(LoroError::OutOfBound { + pos: end, + len, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + }); + } + + let (entity_range, skip, missing_style_key, event_start, event_end) = doc_state + .with_state_mut(inner.container_idx, |state| { + let state = state.as_richtext_state_mut().unwrap(); + let event_start = state.index_to_event_index(start, pos_type); + let event_end = state.index_to_event_index(end, pos_type); + let (entity_range, styles) = + state.get_entity_range_and_styles_at_range(start..end, pos_type); + + let skip = styles + .as_ref() + .map(|styles| styles.has_key_value(&key, &value)) + .unwrap_or(false); + let has_target_style = state.has_style_key_in_entity_range( + entity_range.clone(), + &StyleKey::Key(key.clone()), + ); + let missing_style_key = is_delete && !has_target_style; + + ( + entity_range, + skip, + missing_style_key, + event_start, + event_end, + ) + }); + + if skip || missing_style_key { + return Ok(()); + } + + let entity_start = entity_range.start; + let entity_end = entity_range.end; + let style_config = doc_state.config.text_style_config.read(); + let flag = if is_delete { + style_config + .get_style_flag_for_unmark(&key) + .ok_or_else(|| LoroError::StyleConfigMissing(key.clone()))? + } else { + style_config + .get_style_flag(&key) + .ok_or_else(|| LoroError::StyleConfigMissing(key.clone()))? + }; + + drop(style_config); + drop(doc_state); + txn.apply_local_op( + inner.container_idx, + crate::op::RawOpContent::List(ListOp::StyleStart { + start: entity_start as u32, + end: entity_end as u32, + key: key.clone(), + value: value.clone(), + info: flag, + }), + EventHint::Mark { + start: event_start as u32, + end: event_end as u32, + style: crate::container::richtext::Style { key, data: value }, + }, + &inner.doc, + )?; + + txn.apply_local_op( + inner.container_idx, + crate::op::RawOpContent::List(ListOp::StyleEnd), + EventHint::MarkEnd, + &inner.doc, + )?; + + Ok(()) + } + + pub fn check(&self) { + match &self.inner { + MaybeDetached::Detached(t) => { + let t = t.lock(); + t.value.check_consistency_between_content_and_style_ranges(); + } + MaybeDetached::Attached(a) => a.with_state(|state| { + state + .as_richtext_state_mut() + .unwrap() + .check_consistency_between_content_and_style_ranges(); + }), + } + } + + pub fn apply_delta(&self, delta: &[TextDelta]) -> LoroResult<()> { + match &self.inner { + MaybeDetached::Detached(t) => { + let _t = t.lock(); + // TODO: implement + Err(LoroError::NotImplemented( + "`apply_delta` on a detached text container", + )) + } + MaybeDetached::Attached(a) => a.with_txn(|txn| self.apply_delta_with_txn(txn, delta)), + } + } + + pub fn apply_delta_with_txn( + &self, + txn: &mut Transaction, + delta: &[TextDelta], + ) -> LoroResult<()> { + let mut index = 0; + struct PendingMark { + start: usize, + end: usize, + attributes: FxHashMap, + } + let mut marks: Vec = Vec::new(); + for d in delta { + match d { + TextDelta::Insert { insert, attributes } => { + let insert_len = event_len(insert.as_str()); + if insert_len == 0 { + continue; + } + + let mut empty_attr = None; + let attr_ref = attributes.as_ref().unwrap_or_else(|| { + empty_attr = Some(FxHashMap::default()); + empty_attr.as_ref().unwrap() + }); + + let end = checked_delta_index_end(index, insert_len, self.len_event())?; + let override_styles = self.insert_with_txn_and_attr( + txn, + index, + insert.as_str(), + Some(attr_ref), + PosType::Event, + )?; + + let mut pending_mark = PendingMark { + start: index, + end, + attributes: FxHashMap::default(), + }; + for (key, value) in override_styles { + pending_mark.attributes.insert(key, value); + } + marks.push(pending_mark); + index = end; + } + TextDelta::Delete { delete } => { + self.delete_with_txn(txn, index, *delete, PosType::Event)?; + } + TextDelta::Retain { attributes, retain } => { + let end = checked_delta_index_end(index, *retain, self.len_event())?; + match attributes { + Some(attr) if !attr.is_empty() => { + let mut pending_mark = PendingMark { + start: index, + end, + attributes: FxHashMap::default(), + }; + for (key, value) in attr { + pending_mark + .attributes + .insert(key.deref().into(), value.clone()); + } + marks.push(pending_mark); + } + _ => {} + } + index = end; + } + } + } + + let mut len = match &self.inner { + MaybeDetached::Detached(_) => self.len_event(), + MaybeDetached::Attached(a) => { + a.with_state(|state| state.as_richtext_state_mut().unwrap().len(PosType::Event)) + } + }; + for pending_mark in marks { + if pending_mark.start >= len { + self.insert_with_txn( + txn, + len, + &"\n".repeat(pending_mark.start - len + 1), + PosType::Event, + )?; + len = pending_mark.start; + } + + for (key, value) in pending_mark.attributes { + self.mark_with_txn( + txn, + pending_mark.start, + pending_mark.end, + key.deref(), + value, + PosType::Event, + )?; + } + } + + Ok(()) + } + + pub fn update(&self, text: &str, options: UpdateOptions) -> Result<(), UpdateTimeoutError> { + let old_str = self.to_string(); + let new = text.chars().map(|x| x as u32).collect::>(); + let old = old_str.chars().map(|x| x as u32).collect::>(); + diff( + &mut OperateProxy::new(text_update::DiffHook::new(self, &new)), + options, + &old, + &new, + )?; + Ok(()) + } + + pub fn update_by_line( + &self, + text: &str, + options: UpdateOptions, + ) -> Result<(), UpdateTimeoutError> { + let hook = text_update::DiffHookForLine::new(self, text); + let old_lines = hook.get_old_arr().to_vec(); + let new_lines = hook.get_new_arr().to_vec(); + diff( + &mut OperateProxy::new(hook), + options, + &old_lines, + &new_lines, + ) + } + + #[allow(clippy::inherent_to_string)] + pub fn to_string(&self) -> String { + match &self.inner { + MaybeDetached::Detached(t) => t.lock().value.to_string(), + MaybeDetached::Attached(a) => a.get_value().into_string().unwrap().unwrap(), + } + } + + pub fn get_cursor(&self, event_index: usize, side: Side) -> Option { + self.get_cursor_internal(event_index, side, true) + } + + /// Get the stable position representation for the target pos + pub(crate) fn get_cursor_internal( + &self, + index: usize, + side: Side, + get_by_event_index: bool, + ) -> Option { + match &self.inner { + MaybeDetached::Detached(_) => None, + MaybeDetached::Attached(a) => { + let (id, len, origin_pos) = a.with_state(|s| { + let s = s.as_richtext_state_mut().unwrap(); + ( + s.get_stable_position(index, get_by_event_index), + if get_by_event_index { + s.len_event() + } else { + s.len_unicode() + }, + if get_by_event_index { + s.event_index_to_unicode_index(index) + } else { + index + }, + ) + }); + + if len == 0 { + return Some(Cursor { + id: None, + container: self.id(), + side: if side == Side::Middle { + Side::Left + } else { + side + }, + origin_pos: 0, + }); + } + + if len <= index { + return Some(Cursor { + id: None, + container: self.id(), + side: Side::Right, + origin_pos: len, + }); + } + + let id = id?; + Some(Cursor { + id: Some(id), + container: self.id(), + side, + origin_pos, + }) + } + } + } + + pub(crate) fn convert_entity_index_to_event_index(&self, entity_index: usize) -> usize { + match &self.inner { + MaybeDetached::Detached(s) => s.lock().value.entity_index_to_event_index(entity_index), + MaybeDetached::Attached(a) => { + let mut pos = 0; + a.with_state(|s| { + let s = s.as_richtext_state_mut().unwrap(); + pos = s.entity_index_to_event_index(entity_index); + }); + pos + } + } + } + + pub fn get_delta(&self) -> Vec { + match &self.inner { + MaybeDetached::Detached(s) => { + let mut delta = Vec::new(); + for span in s.lock().value.iter() { + if span.text.as_str().is_empty() { + continue; + } + + let next_attr = span.attributes.to_option_map(); + match delta.last_mut() { + Some(TextDelta::Insert { insert, attributes }) + if &next_attr == attributes => + { + insert.push_str(span.text.as_str()); + continue; + } + _ => {} + } + + delta.push(TextDelta::Insert { + insert: span.text.as_str().to_string(), + attributes: next_attr, + }) + } + delta + } + MaybeDetached::Attached(_a) => self + .with_state(|state| { + let state = state.as_richtext_state_mut().unwrap(); + Ok(state.get_delta()) + }) + .unwrap(), + } + } + + pub fn is_deleted(&self) -> bool { + match &self.inner { + MaybeDetached::Detached(_) => false, + MaybeDetached::Attached(a) => a.is_deleted(), + } + } + + pub fn push_str(&self, s: &str) -> LoroResult<()> { + self.insert_utf8(self.len_utf8(), s) + } + + pub fn clear(&self) -> LoroResult<()> { + match &self.inner { + MaybeDetached::Detached(mutex) => { + let mut t = mutex.lock(); + let len = t.value.len_unicode(); + let ranges = t.value.get_text_entity_ranges(0, len, PosType::Unicode)?; + for range in ranges.iter().rev() { + t.value + .drain_by_entity_index(range.entity_start, range.entity_len(), None); + } + Ok(()) + } + MaybeDetached::Attached(a) => a.with_txn(|txn| { + let len = a.with_state(|s| s.as_richtext_state_mut().unwrap().len_unicode()); + self.delete_with_txn_inline(txn, 0, len, PosType::Unicode) + }), + } + } + + /// Convert a position `index` from one coordinate system to another. + /// + /// Supported `PosType` conversions: `Event`, `Unicode`, `Utf16`, and `Bytes`. + /// Returns `None` if the index is out of bounds or the conversion is unsupported. + pub fn convert_pos(&self, index: usize, from: PosType, to: PosType) -> Option { + if from == to { + return Some(index); + } + + if matches!(from, PosType::Entity) || matches!(to, PosType::Entity) { + return None; + } + + // Normalize to event + unicode indices for the given position. + let (event_index, unicode_index) = match &self.inner { + MaybeDetached::Detached(t) => { + let t = t.lock(); + if index > t.value.len(from) { + return None; + } + let event_index = if from == PosType::Event { + index + } else { + t.value.index_to_event_index(index, from) + }; + let unicode_index = if from == PosType::Unicode { + index + } else { + t.value.event_index_to_unicode_index(event_index) + }; + (event_index, unicode_index) + } + MaybeDetached::Attached(a) if a.has_decoded_state() => { + let res: Option<(usize, usize)> = a.with_state(|state| { + let state = state.as_richtext_state_mut().unwrap(); + if index > state.len(from) { + return None; + } + + let event_index = if from == PosType::Event { + index + } else { + state.index_to_event_index(index, from) + }; + let unicode_index = if from == PosType::Unicode { + index + } else { + state.event_index_to_unicode_index(event_index) + }; + Some((event_index, unicode_index)) + }); + + res? + } + MaybeDetached::Attached(a) => { + let value = a.get_value(); + let s = value.as_string().unwrap(); + let unicode_index = text_pos_to_unicode(s, index, from)?; + let event_index = unicode_to_text_pos(s, unicode_index, PosType::Event)?; + (event_index, unicode_index) + } + }; + + let result = match to { + PosType::Unicode => Some(unicode_index), + PosType::Event => Some(event_index), + PosType::Bytes | PosType::Utf16 => { + // Map the event-index position onto the target coordinate via the + // rope's prefix caches. This is O(log n); materializing the prefix + // string would be O(n) and makes repeated edits O(n^2). + match &self.inner { + MaybeDetached::Detached(t) => { + let t = t.lock(); + if event_index > t.value.len_event() { + return None; + } + Some(t.value.event_index_to_index(event_index, to)) + } + MaybeDetached::Attached(a) if a.has_decoded_state() => a.with_state(|state| { + let state = state.as_richtext_state_mut().unwrap(); + if event_index > state.len_event() { + return None; + } + Some(state.event_index_to_index(event_index, to)) + }), + MaybeDetached::Attached(a) => { + let value = a.get_value(); + let s = value.as_string().unwrap(); + unicode_to_text_pos(s, unicode_index, to) + } + } + } + PosType::Entity => None, + }; + result + } +} + +fn event_len(s: &str) -> usize { + if cfg!(feature = "wasm") { + count_utf16_len(s.as_bytes()) + } else { + s.chars().count() + } +} + +fn text_len(s: &str, pos_type: PosType) -> Option { + Some(match pos_type { + PosType::Bytes => s.len(), + PosType::Unicode => s.chars().count(), + PosType::Utf16 => count_utf16_len(s.as_bytes()), + PosType::Event => event_len(s), + PosType::Entity => return None, + }) +} + +fn text_pos_to_unicode(s: &str, index: usize, pos_type: PosType) -> Option { + match pos_type { + PosType::Unicode => (index <= s.chars().count()).then_some(index), + PosType::Bytes => { + if index > s.len() { + None + } else { + Some( + s.char_indices() + .take_while(|(pos, c)| *pos + c.len_utf8() <= index) + .count(), + ) + } + } + PosType::Utf16 => utf16_to_unicode_pos(s, index), + PosType::Event if cfg!(feature = "wasm") => utf16_to_unicode_pos(s, index), + PosType::Event => (index <= s.chars().count()).then_some(index), + PosType::Entity => None, + } +} + +fn unicode_to_text_pos(s: &str, index: usize, pos_type: PosType) -> Option { + match pos_type { + PosType::Unicode => (index <= s.chars().count()).then_some(index), + PosType::Bytes => unicode_to_byte_pos(s, index), + PosType::Utf16 => unicode_to_utf16_pos(s, index), + PosType::Event if cfg!(feature = "wasm") => unicode_to_utf16_pos(s, index), + PosType::Event => (index <= s.chars().count()).then_some(index), + PosType::Entity => None, + } +} + +fn unicode_to_byte_pos(s: &str, index: usize) -> Option { + if index == 0 { + return Some(0); + } + + let mut unicode_pos = 0; + for (byte_pos, _) in s.char_indices() { + if unicode_pos == index { + return Some(byte_pos); + } + unicode_pos += 1; + } + + (unicode_pos == index).then_some(s.len()) +} + +fn unicode_to_utf16_pos(s: &str, index: usize) -> Option { + let mut unicode_pos = 0; + let mut utf16_pos = 0; + if index == 0 { + return Some(0); + } + + for c in s.chars() { + unicode_pos += 1; + utf16_pos += c.len_utf16(); + if unicode_pos == index { + return Some(utf16_pos); + } + } + + (unicode_pos == index).then_some(utf16_pos) +} + +fn utf16_to_unicode_pos(s: &str, index: usize) -> Option { + let mut unicode_pos = 0; + let mut utf16_pos = 0; + if index == 0 { + return Some(0); + } + + for c in s.chars() { + let next_utf16_pos = utf16_pos + c.len_utf16(); + if index < next_utf16_pos { + return Some(unicode_pos); + } + if index == next_utf16_pos { + return Some(unicode_pos + 1); + } + utf16_pos = next_utf16_pos; + unicode_pos += 1; + } + + (index == utf16_pos).then_some(unicode_pos) +} + +fn text_boundary_error(pos: usize, pos_type: PosType) -> LoroError { + match pos_type { + PosType::Bytes => LoroError::UTF8InUnicodeCodePoint { pos }, + PosType::Utf16 => LoroError::UTF16InUnicodeCodePoint { pos }, + PosType::Event if cfg!(feature = "wasm") => LoroError::UTF16InUnicodeCodePoint { pos }, + _ => LoroError::OutOfBound { + pos, + len: 0, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + }, + } +} + +fn text_char_at(s: &str, pos: usize, pos_type: PosType) -> LoroResult { + let len = text_len(s, pos_type).unwrap_or(0); + if pos >= len { + return Err(LoroError::OutOfBound { + pos, + len, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + }); + } + + let unicode_pos = + text_pos_to_unicode(s, pos, pos_type).ok_or_else(|| text_boundary_error(pos, pos_type))?; + s.chars().nth(unicode_pos).ok_or(LoroError::OutOfBound { + pos, + len, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + }) +} + +fn text_slice(s: &str, start: usize, end: usize, pos_type: PosType) -> LoroResult { + if end < start { + return Err(LoroError::EndIndexLessThanStartIndex { start, end }); + } + if start == end { + return Ok(String::new()); + } + + let len = text_len(s, pos_type).unwrap_or(0); + if end > len { + return Err(LoroError::OutOfBound { + pos: end, + len, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + }); + } + + let start = text_pos_to_unicode(s, start, pos_type) + .ok_or_else(|| text_boundary_error(start, pos_type))?; + let end = + text_pos_to_unicode(s, end, pos_type).ok_or_else(|| text_boundary_error(end, pos_type))?; + let start = unicode_to_byte_pos(s, start).expect("unicode index must map to a byte boundary"); + let end = unicode_to_byte_pos(s, end).expect("unicode index must map to a byte boundary"); + Ok(s[start..end].to_string()) +} + +impl ListHandler { + /// Create a new container that is detached from the document. + /// The edits on a detached container will not be persisted. + /// To attach the container to the document, please insert it into an attached container. + pub fn new_detached() -> Self { + Self { + inner: MaybeDetached::new_detached(Vec::new()), + } + } + + pub fn insert(&self, pos: usize, v: impl Into) -> LoroResult<()> { + match &self.inner { + MaybeDetached::Detached(l) => { + let mut list = l.lock(); + let len = list.value.len(); + if pos > len { + return Err(LoroError::OutOfBound { + pos, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + len, + }); + } + let value = v.into(); + ensure_no_regular_container_value(&value)?; + list.value.insert(pos, ValueOrHandler::Value(value)); + Ok(()) + } + MaybeDetached::Attached(a) => { + a.with_txn(|txn| self.insert_with_txn(txn, pos, v.into())) + } + } + } + + pub fn insert_with_txn( + &self, + txn: &mut Transaction, + pos: usize, + v: LoroValue, + ) -> LoroResult<()> { + if pos > self.len() { + return Err(LoroError::OutOfBound { + pos, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + len: self.len(), + }); + } + + let inner = self.inner.try_attached_state()?; + ensure_no_regular_container_value(&v)?; + + txn.apply_local_op( + inner.container_idx, + crate::op::RawOpContent::List(crate::container::list::list_op::ListOp::Insert { + slice: ListSlice::RawData(Cow::Owned(vec![v.clone()])), + pos, + }), + EventHint::InsertList { len: 1, pos }, + &inner.doc, + ) + } + + pub fn push(&self, v: impl Into) -> LoroResult<()> { + match &self.inner { + MaybeDetached::Detached(l) => { + let mut list = l.lock(); + let value = v.into(); + ensure_no_regular_container_value(&value)?; + list.value.push(ValueOrHandler::Value(value)); + Ok(()) + } + MaybeDetached::Attached(a) => a.with_txn(|txn| self.push_with_txn(txn, v.into())), + } + } + + pub fn push_with_txn(&self, txn: &mut Transaction, v: LoroValue) -> LoroResult<()> { + let pos = self.len(); + self.insert_with_txn(txn, pos, v) + } + + pub fn pop(&self) -> LoroResult> { + match &self.inner { + MaybeDetached::Detached(l) => { + let mut list = l.lock(); + Ok(list.value.pop().map(|v| v.to_value())) + } + MaybeDetached::Attached(a) => a.with_txn(|txn| self.pop_with_txn(txn)), + } + } + + pub fn pop_with_txn(&self, txn: &mut Transaction) -> LoroResult> { + let len = self.len(); + if len == 0 { + return Ok(None); + } + + let v = self.get(len - 1); + self.delete_with_txn(txn, len - 1, 1)?; + Ok(v) + } + + pub fn insert_container(&self, pos: usize, child: H) -> LoroResult { + match &self.inner { + MaybeDetached::Detached(l) => { + let mut list = l.lock(); + if pos > list.value.len() { + return Err(LoroError::OutOfBound { + pos, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + len: list.value.len(), + }); + } + list.value + .insert(pos, ValueOrHandler::Handler(child.to_handler())); + Ok(child) + } + MaybeDetached::Attached(a) => { + a.with_txn(|txn| self.insert_container_with_txn(txn, pos, child)) + } + } + } + + pub fn push_container(&self, child: H) -> LoroResult { + self.insert_container(self.len(), child) + } + + pub fn insert_container_with_txn( + &self, + txn: &mut Transaction, + pos: usize, + child: H, + ) -> LoroResult { + if pos > self.len() { + return Err(LoroError::OutOfBound { + pos, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + len: self.len(), + }); + } + + let inner = self.inner.try_attached_state()?; + let id = txn.next_id(); + let container_id = ContainerID::new_normal(id, child.kind()); + let v = LoroValue::Container(container_id.clone()); + txn.apply_local_op( + inner.container_idx, + crate::op::RawOpContent::List(crate::container::list::list_op::ListOp::Insert { + slice: ListSlice::RawData(Cow::Owned(vec![v.clone()])), + pos, + }), + EventHint::InsertList { len: 1, pos }, + &inner.doc, + )?; + let ans = child.attach(txn, inner, container_id)?; + Ok(ans) + } + + pub fn delete(&self, pos: usize, len: usize) -> LoroResult<()> { + match &self.inner { + MaybeDetached::Detached(l) => { + let mut list = l.lock(); + let end = checked_range_end( + pos, + len, + list.value.len(), + || format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + )?; + list.value.drain(pos..end); + Ok(()) + } + MaybeDetached::Attached(a) => a.with_txn(|txn| self.delete_with_txn(txn, pos, len)), + } + } + + pub fn delete_with_txn(&self, txn: &mut Transaction, pos: usize, len: usize) -> LoroResult<()> { + if len == 0 { + return Ok(()); + } + + let list_len = self.len(); + let end = checked_range_end( + pos, + len, + list_len, + || format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + )?; + + let inner = self.inner.try_attached_state()?; + let ids: Vec<_> = inner.with_state(|state| { + let list = state.as_list_state().unwrap(); + (pos..end).map(|i| list.get_id_at(i).unwrap()).collect() + }); + + for id in ids.into_iter() { + txn.apply_local_op( + inner.container_idx, + crate::op::RawOpContent::List(ListOp::Delete(DeleteSpanWithId::new( + id.id(), + pos as isize, + 1, + ))), + EventHint::DeleteList(DeleteSpan::new(pos as isize, 1)), + &inner.doc, + )?; + } + + Ok(()) + } + + pub fn get_child_handler(&self, index: usize) -> LoroResult { + match &self.inner { + MaybeDetached::Detached(l) => { + let list = l.lock(); + let value = list.value.get(index).ok_or(LoroError::OutOfBound { + pos: index, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + len: list.value.len(), + })?; + match value { + ValueOrHandler::Handler(h) => Ok(h.clone()), + _ => Err(LoroError::ArgErr( + format!( + "Expected container at index {}, but found {:?}", + index, value + ) + .into_boxed_str(), + )), + } + } + MaybeDetached::Attached(_) => { + let Some(value) = self.get_(index) else { + return Err(LoroError::OutOfBound { + pos: index, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + len: self.len(), + }); + }; + match value { + ValueOrHandler::Handler(handler) => Ok(handler), + ValueOrHandler::Value(value) => Err(LoroError::ArgErr( + format!( + "Expected container at index {}, but found {:?}", + index, value + ) + .into_boxed_str(), + )), + } + } + } + } + + pub fn len(&self) -> usize { + match &self.inner { + MaybeDetached::Detached(l) => l.lock().value.len(), + MaybeDetached::Attached(a) => { + a.with_doc_state(|state| state.get_list_len(a.container_idx)) + } + } + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + pub fn get_deep_value_with_id(&self) -> LoroResult { + let inner = self.inner.try_attached_state()?; + Ok(inner.with_doc_state(|state| { + state.get_container_deep_value_with_id(inner.container_idx, None) + })) + } + + pub fn get(&self, index: usize) -> Option { + match &self.inner { + MaybeDetached::Detached(l) => l.lock().value.get(index).map(|x| x.to_value()), + MaybeDetached::Attached(a) => { + a.with_doc_state(|state| state.get_list_value_at(a.container_idx, index)) + } + } + } + + /// Get value at given index, if it's a container, return a handler to the container + pub fn get_(&self, index: usize) -> Option { + match &self.inner { + MaybeDetached::Detached(l) => { + let l = l.lock(); + l.value.get(index).cloned() + } + MaybeDetached::Attached(inner) => { + let value = inner + .with_doc_state(|state| state.get_list_value_at(inner.container_idx, index)); + value.map(|value| value_to_value_or_handler(inner, value)) + } + } + } + + pub fn for_each(&self, mut f: I) + where + I: FnMut(ValueOrHandler), + { + match &self.inner { + MaybeDetached::Detached(l) => { + let l = l.lock(); + for v in l.value.iter() { + f(v.clone()) + } + } + MaybeDetached::Attached(inner) => { + let temp = inner.with_doc_state(|state| { + state + .get_list_values(inner.container_idx) + .into_iter() + .map(|value| value_to_value_or_handler(inner, value)) + .collect::>() + }); + for v in temp.into_iter() { + f(v); + } + } + } + } + + pub fn get_cursor(&self, pos: usize, side: Side) -> Option { + match &self.inner { + MaybeDetached::Detached(_) => None, + MaybeDetached::Attached(a) => { + let (id, len) = a.with_state(|s| { + let l = s.as_list_state().unwrap(); + (l.get_id_at(pos), l.len()) + }); + + if len == 0 { + return Some(Cursor { + id: None, + container: self.id(), + side: if side == Side::Middle { + Side::Left + } else { + side + }, + origin_pos: 0, + }); + } + + if len <= pos { + return Some(Cursor { + id: None, + container: self.id(), + side: Side::Right, + origin_pos: len, + }); + } + + let id = id?; + Some(Cursor { + id: Some(id.id()), + container: self.id(), + side, + origin_pos: pos, + }) + } + } + } + + fn apply_delta( + &self, + delta: loro_delta::DeltaRope< + loro_delta::array_vec::ArrayVec, + crate::event::ListDeltaMeta, + >, + on_container_remap: &mut dyn FnMut(ContainerID, ContainerID), + ) -> LoroResult<()> { + match &self.inner { + MaybeDetached::Detached(_) => unimplemented!(), + MaybeDetached::Attached(_) => { + let mut index = 0; + for item in delta.iter() { + match item { + loro_delta::DeltaItem::Retain { len, .. } => { + index += len; + } + loro_delta::DeltaItem::Replace { value, delete, .. } => { + if *delete > 0 { + self.delete(index, *delete)?; + } + + for v in value.iter() { + match v { + ValueOrHandler::Value(LoroValue::Container(old_id)) => { + let new_h = self.insert_container( + index, + Handler::new_unattached(old_id.container_type()), + )?; + let new_id = new_h.id(); + on_container_remap(old_id.clone(), new_id); + } + ValueOrHandler::Handler(h) => { + let old_id = h.id(); + let new_h = self.insert_container( + index, + Handler::new_unattached(old_id.container_type()), + )?; + let new_id = new_h.id(); + on_container_remap(old_id, new_id); + } + ValueOrHandler::Value(v) => { + self.insert(index, v.clone())?; + } + } + + index += 1; + } + } + } + } + } + } + + Ok(()) + } + + pub fn is_deleted(&self) -> bool { + match &self.inner { + MaybeDetached::Detached(_) => false, + MaybeDetached::Attached(a) => a.is_deleted(), + } + } + + pub fn clear(&self) -> LoroResult<()> { + match &self.inner { + MaybeDetached::Detached(l) => { + let mut l = l.lock(); + l.value.clear(); + Ok(()) + } + MaybeDetached::Attached(a) => a.with_txn(|txn| self.clear_with_txn(txn)), + } + } + + pub fn clear_with_txn(&self, txn: &mut Transaction) -> LoroResult<()> { + self.delete_with_txn(txn, 0, self.len()) + } + + pub fn get_id_at(&self, pos: usize) -> Option { + match &self.inner { + MaybeDetached::Detached(_) => None, + MaybeDetached::Attached(a) => a.with_state(|state| { + state + .as_list_state() + .unwrap() + .get_id_at(pos) + .map(|x| x.id()) + }), + } + } +} + +impl MovableListHandler { + pub fn insert(&self, pos: usize, v: impl Into) -> LoroResult<()> { + match &self.inner { + MaybeDetached::Detached(d) => { + let mut d = d.lock(); + if pos > d.value.len() { + return Err(LoroError::OutOfBound { + pos, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + len: d.value.len(), + }); + } + let value = v.into(); + ensure_no_regular_container_value(&value)?; + d.value.insert(pos, ValueOrHandler::Value(value)); + Ok(()) + } + MaybeDetached::Attached(a) => { + a.with_txn(|txn| self.insert_with_txn(txn, pos, v.into())) + } + } + } + + #[instrument(skip_all)] + pub fn insert_with_txn( + &self, + txn: &mut Transaction, + pos: usize, + v: LoroValue, + ) -> LoroResult<()> { + if pos > self.len() { + return Err(LoroError::OutOfBound { + pos, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + len: self.len(), + }); + } + + ensure_no_regular_container_value(&v)?; + + let op_index = self.with_state(|state| { + let list = state.as_movable_list_state().unwrap(); + Ok(list + .convert_index(pos, IndexType::ForUser, IndexType::ForOp) + .unwrap()) + })?; + + let inner = self.inner.try_attached_state()?; + txn.apply_local_op( + inner.container_idx, + crate::op::RawOpContent::List(crate::container::list::list_op::ListOp::Insert { + slice: ListSlice::RawData(Cow::Owned(vec![v.clone()])), + pos: op_index, + }), + EventHint::InsertList { len: 1, pos }, + &inner.doc, + ) + } + + #[inline] + pub fn mov(&self, from: usize, to: usize) -> LoroResult<()> { + match &self.inner { + MaybeDetached::Detached(d) => { + let mut d = d.lock(); + if from >= d.value.len() { + return Err(LoroError::OutOfBound { + pos: from, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + len: d.value.len(), + }); + } + if to >= d.value.len() { + return Err(LoroError::OutOfBound { + pos: to, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + len: d.value.len(), + }); + } + let v = d.value.remove(from); + d.value.insert(to, v); + Ok(()) + } + MaybeDetached::Attached(a) => a.with_txn(|txn| self.move_with_txn(txn, from, to)), + } + } + + /// Move element from `from` to `to`. After this op, elem will be at pos `to`. + #[instrument(skip_all)] + pub fn move_with_txn(&self, txn: &mut Transaction, from: usize, to: usize) -> LoroResult<()> { + if from == to { + return Ok(()); + } + + if from >= self.len() { + return Err(LoroError::OutOfBound { + pos: from, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + len: self.len(), + }); + } + + if to >= self.len() { + return Err(LoroError::OutOfBound { + pos: to, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + len: self.len(), + }); + } + + let (op_from, op_to, elem_id, value) = self.with_state(|state| { + let list = state.as_movable_list_state().unwrap(); + let (elem_id, elem) = list + .get_elem_at_given_pos(from, IndexType::ForUser) + .unwrap(); + Ok(( + list.convert_index(from, IndexType::ForUser, IndexType::ForOp) + .unwrap(), + list.convert_index(to, IndexType::ForUser, IndexType::ForOp) + .unwrap(), + elem_id, + elem.value().clone(), + )) + })?; + + let inner = self.inner.try_attached_state()?; + txn.apply_local_op( + inner.container_idx, + crate::op::RawOpContent::List(crate::container::list::list_op::ListOp::Move { + from: op_from as u32, + to: op_to as u32, + elem_id: elem_id.to_id(), + }), + EventHint::Move { + value, + from: from as u32, + to: to as u32, + }, + &inner.doc, + ) + } + + pub fn push(&self, v: LoroValue) -> LoroResult<()> { + match &self.inner { + MaybeDetached::Detached(d) => { + let mut d = d.lock(); + d.value.push(v.into()); + Ok(()) + } + MaybeDetached::Attached(a) => a.with_txn(|txn| self.push_with_txn(txn, v)), + } + } + + pub fn push_with_txn(&self, txn: &mut Transaction, v: LoroValue) -> LoroResult<()> { + let pos = self.len(); + self.insert_with_txn(txn, pos, v) + } + + pub fn pop_(&self) -> LoroResult> { + match &self.inner { + MaybeDetached::Detached(d) => { + let mut d = d.lock(); + Ok(d.value.pop()) + } + MaybeDetached::Attached(a) => { + if self.is_empty() { + return Ok(None); + } + let last = self.len() - 1; + let ans = self.get_(last); + a.with_txn(|txn| self.pop_with_txn(txn))?; + Ok(ans) + } + } + } + + pub fn pop(&self) -> LoroResult> { + match &self.inner { + MaybeDetached::Detached(a) => { + let mut a = a.lock(); + Ok(a.value.pop().map(|x| x.to_value())) + } + MaybeDetached::Attached(a) => a.with_txn(|txn| self.pop_with_txn(txn)), + } + } + + pub fn pop_with_txn(&self, txn: &mut Transaction) -> LoroResult> { + let len = self.len(); + if len == 0 { + return Ok(None); + } + + let v = self.get(len - 1); + self.delete_with_txn(txn, len - 1, 1)?; + Ok(v) + } + + pub fn insert_container(&self, pos: usize, child: H) -> LoroResult { + match &self.inner { + MaybeDetached::Detached(d) => { + let mut d = d.lock(); + if pos > d.value.len() { + return Err(LoroError::OutOfBound { + pos, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + len: d.value.len(), + }); + } + d.value + .insert(pos, ValueOrHandler::Handler(child.to_handler())); + Ok(child) + } + MaybeDetached::Attached(a) => { + a.with_txn(|txn| self.insert_container_with_txn(txn, pos, child)) + } + } + } + + pub fn push_container(&self, child: H) -> LoroResult { + self.insert_container(self.len(), child) + } + + pub fn insert_container_with_txn( + &self, + txn: &mut Transaction, + pos: usize, + child: H, + ) -> LoroResult { + if pos > self.len() { + return Err(LoroError::OutOfBound { + pos, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + len: self.len(), + }); + } + + let op_index = self.with_state(|state| { + let list = state.as_movable_list_state().unwrap(); + Ok(list + .convert_index(pos, IndexType::ForUser, IndexType::ForOp) + .unwrap()) + })?; + + let id = txn.next_id(); + let container_id = ContainerID::new_normal(id, child.kind()); + let v = LoroValue::Container(container_id.clone()); + let inner = self.inner.try_attached_state()?; + txn.apply_local_op( + inner.container_idx, + crate::op::RawOpContent::List(crate::container::list::list_op::ListOp::Insert { + slice: ListSlice::RawData(Cow::Owned(vec![v.clone()])), + pos: op_index, + }), + EventHint::InsertList { len: 1, pos }, + &inner.doc, + )?; + child.attach(txn, inner, container_id) + } + + pub fn set(&self, index: usize, value: impl Into) -> LoroResult<()> { + match &self.inner { + MaybeDetached::Detached(d) => { + let mut d = d.lock(); + if index >= d.value.len() { + return Err(LoroError::OutOfBound { + pos: index, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + len: d.value.len(), + }); + } + let value = value.into(); + ensure_no_regular_container_value(&value)?; + d.value[index] = ValueOrHandler::Value(value); + Ok(()) + } + MaybeDetached::Attached(a) => { + a.with_txn(|txn| self.set_with_txn(txn, index, value.into())) + } + } + } + + pub fn set_with_txn( + &self, + txn: &mut Transaction, + index: usize, + value: LoroValue, + ) -> LoroResult<()> { + if index >= self.len() { + return Err(LoroError::OutOfBound { + pos: index, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + len: self.len(), + }); + } + + let inner = self.inner.try_attached_state()?; + let Some(elem_id) = self.with_state(|state| { + let list = state.as_movable_list_state().unwrap(); + Ok(list.get_elem_id_at(index, IndexType::ForUser)) + })? + else { + unreachable!() + }; + ensure_no_regular_container_value(&value)?; + + let op = crate::op::RawOpContent::List(crate::container::list::list_op::ListOp::Set { + elem_id: elem_id.to_id(), + value: value.clone(), + }); + + let hint = EventHint::SetList { index, value }; + txn.apply_local_op(inner.container_idx, op, hint, &inner.doc) + } + + pub fn set_container(&self, pos: usize, child: H) -> LoroResult { + match &self.inner { + MaybeDetached::Detached(d) => { + let mut d = d.lock(); + if pos >= d.value.len() { + return Err(LoroError::OutOfBound { + pos, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + len: d.value.len(), + }); + } + d.value[pos] = ValueOrHandler::Handler(child.to_handler()); + Ok(child) + } + MaybeDetached::Attached(a) => { + a.with_txn(|txn| self.set_container_with_txn(txn, pos, child)) + } + } + } + + pub fn set_container_with_txn( + &self, + txn: &mut Transaction, + pos: usize, + child: H, + ) -> LoroResult { + let id = txn.next_id(); + let container_id = ContainerID::new_normal(id, child.kind()); + let v = LoroValue::Container(container_id.clone()); + let Some(elem_id) = self.with_state(|state| { + let list = state.as_movable_list_state().unwrap(); + Ok(list.get_elem_id_at(pos, IndexType::ForUser)) + })? + else { + let len = self.len(); + if pos >= len { + return Err(LoroError::OutOfBound { + pos, + len, + info: "".into(), + }); + } else { + unreachable!() + } + }; + let inner = self.inner.try_attached_state()?; + txn.apply_local_op( + inner.container_idx, + crate::op::RawOpContent::List(crate::container::list::list_op::ListOp::Set { + elem_id: elem_id.to_id(), + value: v.clone(), + }), + EventHint::SetList { + index: pos, + value: v, + }, + &inner.doc, + )?; + + child.attach(txn, inner, container_id) + } + + pub fn delete(&self, pos: usize, len: usize) -> LoroResult<()> { + match &self.inner { + MaybeDetached::Detached(d) => { + let mut d = d.lock(); + let end = checked_range_end( + pos, + len, + d.value.len(), + || format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + )?; + d.value.drain(pos..end); + Ok(()) + } + MaybeDetached::Attached(a) => a.with_txn(|txn| self.delete_with_txn(txn, pos, len)), + } + } + + #[instrument(skip_all)] + pub fn delete_with_txn(&self, txn: &mut Transaction, pos: usize, len: usize) -> LoroResult<()> { + if len == 0 { + return Ok(()); + } + + let list_len = self.len(); + let end = checked_range_end( + pos, + len, + list_len, + || format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + )?; + + let (ids, new_poses) = self.with_state(|state| { + let list = state.as_movable_list_state().unwrap(); + let ids: Vec<_> = (pos..end) + .map(|i| list.get_list_id_at(i, IndexType::ForUser).unwrap()) + .collect(); + let poses: Vec<_> = (pos..end) + // need to -i because we delete the previous ones + .map(|user_index| { + let op_index = list + .convert_index(user_index, IndexType::ForUser, IndexType::ForOp) + .unwrap(); + assert!(op_index >= user_index); + op_index - (user_index - pos) + }) + .collect(); + Ok((ids, poses)) + })?; + + loro_common::info!(?pos, ?len, ?ids, ?new_poses, "delete_with_txn"); + let user_pos = pos; + let inner = self.inner.try_attached_state()?; + for (id, op_pos) in ids.into_iter().zip(new_poses.into_iter()) { + txn.apply_local_op( + inner.container_idx, + crate::op::RawOpContent::List(ListOp::Delete(DeleteSpanWithId::new( + id, + op_pos as isize, + 1, + ))), + EventHint::DeleteList(DeleteSpan::new(user_pos as isize, 1)), + &inner.doc, + )?; + } + + Ok(()) + } + + pub fn get_child_handler(&self, index: usize) -> LoroResult { + match &self.inner { + MaybeDetached::Detached(l) => { + let list = l.lock(); + let value = list.value.get(index).ok_or(LoroError::OutOfBound { + pos: index, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + len: list.value.len(), + })?; + match value { + ValueOrHandler::Handler(h) => Ok(h.clone()), + _ => Err(LoroError::ArgErr( + format!( + "Expected container at index {}, but found {:?}", + index, value + ) + .into_boxed_str(), + )), + } + } + MaybeDetached::Attached(_) => { + let Some(value) = self.get_(index) else { + return Err(LoroError::OutOfBound { + pos: index, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + len: self.len(), + }); + }; + match value { + ValueOrHandler::Handler(handler) => Ok(handler), + ValueOrHandler::Value(value) => Err(LoroError::ArgErr( + format!( + "Expected container at index {}, but found {:?}", + index, value + ) + .into_boxed_str(), + )), + } + } + } + } + + pub fn len(&self) -> usize { + match &self.inner { + MaybeDetached::Detached(d) => { + let d = d.lock(); + d.value.len() + } + MaybeDetached::Attached(a) => { + a.with_doc_state(|state| state.get_list_len(a.container_idx)) + } + } + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + pub fn get_deep_value_with_id(&self) -> LoroValue { + let inner = self.inner.try_attached_state().unwrap(); + inner + .doc + .state + .lock() + .get_container_deep_value_with_id(inner.container_idx, None) + } + + pub fn get(&self, index: usize) -> Option { + match &self.inner { + MaybeDetached::Detached(d) => { + let d = d.lock(); + d.value.get(index).map(|v| v.to_value()) + } + MaybeDetached::Attached(a) => { + a.with_doc_state(|state| state.get_list_value_at(a.container_idx, index)) + } + } + } + + /// Get value at given index, if it's a container, return a handler to the container + pub fn get_(&self, index: usize) -> Option { + match &self.inner { + MaybeDetached::Detached(d) => { + let d = d.lock(); + d.value.get(index).cloned() + } + MaybeDetached::Attached(m) => { + let value = + m.with_doc_state(|state| state.get_list_value_at(m.container_idx, index)); + value.map(|value| value_to_value_or_handler(m, value)) + } + } + } + + pub fn for_each(&self, mut f: I) + where + I: FnMut(ValueOrHandler), + { + match &self.inner { + MaybeDetached::Detached(d) => { + let d = d.lock(); + for v in d.value.iter() { + f(v.clone()); + } + } + MaybeDetached::Attached(m) => { + let temp = m.with_doc_state(|state| { + state + .get_list_values(m.container_idx) + .into_iter() + .map(|value| value_to_value_or_handler(m, value)) + .collect::>() + }); + + for v in temp.into_iter() { + f(v); + } + } + } + } + + pub fn log_internal_state(&self) -> String { + match &self.inner { + MaybeDetached::Detached(d) => { + let d = d.lock(); + format!("{:#?}", &d.value) + } + MaybeDetached::Attached(a) => a.with_state(|state| { + let a = state.as_movable_list_state().unwrap(); + format!("{a:#?}") + }), + } + } + + pub fn new_detached() -> MovableListHandler { + MovableListHandler { + inner: MaybeDetached::new_detached(Default::default()), + } + } + + pub fn get_cursor(&self, pos: usize, side: Side) -> Option { + match &self.inner { + MaybeDetached::Detached(_) => None, + MaybeDetached::Attached(inner) => { + let (id, len) = inner.with_state(|s| { + let l = s.as_movable_list_state().unwrap(); + (l.get_list_item_id_at(pos), l.len()) + }); + + if len == 0 { + return Some(Cursor { + id: None, + container: self.id(), + side: if side == Side::Middle { + Side::Left + } else { + side + }, + origin_pos: 0, + }); + } + + if len <= pos { + return Some(Cursor { + id: None, + container: self.id(), + side: Side::Right, + origin_pos: len, + }); + } + + let id = id?; + Some(Cursor { + id: Some(id.id()), + container: self.id(), + side, + origin_pos: pos, + }) + } + } + } + + pub(crate) fn op_pos_to_user_pos(&self, new_pos: usize) -> usize { + match &self.inner { + MaybeDetached::Detached(_) => new_pos, + MaybeDetached::Attached(inner) => { + let mut pos = new_pos; + inner.with_state(|s| { + let l = s.as_movable_list_state().unwrap(); + pos = l + .convert_index(new_pos, IndexType::ForOp, IndexType::ForUser) + .unwrap_or(l.len()); + }); + pos + } + } + } + + pub fn is_deleted(&self) -> bool { + match &self.inner { + MaybeDetached::Detached(_) => false, + MaybeDetached::Attached(a) => a.is_deleted(), + } + } + + pub fn clear(&self) -> LoroResult<()> { + match &self.inner { + MaybeDetached::Detached(d) => { + let mut d = d.lock(); + d.value.clear(); + Ok(()) + } + MaybeDetached::Attached(a) => a.with_txn(|txn| self.clear_with_txn(txn)), + } + } + + pub fn clear_with_txn(&self, txn: &mut Transaction) -> LoroResult<()> { + self.delete_with_txn(txn, 0, self.len()) + } + + pub fn get_creator_at(&self, pos: usize) -> Option { + match &self.inner { + MaybeDetached::Detached(_) => None, + MaybeDetached::Attached(a) => { + a.with_state(|state| state.as_movable_list_state().unwrap().get_creator_at(pos)) + } + } + } + + pub fn get_last_mover_at(&self, pos: usize) -> Option { + match &self.inner { + MaybeDetached::Detached(_) => None, + MaybeDetached::Attached(a) => a.with_state(|state| { + state + .as_movable_list_state() + .unwrap() + .get_last_mover_at(pos) + }), + } + } + + pub fn get_last_editor_at(&self, pos: usize) -> Option { + match &self.inner { + MaybeDetached::Detached(_) => None, + MaybeDetached::Attached(a) => a.with_state(|state| { + state + .as_movable_list_state() + .unwrap() + .get_last_editor_at(pos) + }), + } + } +} + +impl MapHandler { + /// Create a new container that is detached from the document. + /// The edits on a detached container will not be persisted. + /// To attach the container to the document, please insert it into an attached container. + pub fn new_detached() -> Self { + Self { + inner: MaybeDetached::new_detached(Default::default()), + } + } + + pub fn insert(&self, key: &str, value: impl Into) -> LoroResult<()> { + match &self.inner { + MaybeDetached::Detached(m) => { + let mut m = m.lock(); + let value = value.into(); + ensure_no_regular_container_value(&value)?; + m.value.insert(key.into(), ValueOrHandler::Value(value)); + Ok(()) + } + MaybeDetached::Attached(a) => { + a.with_txn(|txn| self.insert_with_txn(txn, key, value.into())) + } + } + } + + /// This method will insert the value even if the same value is already in the given entry. + fn insert_without_skipping(&self, key: &str, value: impl Into) -> LoroResult<()> { + match &self.inner { + MaybeDetached::Detached(m) => { + let mut m = m.lock(); + let value = value.into(); + ensure_no_regular_container_value(&value)?; + m.value.insert(key.into(), ValueOrHandler::Value(value)); + Ok(()) + } + MaybeDetached::Attached(a) => a.with_txn(|txn| { + let this = &self; + let value = value.into(); + ensure_no_regular_container_value(&value)?; + + let inner = this.inner.try_attached_state()?; + txn.apply_local_op( + inner.container_idx, + crate::op::RawOpContent::Map(crate::container::map::MapSet { + key: key.into(), + value: Some(value.clone()), + }), + EventHint::Map { + key: key.into(), + value: Some(value.clone()), + }, + &inner.doc, + ) + }), + } + } + + pub fn insert_with_txn( + &self, + txn: &mut Transaction, + key: &str, + value: LoroValue, + ) -> LoroResult<()> { + ensure_no_regular_container_value(&value)?; + + if self.get(key).map(|x| x == value).unwrap_or(false) { + // skip if the value is already set + return Ok(()); + } + + let inner = self.inner.try_attached_state()?; + txn.apply_local_op( + inner.container_idx, + crate::op::RawOpContent::Map(crate::container::map::MapSet { + key: key.into(), + value: Some(value.clone()), + }), + EventHint::Map { + key: key.into(), + value: Some(value.clone()), + }, + &inner.doc, + ) + } + + pub fn insert_container(&self, key: &str, handler: T) -> LoroResult { + match &self.inner { + MaybeDetached::Detached(m) => { + let mut m = m.lock(); + let to_insert = handler.to_handler(); + m.value + .insert(key.into(), ValueOrHandler::Handler(to_insert.clone())); + Ok(handler) + } + MaybeDetached::Attached(a) => { + a.with_txn(|txn| self.insert_container_with_txn(txn, key, handler)) + } + } + } + + pub fn insert_container_with_txn( + &self, + txn: &mut Transaction, + key: &str, + child: H, + ) -> LoroResult { + let inner = self.inner.try_attached_state()?; + let id = txn.next_id(); + let container_id = ContainerID::new_normal(id, child.kind()); + txn.apply_local_op( + inner.container_idx, + crate::op::RawOpContent::Map(crate::container::map::MapSet { + key: key.into(), + value: Some(LoroValue::Container(container_id.clone())), + }), + EventHint::Map { + key: key.into(), + value: Some(LoroValue::Container(container_id.clone())), + }, + &inner.doc, + )?; + + child.attach(txn, inner, container_id) + } + + pub fn delete(&self, key: &str) -> LoroResult<()> { + match &self.inner { + MaybeDetached::Detached(m) => { + let mut m = m.lock(); + m.value.remove(key); + Ok(()) + } + MaybeDetached::Attached(a) => a.with_txn(|txn| self.delete_with_txn(txn, key)), + } + } + + pub fn delete_with_txn(&self, txn: &mut Transaction, key: &str) -> LoroResult<()> { + let inner = self.inner.try_attached_state()?; + txn.apply_local_op( + inner.container_idx, + crate::op::RawOpContent::Map(crate::container::map::MapSet { + key: key.into(), + value: None, + }), + EventHint::Map { + key: key.into(), + value: None, + }, + &inner.doc, + ) + } + + pub fn for_each(&self, mut f: I) + where + I: FnMut(&str, ValueOrHandler), + { + match &self.inner { + MaybeDetached::Detached(m) => { + let m = m.lock(); + for (k, v) in m.value.iter() { + f(k, v.clone()); + } + } + MaybeDetached::Attached(inner) => { + let temp = inner.with_doc_state(|state| { + state + .get_map_entries(inner.container_idx) + .into_iter() + .map(|(key, value)| { + let translated = loro_common::translate_mergeable_marker_value( + &inner.id, + key.as_ref(), + value, + ); + ( + key.to_string(), + value_to_value_or_handler(inner, translated), + ) + }) + .collect::>() + }); + + for (k, v) in temp.into_iter() { + f(&k, v.clone()); + } + } + } + } + + pub fn get_child_handler(&self, key: &str) -> LoroResult { + match &self.inner { + MaybeDetached::Detached(m) => { + let m = m.lock(); + let value = m.value.get(key).unwrap(); + match value { + ValueOrHandler::Value(v) => Err(LoroError::ArgErr( + format!("Expected Handler but found {:?}", v).into_boxed_str(), + )), + ValueOrHandler::Handler(h) => Ok(h.clone()), + } + } + MaybeDetached::Attached(_) => { + let Some(value) = self.get_(key) else { + return Err(LoroError::ArgErr( + format!("Key {key} does not exist").into_boxed_str(), + )); + }; + match value { + ValueOrHandler::Handler(handler) => Ok(handler), + ValueOrHandler::Value(value) => Err(LoroError::ArgErr( + format!("Expected Handler but found {:?}", value).into_boxed_str(), + )), + } + } + } + } + + pub fn get_deep_value_with_id(&self) -> LoroResult { + match &self.inner { + MaybeDetached::Detached(_) => Err(LoroError::MisuseDetachedContainer { + method: "get_deep_value_with_id", + }), + MaybeDetached::Attached(inner) => Ok(inner.with_doc_state(|state| { + state.get_container_deep_value_with_id(inner.container_idx, None) + })), + } + } + + pub fn get(&self, key: &str) -> Option { + match &self.inner { + MaybeDetached::Detached(m) => { + let m = m.lock(); + m.value.get(key).map(|v| v.to_value()) + } + MaybeDetached::Attached(inner) => { + let value = inner + .with_doc_state(|state| state.get_map_value_by_key(inner.container_idx, key))?; + Some(loro_common::translate_mergeable_marker_value( + &inner.id, key, value, + )) + } + } + } + + /// Get the value at given key, if value is a container, return a handler to the container + pub fn get_(&self, key: &str) -> Option { + match &self.inner { + MaybeDetached::Detached(m) => { + let m = m.lock(); + m.value.get(key).cloned() + } + MaybeDetached::Attached(inner) => { + let value = inner + .with_doc_state(|state| state.get_map_value_by_key(inner.container_idx, key))?; + let value = loro_common::translate_mergeable_marker_value(&inner.id, key, value); + Some(value_to_value_or_handler(inner, value)) + } + } + } + + /// Get or create a regular child container at `key`. + /// + /// This legacy method creates regular op-id child containers when the key is empty or `null`. + /// It is not mergeable: concurrent first creation at the same map key can fork child state and + /// leave one branch hidden by map conflict resolution. Prefer `ensure_mergeable_*` for lazy + /// map-key child creation. + #[deprecated( + note = "use ensure_mergeable_map/list/movable_list/text/tree/counter for lazy map-key child creation; this method creates regular op-id children" + )] + pub fn get_or_create_container(&self, key: &str, child: C) -> LoroResult { + if let Some(ans) = self.get_(key) { + if let ValueOrHandler::Handler(h) = ans { + let kind = h.kind(); + return C::from_handler(h).ok_or_else(move || { + LoroError::ArgErr( + format!("Expected value type {} but found {:?}", child.kind(), kind) + .into_boxed_str(), + ) + }); + } else if let ValueOrHandler::Value(LoroValue::Null) = ans { + // do nothing + } else { + return Err(LoroError::ArgErr( + format!("Expected value type {} but found {:?}", child.kind(), ans) + .into_boxed_str(), + )); + } + } + + self.insert_container(key, child) + } + + /// Shared implementation for all `ensure_mergeable_*` methods. + /// + /// Computes a deterministic [`ContainerID::Root`] in the mergeable namespace from + /// `(parent.id, key, child.kind())` and constructs the handler from it. Two peers calling this + /// with the same `(parent, key, kind)` receive handlers with identical container ids, which is + /// what makes the child container mergeable on concurrent first-write. + /// + /// # Errors + /// + /// Returns [`LoroError::MisuseDetachedContainer`] when called on a detached handler. The + /// deterministic cid is computed from the parent's cid, which a detached parent does not have + /// yet; falling back to a non-deterministic regular child would silently drop the mergeable + /// guarantee at attach time. Detached callers must attach the parent first. + /// + /// Returns [`LoroError::ArgErr`] if the parent slot already holds a non-mergeable value, or if + /// `C::from_handler` rejects the handler built from the deterministic cid (unreachable by + /// construction; guards against future drift between `from_handler` and `kind`). + fn ensure_mergeable_container(&self, key: &str, child: C) -> LoroResult { + let MaybeDetached::Attached(parent) = &self.inner else { + return Err(LoroError::MisuseDetachedContainer { + method: "ensure_mergeable_container", + }); + }; + + // Compare against the raw marker bytes (skipping `MapHandler::get`'s user-facing + // marker → Container translation) so the non-mergeable-occupant guard sees the real + // slot value and the same-kind idempotent-skip can match. + let existing_raw = + parent.with_doc_state(|state| state.get_map_value_by_key(parent.container_idx, key)); + + // A non-mergeable occupant (scalar, arbitrary binary, regular child container) would be + // silently clobbered by the marker write, so reject rather than overwrite under a + // `get_`-named API. Only the exact binary marker for this `(parent, key, kind)` is + // accepted as an existing mergeable occupant. + if let Some(existing) = &existing_raw { + if !matches!(existing, LoroValue::Null) + && loro_common::parse_mergeable_marker(&parent.id, key, existing).is_none() + { + return Err(LoroError::ArgErr( + format!( + "Cannot create a mergeable {} at key {key:?}: the key already holds a non-mergeable value", + child.kind() + ) + .into_boxed_str(), + )); + } + } + + let cid = ContainerID::new_mergeable(&parent.id, key, child.kind()); + let marker = loro_common::mergeable_marker(&parent.id, key, child.kind()); + + // Idempotent-skip on same marker: `MapHandler::get` translates markers to Container, so + // `insert_with_txn`'s equality check can't see this collision — do it directly. + // A different-kind marker is a deliberate kind change; let the insert through. + if existing_raw.as_ref() != Some(&marker) { + self.insert(key, marker)?; + } + + C::from_handler(create_handler(parent, cid.clone())).ok_or_else(|| { + LoroError::ArgErr( + format!( + "Expected value type {} but found {}", + child.kind(), + cid.container_type() + ) + .into_boxed_str(), + ) + }) + } + + #[cfg(feature = "counter")] + /// Ensure a mergeable Counter child exists under `key` and return its handler. + /// + /// Returns [`LoroError::MisuseDetachedContainer`] when called on a detached map. + /// Returns [`LoroError::ArgErr`] if the parent slot already holds a non-mergeable value. + /// Repeated same-kind calls are idempotent; different mergeable kinds deliberately rewrite + /// the active marker while preserving each mergeable child's deterministic state. + pub fn ensure_mergeable_counter(&self, key: &str) -> LoroResult { + self.ensure_mergeable_container(key, counter::CounterHandler::new_detached()) + } + + /// Ensure a mergeable Map child exists under `key` and return its handler. + /// + /// Returns [`LoroError::MisuseDetachedContainer`] when called on a detached map. + /// Returns [`LoroError::ArgErr`] if the parent slot already holds a non-mergeable value. + /// Repeated same-kind calls are idempotent; different mergeable kinds deliberately rewrite + /// the active marker while preserving each mergeable child's deterministic state. + /// + /// Prefer to avoid very deep mergeable-map chains: mergeable cids encode their flattened + /// logical path, so cid size still grows with depth and rides through every op/snapshot + /// reference to it. See [`MERGEABLE_NAMESPACE_PREFIX`](loro_common::MERGEABLE_NAMESPACE_PREFIX). + pub fn ensure_mergeable_map(&self, key: &str) -> LoroResult { + self.ensure_mergeable_container(key, MapHandler::new_detached()) + } + + /// Ensure a mergeable List child exists under `key` and return its handler. + /// + /// Returns [`LoroError::MisuseDetachedContainer`] when called on a detached map. + /// Returns [`LoroError::ArgErr`] if the parent slot already holds a non-mergeable value. + /// Repeated same-kind calls are idempotent; different mergeable kinds deliberately rewrite + /// the active marker while preserving each mergeable child's deterministic state. + pub fn ensure_mergeable_list(&self, key: &str) -> LoroResult { + self.ensure_mergeable_container(key, ListHandler::new_detached()) + } + + /// Ensure a mergeable MovableList child exists under `key` and return its handler. + /// + /// Returns [`LoroError::MisuseDetachedContainer`] when called on a detached map. + /// Returns [`LoroError::ArgErr`] if the parent slot already holds a non-mergeable value. + /// Repeated same-kind calls are idempotent; different mergeable kinds deliberately rewrite + /// the active marker while preserving each mergeable child's deterministic state. + pub fn ensure_mergeable_movable_list(&self, key: &str) -> LoroResult { + self.ensure_mergeable_container(key, MovableListHandler::new_detached()) + } + + /// Ensure a mergeable Text child exists under `key` and return its handler. + /// + /// Returns [`LoroError::MisuseDetachedContainer`] when called on a detached map. + /// Returns [`LoroError::ArgErr`] if the parent slot already holds a non-mergeable value. + /// Repeated same-kind calls are idempotent; different mergeable kinds deliberately rewrite + /// the active marker while preserving each mergeable child's deterministic state. + pub fn ensure_mergeable_text(&self, key: &str) -> LoroResult { + self.ensure_mergeable_container(key, TextHandler::new_detached()) + } + + /// Ensure a mergeable Tree child exists under `key` and return its handler. + /// + /// Returns [`LoroError::MisuseDetachedContainer`] when called on a detached map. + /// Returns [`LoroError::ArgErr`] if the parent slot already holds a non-mergeable value. + /// Repeated same-kind calls are idempotent; different mergeable kinds deliberately rewrite + /// the active marker while preserving each mergeable child's deterministic state. + pub fn ensure_mergeable_tree(&self, key: &str) -> LoroResult { + self.ensure_mergeable_container(key, TreeHandler::new_detached()) + } + + pub fn contains_key(&self, key: &str) -> bool { + self.get(key).is_some() + } + + pub fn len(&self) -> usize { + match &self.inner { + MaybeDetached::Detached(m) => m.lock().value.len(), + MaybeDetached::Attached(a) => { + a.with_doc_state(|state| state.get_map_len(a.container_idx)) + } + } + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + pub fn is_deleted(&self) -> bool { + match &self.inner { + MaybeDetached::Detached(_) => false, + MaybeDetached::Attached(a) => a.is_deleted(), + } + } + + pub fn clear(&self) -> LoroResult<()> { + match &self.inner { + MaybeDetached::Detached(m) => { + let mut m = m.lock(); + m.value.clear(); + Ok(()) + } + MaybeDetached::Attached(a) => a.with_txn(|txn| self.clear_with_txn(txn)), + } + } + + pub fn clear_with_txn(&self, txn: &mut Transaction) -> LoroResult<()> { + let keys: Vec = self.inner.try_attached_state()?.with_state(|state| { + state + .as_map_state() + .unwrap() + .iter() + .map(|(k, _)| k.clone()) + .collect() + }); + + for key in keys { + self.delete_with_txn(txn, &key)?; + } + + Ok(()) + } + + pub fn keys(&self) -> impl Iterator + '_ { + let keys: Vec = match &self.inner { + MaybeDetached::Detached(m) => { + let m = m.lock(); + m.value.keys().map(|x| x.as_str().into()).collect() + } + MaybeDetached::Attached(a) => { + a.with_doc_state(|state| state.get_map_keys(a.container_idx)) + } + }; + + keys.into_iter() + } + + pub fn values(&self) -> impl Iterator + '_ { + let values: Vec = match &self.inner { + MaybeDetached::Detached(m) => { + let m = m.lock(); + m.value.values().cloned().collect() + } + MaybeDetached::Attached(a) => a.with_doc_state(|state| { + // A mergeable child's marker lives in the parent map's value table; iterate + // entries (key + value) so the user-boundary translation can resolve each marker + // to the deterministic child cid before wrapping into a handler. + state + .get_map_entries(a.container_idx) + .into_iter() + .map(|(key, value)| { + let translated = loro_common::translate_mergeable_marker_value( + &a.id, + key.as_ref(), + value, + ); + value_to_value_or_handler(a, translated) + }) + .collect() + }), + }; + + values.into_iter() + } + + pub fn get_last_editor(&self, key: &str) -> Option { + match &self.inner { + MaybeDetached::Detached(_) => None, + MaybeDetached::Attached(a) => a.with_state(|state| { + let m = state.as_map_state().unwrap(); + m.get_last_edit_peer(key) + }), + } + } +} + +fn with_txn(doc: &LoroDoc, f: impl FnOnce(&mut Transaction) -> LoroResult) -> LoroResult { + let txn = &doc.txn; + let mut txn = txn.lock(); + loop { + if let Some(txn) = &mut *txn { + return f(txn); + } else if cfg!(target_arch = "wasm32") || !doc.can_edit() { + return Err(LoroError::AutoCommitNotStarted); + } else { + drop(txn); + #[cfg(loom)] + loom::thread::yield_now(); + doc.start_auto_commit(); + txn = doc.txn.lock(); + } + } +} + +#[cfg(feature = "counter")] +pub mod counter { + + use loro_common::LoroResult; + + use crate::{ + txn::{EventHint, Transaction}, + HandlerTrait, + }; + + use super::{create_handler, Handler, MaybeDetached}; + + #[derive(Clone)] + pub struct CounterHandler { + pub(super) inner: MaybeDetached, + } + + impl CounterHandler { + pub fn new_detached() -> Self { + Self { + inner: MaybeDetached::new_detached(0.), + } + } + + pub fn increment(&self, n: f64) -> LoroResult<()> { + match &self.inner { + MaybeDetached::Detached(d) => { + let d = &mut d.lock().value; + *d += n; + Ok(()) + } + MaybeDetached::Attached(a) => a.with_txn(|txn| self.increment_with_txn(txn, n)), + } + } + + pub fn decrement(&self, n: f64) -> LoroResult<()> { + match &self.inner { + MaybeDetached::Detached(d) => { + let d = &mut d.lock().value; + *d -= n; + Ok(()) + } + MaybeDetached::Attached(a) => a.with_txn(|txn| self.increment_with_txn(txn, -n)), + } + } + + fn increment_with_txn(&self, txn: &mut Transaction, n: f64) -> LoroResult<()> { + let inner = self.inner.try_attached_state()?; + txn.apply_local_op( + inner.container_idx, + crate::op::RawOpContent::Counter(n), + EventHint::Counter(n), + &inner.doc, + ) + } + + pub fn is_deleted(&self) -> bool { + match &self.inner { + MaybeDetached::Detached(_) => false, + MaybeDetached::Attached(a) => a.is_deleted(), + } + } + + pub fn clear(&self) -> LoroResult<()> { + self.decrement(self.get_value().into_double().unwrap()) + } + } + + impl std::fmt::Debug for CounterHandler { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match &self.inner { + MaybeDetached::Detached(_) => write!(f, "CounterHandler Detached"), + MaybeDetached::Attached(a) => write!(f, "CounterHandler {}", a.id), + } + } + } + + impl HandlerTrait for CounterHandler { + fn is_attached(&self) -> bool { + matches!(&self.inner, MaybeDetached::Attached(..)) + } + + fn attached_handler(&self) -> Option<&crate::BasicHandler> { + self.inner.attached_handler() + } + + fn get_value(&self) -> loro_common::LoroValue { + match &self.inner { + MaybeDetached::Detached(t) => { + let t = t.lock(); + t.value.into() + } + MaybeDetached::Attached(a) => a.get_value(), + } + } + + fn get_deep_value(&self) -> loro_common::LoroValue { + self.get_value() + } + + fn kind(&self) -> loro_common::ContainerType { + loro_common::ContainerType::Counter + } + + fn to_handler(&self) -> super::Handler { + Handler::Counter(self.clone()) + } + + fn from_handler(h: super::Handler) -> Option { + match h { + Handler::Counter(x) => Some(x), + _ => None, + } + } + + fn attach( + &self, + txn: &mut crate::txn::Transaction, + parent: &crate::BasicHandler, + self_id: loro_common::ContainerID, + ) -> loro_common::LoroResult { + match &self.inner { + MaybeDetached::Detached(v) => { + let mut v = v.lock(); + let inner = create_handler(parent, self_id); + let c = inner.into_counter().unwrap(); + + c.increment_with_txn(txn, v.value)?; + + v.attached = c.attached_handler().cloned(); + Ok(c) + } + MaybeDetached::Attached(a) => { + let new_inner = create_handler(a, self_id); + let ans = new_inner.into_counter().unwrap(); + let delta = *self.get_value().as_double().unwrap(); + ans.increment_with_txn(txn, delta)?; + Ok(ans) + } + } + } + + fn get_attached(&self) -> Option { + match &self.inner { + MaybeDetached::Attached(a) => Some(Self { + inner: MaybeDetached::Attached(a.clone()), + }), + MaybeDetached::Detached(v) => v.lock().attached.clone().map(|x| Self { + inner: MaybeDetached::Attached(x), + }), + } + } + + fn doc(&self) -> Option { + match &self.inner { + MaybeDetached::Detached(_) => None, + MaybeDetached::Attached(a) => Some(a.doc()), + } + } + } +} + +#[cfg(test)] +mod test { + use std::borrow::Cow; + + use super::{ + Handler, HandlerTrait, ListHandler, MapHandler, MovableListHandler, TextDelta, TextHandler, + ValueOrHandler, + }; + use crate::container::list::list_op::ListOp; + use crate::cursor::PosType; + use crate::loro::ExportMode; + use crate::op::ListSlice; + use crate::state::TreeParentId; + use crate::txn::EventHint; + use crate::version::Frontiers; + use crate::LoroDoc; + use crate::{fx_map, ToJson}; + use loro_common::{ContainerID, ContainerType, LoroError, LoroValue, ID}; + use serde_json::json; + + fn recheck_fast_blob(mut bytes: Vec) -> Vec { + let checksum = xxhash_rust::xxh32::xxh32(&bytes[20..], u32::from_le_bytes(*b"LORO")); + bytes[16..20].copy_from_slice(&checksum.to_le_bytes()); + bytes + } + + fn replace_fast_snapshot_state_bytes(mut snapshot: Vec, state_bytes: &[u8]) -> Vec { + let mut body = &snapshot[22..]; + let oplog_len = u32::from_le_bytes(body[..4].try_into().unwrap()) as usize; + body = &body[4 + oplog_len..]; + let old_state_len = u32::from_le_bytes(body[..4].try_into().unwrap()) as usize; + let state_len_pos = 22 + 4 + oplog_len; + let state_start = state_len_pos + 4; + let state_end = state_start + old_state_len; + snapshot[state_len_pos..state_start] + .copy_from_slice(&(state_bytes.len() as u32).to_le_bytes()); + snapshot.splice(state_start..state_end, state_bytes.iter().copied()); + recheck_fast_blob(snapshot) + } + + fn insert_many_with_single_list_op( + txn: &mut crate::txn::Transaction, + list: &crate::handler::ListHandler, + pos: usize, + values: Vec, + ) { + let len = values.len(); + let inner = list.inner.try_attached_state().unwrap(); + txn.apply_local_op( + inner.container_idx, + crate::op::RawOpContent::List(ListOp::Insert { + slice: ListSlice::RawData(Cow::Owned(values)), + pos, + }), + EventHint::InsertList { + len: len as u32, + pos, + }, + &inner.doc, + ) + .unwrap(); + } + + #[test] + fn richtext_handler() { + let loro = LoroDoc::new(); + loro.set_peer_id(1).unwrap(); + let loro2 = LoroDoc::new(); + loro2.set_peer_id(2).unwrap(); + + let mut txn = loro.txn().unwrap(); + let text = txn.get_text("hello"); + text.insert_with_txn(&mut txn, 0, "hello", PosType::Unicode) + .unwrap(); + txn.commit().unwrap(); + let exported = loro.export(ExportMode::all_updates()).unwrap(); + + loro2.import(&exported).unwrap(); + let mut txn = loro2.txn().unwrap(); + let text = txn.get_text("hello"); + assert_eq!(&**text.get_value().as_string().unwrap(), "hello"); + text.insert_with_txn(&mut txn, 5, " world", PosType::Unicode) + .unwrap(); + assert_eq!(&**text.get_value().as_string().unwrap(), "hello world"); + txn.commit().unwrap(); + + loro.import(&loro2.export(ExportMode::all_updates()).unwrap()) + .unwrap(); + let txn = loro.txn().unwrap(); + let text = txn.get_text("hello"); + assert_eq!(&**text.get_value().as_string().unwrap(), "hello world"); + txn.commit().unwrap(); + + // test checkout + loro.checkout(&Frontiers::from_id(ID::new(2, 1))).unwrap(); + assert_eq!(&**text.get_value().as_string().unwrap(), "hello w"); + } + + #[test] + fn richtext_handler_concurrent() { + let loro = LoroDoc::new(); + let mut txn = loro.txn().unwrap(); + let handler = loro.get_text("richtext"); + handler + .insert_with_txn(&mut txn, 0, "hello", PosType::Unicode) + .unwrap(); + txn.commit().unwrap(); + for i in 0..100 { + let new_loro = LoroDoc::new(); + new_loro + .import(&loro.export(ExportMode::all_updates()).unwrap()) + .unwrap(); + let mut txn = new_loro.txn().unwrap(); + let handler = new_loro.get_text("richtext"); + handler + .insert_with_txn(&mut txn, i % 5, &i.to_string(), PosType::Unicode) + .unwrap(); + txn.commit().unwrap(); + loro.import( + &new_loro + .export(ExportMode::updates(&loro.oplog_vv())) + .unwrap(), + ) + .unwrap(); + } + } + + #[test] + fn cross_doc_txn_is_rejected() { + // `insert_with_txn`/`delete_with_txn` are public API, so a transaction + // from one document can be fed to another document's handler. That must + // be rejected with `UnmatchedContext` rather than silently stamping the + // target doc's state/oplog with the wrong peer+counter. Regression test + // for the always-on (release included) context check in + // `Transaction::apply_local_op`. + let doc_a = LoroDoc::new(); + doc_a.set_peer_id(1).unwrap(); + let doc_b = LoroDoc::new(); + doc_b.set_peer_id(2).unwrap(); + + // Seed doc_b so it has real state we can prove stays untouched. + { + let mut txn_b = doc_b.txn().unwrap(); + doc_b + .get_text("text") + .insert_with_txn(&mut txn_b, 0, "ok", PosType::Unicode) + .unwrap(); + txn_b.commit().unwrap(); + } + let vv_before = doc_b.oplog_vv(); + + // Feed doc_a's transaction to doc_b's handler. + let mut txn_a = doc_a.txn().unwrap(); + let text_b = doc_b.get_text("text"); + let insert_err = text_b + .insert_with_txn(&mut txn_a, 0, "x", PosType::Unicode) + .unwrap_err(); + assert!(matches!(insert_err, LoroError::UnmatchedContext { .. })); + let delete_err = text_b + .delete_with_txn(&mut txn_a, 0, 1, PosType::Unicode) + .unwrap_err(); + assert!(matches!(delete_err, LoroError::UnmatchedContext { .. })); + txn_a.commit().unwrap(); + + // doc_b is unchanged: content and version vector identical. + assert_eq!(&**text_b.get_value().as_string().unwrap(), "ok"); + assert_eq!(doc_b.oplog_vv(), vv_before); + } + + #[test] + fn list_import_batch_stays_consistent_after_repeated_tail_splits() { + let doc_a = LoroDoc::new(); + doc_a.set_peer_id(1).unwrap(); + let mut txn = doc_a.txn().unwrap(); + let list_a = txn.get_list("list"); + insert_many_with_single_list_op( + &mut txn, + &list_a, + 0, + (0..300).map(|i| LoroValue::I64(i)).collect(), + ); + txn.commit().unwrap(); + + let doc_b = LoroDoc::new(); + doc_b.set_peer_id(2).unwrap(); + doc_b + .import(&doc_a.export(ExportMode::all_updates()).unwrap()) + .unwrap(); + + let list_b = doc_b.get_list("list"); + let mut vv = doc_a.oplog_vv(); + let mut updates = Vec::new(); + for (i, pos) in [100, 201, 252, 278].into_iter().enumerate() { + list_b.insert(pos, 1000 + i as i64).unwrap(); + updates.push(doc_b.export(ExportMode::updates(&vv)).unwrap()); + vv = doc_b.oplog_vv(); + } + + doc_a.import_batch(&updates).unwrap(); + doc_a.check_state_diff_calc_consistency_slow(); + doc_b.check_state_diff_calc_consistency_slow(); + assert_eq!(doc_a.get_deep_value(), doc_b.get_deep_value()); + } + + #[test] + fn richtext_handler_mark() { + let loro = LoroDoc::new_auto_commit(); + let handler = loro.get_text("richtext"); + handler.insert(0, "hello world", PosType::Unicode).unwrap(); + handler + .mark(0, 5, "bold", true.into(), PosType::Event) + .unwrap(); + loro.commit_then_renew(); + + // assert has bold + let value = handler.get_richtext_value(); + assert_eq!(value[0]["insert"], "hello".into()); + let meta = value[0]["attributes"].as_map().unwrap(); + assert_eq!(meta.len(), 1); + meta.get("bold").unwrap(); + + let loro2 = LoroDoc::new_auto_commit(); + loro2 + .import(&loro.export(ExportMode::all_updates()).unwrap()) + .unwrap(); + let handler2 = loro2.get_text("richtext"); + assert_eq!(&**handler2.get_value().as_string().unwrap(), "hello world"); + + // assert has bold + let value = handler2.get_richtext_value(); + assert_eq!(value[0]["insert"], "hello".into()); + let meta = value[0]["attributes"].as_map().unwrap(); + assert_eq!(meta.len(), 1); + meta.get("bold").unwrap(); + + // insert after bold should be bold + { + handler2.insert(5, " new", PosType::Unicode).unwrap(); + let value = handler2.get_richtext_value(); + assert_eq!( + value.to_json_value(), + serde_json::json!([ + {"insert": "hello new", "attributes": {"bold": true}}, + {"insert": " world"} + ]) + ); + } + } + + #[test] + fn richtext_snapshot() { + let loro = LoroDoc::new(); + let mut txn = loro.txn().unwrap(); + let handler = loro.get_text("richtext"); + handler + .insert_with_txn(&mut txn, 0, "hello world", PosType::Unicode) + .unwrap(); + handler + .mark_with_txn(&mut txn, 0, 5, "bold", true.into(), PosType::Event) + .unwrap(); + txn.commit().unwrap(); + + let loro2 = LoroDoc::new(); + loro2 + .import(&loro.export(ExportMode::snapshot()).unwrap()) + .unwrap(); + let handler2 = loro2.get_text("richtext"); + assert_eq!( + handler2.get_richtext_value().to_json_value(), + serde_json::json!([ + {"insert": "hello", "attributes": {"bold": true}}, + {"insert": " world"} + ]) + ); + } + + #[test] + fn text_snapshot_string_queries_do_not_decode_state() { + let loro = LoroDoc::new_auto_commit(); + let text = loro.get_text("text"); + text.insert(0, "a😀文", PosType::Unicode).unwrap(); + text.mark(1, 3, "bold", true.into(), PosType::Unicode) + .unwrap(); + + let restored = LoroDoc::new(); + restored + .import(&loro.export(ExportMode::snapshot()).unwrap()) + .unwrap(); + let text = restored.get_text("text"); + assert!(!text.attached_handler().unwrap().has_decoded_state()); + + assert_eq!(text.len_unicode(), 3); + assert_eq!(text.len_utf16(), 4); + assert_eq!(text.len_utf8(), "a😀文".len()); + assert_eq!(text.char_at(1, PosType::Unicode).unwrap(), '😀'); + assert_eq!(text.slice(1, 3, PosType::Unicode).unwrap(), "😀文"); + assert_eq!( + text.convert_pos(2, PosType::Unicode, PosType::Utf16), + Some(3) + ); + assert!(matches!( + text.delete_utf16(2, 1), + Err(LoroError::UTF16InUnicodeCodePoint { pos: 2 }) + )); + assert!(matches!( + text.delete_utf8(2, 1), + Err(LoroError::UTF8InUnicodeCodePoint { pos: 2 }) + )); + assert!(matches!( + text.slice_delta(2, 3, PosType::Utf16), + Err(LoroError::UTF16InUnicodeCodePoint { pos: 2 }) + )); + assert!(matches!( + text.slice_delta(2, 3, PosType::Bytes), + Err(LoroError::UTF8InUnicodeCodePoint { pos: 2 }) + )); + assert!(!text.attached_handler().unwrap().has_decoded_state()); + + assert_eq!(text.get_delta().len(), 2); + assert!(text.attached_handler().unwrap().has_decoded_state()); + } + + #[test] + fn text_lazy_event_queries_match_decoded_state() { + let loro = LoroDoc::new_auto_commit(); + let text = loro.get_text("text"); + text.insert(0, "ab😀cd", PosType::Unicode).unwrap(); + text.mark(1, 4, "bold", true.into(), PosType::Unicode) + .unwrap(); + text.mark(2, 3, "link", "x".into(), PosType::Unicode) + .unwrap(); + + let lazy_doc = LoroDoc::new(); + lazy_doc + .import(&loro.export(ExportMode::snapshot()).unwrap()) + .unwrap(); + let lazy_text = lazy_doc.get_text("text"); + + let decoded_doc = LoroDoc::new(); + decoded_doc + .import(&loro.export(ExportMode::snapshot()).unwrap()) + .unwrap(); + let decoded_text = decoded_doc.get_text("text"); + decoded_text.get_delta(); + + assert!(!lazy_text.attached_handler().unwrap().has_decoded_state()); + assert!(decoded_text.attached_handler().unwrap().has_decoded_state()); + + for pos_type in [ + PosType::Event, + PosType::Unicode, + PosType::Utf16, + PosType::Bytes, + ] { + assert_eq!(lazy_text.len(pos_type), decoded_text.len(pos_type)); + for pos in 0..=decoded_text.len(pos_type) { + assert_eq!( + lazy_text.convert_pos(pos, pos_type, PosType::Unicode), + decoded_text.convert_pos(pos, pos_type, PosType::Unicode), + "convert {pos_type:?} pos {pos} to unicode" + ); + assert_eq!( + lazy_text.convert_pos(pos, pos_type, PosType::Event), + decoded_text.convert_pos(pos, pos_type, PosType::Event), + "convert {pos_type:?} pos {pos} to event" + ); + if pos < decoded_text.len(pos_type) { + assert_eq!( + lazy_text.char_at(pos, pos_type), + decoded_text.char_at(pos, pos_type), + "char_at {pos_type:?} pos {pos}" + ); + } + for end in pos..=decoded_text.len(pos_type) { + assert_eq!( + lazy_text.slice(pos, end, pos_type), + decoded_text.slice(pos, end, pos_type), + "slice {pos_type:?} {pos}..{end}" + ); + } + } + } + } + + #[test] + fn deep_value_with_id_uses_lazy_values_for_snapshot_roots() { + let loro = LoroDoc::new_auto_commit(); + let text = loro.get_text("text"); + text.insert(0, "hello", PosType::Unicode).unwrap(); + let map = loro.get_map("map"); + map.insert("key", "value").unwrap(); + let list = loro.get_list("list"); + list.push("item").unwrap(); + + let restored = LoroDoc::new(); + restored + .import(&loro.export(ExportMode::snapshot()).unwrap()) + .unwrap(); + let text = restored.get_text("text"); + let map = restored.get_map("map"); + let list = restored.get_list("list"); + + let value = restored.get_deep_value_with_id(); + assert_eq!(value["text"]["value"], "hello".into()); + assert_eq!(value["map"]["value"]["key"], "value".into()); + assert_eq!(value["list"]["value"][0], "item".into()); + assert!(!text.attached_handler().unwrap().has_decoded_state()); + assert!(!map.attached_handler().unwrap().has_decoded_state()); + assert!(!list.attached_handler().unwrap().has_decoded_state()); + } + + #[test] + fn lazy_value_reads_do_not_write_stale_snapshot_after_mutation() { + let loro = LoroDoc::new_auto_commit(); + let map = loro.get_map("map"); + map.insert("key", "old").unwrap(); + let child = map + .insert_container("child", MapHandler::new_detached()) + .unwrap(); + child.insert("nested", "old").unwrap(); + let list = loro.get_list("list"); + list.push("old").unwrap(); + let child_list = list.push_container(ListHandler::new_detached()).unwrap(); + child_list.push("nested-old").unwrap(); + + let restored = LoroDoc::new(); + restored + .import(&loro.export(ExportMode::snapshot()).unwrap()) + .unwrap(); + let map = restored.get_map("map"); + let list = restored.get_list("list"); + + assert_eq!(map.get("key").unwrap(), "old".into()); + assert_eq!(list.get(0).unwrap(), "old".into()); + let child = match map.get_("child").unwrap() { + ValueOrHandler::Handler(handler) => handler.into_map().unwrap(), + ValueOrHandler::Value(value) => panic!("expected child map, got {value:?}"), + }; + let child_list = match list.get_(1).unwrap() { + ValueOrHandler::Handler(handler) => handler.into_list().unwrap(), + ValueOrHandler::Value(value) => panic!("expected child list, got {value:?}"), + }; + + map.insert("key", "new").unwrap(); + child.insert("nested", "new").unwrap(); + list.delete(0, 1).unwrap(); + list.insert(0, "new").unwrap(); + child_list.delete(0, 1).unwrap(); + child_list.insert(0, "nested-new").unwrap(); + restored.commit_then_renew(); + + let roundtrip = LoroDoc::new(); + roundtrip + .import(&restored.export(ExportMode::snapshot()).unwrap()) + .unwrap(); + assert_eq!( + roundtrip.get_deep_value().to_json_value(), + serde_json::json!({ + "map": { "key": "new", "child": { "nested": "new" } }, + "list": ["new", ["nested-new"]] + }) + ); + } + + #[test] + fn fast_snapshot_with_trailing_bytes_is_rejected_on_import() { + let loro = LoroDoc::new_auto_commit(); + let map = loro.get_map("map"); + map.insert("key", "value").unwrap(); + let mut snapshot = loro.export(ExportMode::snapshot()).unwrap(); + snapshot.push(0xff); + let corrupted = recheck_fast_blob(snapshot); + + let doc = LoroDoc::new(); + assert!(doc.import(&corrupted).is_err()); + } + + #[test] + fn fast_snapshot_with_trailing_bytes_is_rejected_by_meta_decoder() { + let loro = LoroDoc::new_auto_commit(); + let map = loro.get_map("map"); + map.insert("key", "value").unwrap(); + let mut snapshot = loro.export(ExportMode::snapshot()).unwrap(); + snapshot.push(0xff); + let corrupted = recheck_fast_blob(snapshot); + + assert!(LoroDoc::decode_import_blob_meta(&corrupted, true).is_err()); + } + + #[test] + fn fast_snapshot_empty_sstable_meta_is_rejected_on_import() { + let loro = LoroDoc::new_auto_commit(); + let map = loro.get_map("map"); + map.insert("key", "value").unwrap(); + let snapshot = loro.export(ExportMode::snapshot()).unwrap(); + + let mut malformed_state = Vec::new(); + malformed_state.extend_from_slice(b"LORO"); + malformed_state.push(0); + malformed_state.extend_from_slice(&0u32.to_le_bytes()); + let checksum = xxhash_rust::xxh32::xxh32(&[], u32::from_le_bytes(*b"LORO")); + malformed_state.extend_from_slice(&checksum.to_le_bytes()); + malformed_state.extend_from_slice(&5u32.to_le_bytes()); + let corrupted = replace_fast_snapshot_state_bytes(snapshot, &malformed_state); + + let doc = LoroDoc::new(); + assert!(doc.import(&corrupted).is_err()); + } + + #[test] + fn tree_meta() { + let loro = LoroDoc::new_auto_commit(); + loro.set_peer_id(1).unwrap(); + let tree = loro.get_tree("root"); + let id = tree.create(TreeParentId::Root).unwrap(); + let meta = tree.get_meta(id).unwrap(); + meta.insert("a", 123).unwrap(); + loro.commit_then_renew(); + let meta = tree.get_meta(id).unwrap(); + assert_eq!(meta.get("a").unwrap(), 123.into()); + assert_eq!( + json!([{"parent":null,"meta":{"a":123},"id":"0@1","index":0,"children":[],"fractional_index":"80"}]), + tree.get_deep_value().to_json_value() + ); + let bytes = loro.export(ExportMode::snapshot()).unwrap(); + let loro2 = LoroDoc::new(); + loro2.import(&bytes).unwrap(); + } + + #[test] + fn tree_meta_event() { + use std::sync::Arc; + let loro = LoroDoc::new_auto_commit(); + let tree = loro.get_tree("root"); + let text = loro.get_text("text"); + + let id = tree.create(TreeParentId::Root).unwrap(); + let meta = tree.get_meta(id).unwrap(); + meta.insert("a", 1).unwrap(); + text.insert(0, "abc", PosType::Unicode).unwrap(); + let _id2 = tree.create(TreeParentId::Root).unwrap(); + meta.insert("b", 2).unwrap(); + + let loro2 = LoroDoc::new_auto_commit(); + let _g = loro2.subscribe_root(Arc::new(|e| { + println!("{} {:?} ", e.event_meta.by, e.event_meta.diff) + })); + loro2 + .import(&loro.export(ExportMode::all_updates()).unwrap()) + .unwrap(); + assert_eq!(loro.get_deep_value(), loro2.get_deep_value()); + } + + #[test] + fn richtext_apply_delta() { + let loro = LoroDoc::new_auto_commit(); + let text = loro.get_text("text"); + text.apply_delta(&[TextDelta::Insert { + insert: "Hello World!".into(), + attributes: None, + }]) + .unwrap(); + dbg!(text.get_richtext_value()); + text.apply_delta(&[ + TextDelta::Retain { + retain: 6, + attributes: Some(fx_map!("italic".into() => loro_common::LoroValue::Bool(true))), + }, + TextDelta::Insert { + insert: "New ".into(), + attributes: Some(fx_map!("bold".into() => loro_common::LoroValue::Bool(true))), + }, + ]) + .unwrap(); + dbg!(text.get_richtext_value()); + loro.commit_then_renew(); + assert_eq!( + text.get_richtext_value().to_json_value(), + json!([ + {"insert": "Hello ", "attributes": {"italic": true}}, + {"insert": "New ", "attributes": {"bold": true}}, + {"insert": "World!"} + + ]) + ) + } + + #[test] + fn richtext_apply_delta_marks_without_growth() { + let loro = LoroDoc::new_auto_commit(); + let text = loro.get_text("text"); + text.insert(0, "abc", PosType::Unicode).unwrap(); + + text.apply_delta(&[TextDelta::Retain { + retain: 3, + attributes: Some(fx_map!("bold".into() => LoroValue::Bool(true))), + }]) + .unwrap(); + loro.commit_then_renew(); + + assert_eq!(text.to_string(), "abc"); + assert_eq!( + text.get_richtext_value().to_json_value(), + json!([{"insert": "abc", "attributes": {"bold": true}}]) + ); + } + + #[test] + fn richtext_apply_delta_grows_for_mark_gap() { + let loro = LoroDoc::new_auto_commit(); + let text = loro.get_text("text"); + + text.apply_delta(&[TextDelta::Retain { + retain: 1, + attributes: Some(fx_map!("bold".into() => LoroValue::Bool(true))), + }]) + .unwrap(); + loro.commit_then_renew(); + + assert_eq!(text.to_string(), "\n"); + assert_eq!( + text.get_richtext_value().to_json_value(), + json!([{"insert": "\n", "attributes": {"bold": true}}]) + ); + } + + #[test] + fn richtext_apply_delta_ignores_empty_inserts() { + let loro = LoroDoc::new_auto_commit(); + let text = loro.get_text("text"); + text.insert(0, "seed", PosType::Unicode).unwrap(); + + text.apply_delta(&[TextDelta::Insert { + insert: "".into(), + attributes: Some(fx_map!("bold".into() => LoroValue::Bool(true))), + }]) + .unwrap(); + loro.commit_then_renew(); + + assert_eq!(text.to_string(), "seed"); + assert_eq!( + text.get_richtext_value().to_json_value(), + json!([{"insert": "seed"}]) + ); + } + + #[test] + fn handler_trait_dispatch_reports_attached_container_identity() { + let loro = LoroDoc::new_auto_commit(); + let handlers = [ + (loro.get_text("text").to_handler(), ContainerType::Text), + (loro.get_map("map").to_handler(), ContainerType::Map), + (loro.get_list("list").to_handler(), ContainerType::List), + ( + loro.get_movable_list("movable").to_handler(), + ContainerType::MovableList, + ), + (loro.get_tree("tree").to_handler(), ContainerType::Tree), + ]; + + for (handler, expected_type) in handlers { + assert!(handler.is_attached()); + assert!(handler.attached_handler().is_some()); + assert!(handler.doc().is_some()); + assert!(handler.get_attached().is_some()); + assert_eq!(handler.kind(), expected_type); + assert_eq!(handler.c_type(), expected_type); + assert_eq!(handler.id().container_type(), expected_type); + assert_eq!( + Handler::from_handler(handler.clone()).unwrap().c_type(), + expected_type + ); + + handler.get_value(); + handler.get_deep_value(); + handler.clear().unwrap(); + } + } + + #[test] + fn handler_trait_dispatch_reports_detached_container_identity() { + let handlers = [ + ( + Handler::new_unattached(ContainerType::Text), + ContainerType::Text, + ), + ( + Handler::new_unattached(ContainerType::Map), + ContainerType::Map, + ), + ( + Handler::new_unattached(ContainerType::List), + ContainerType::List, + ), + ( + Handler::new_unattached(ContainerType::MovableList), + ContainerType::MovableList, + ), + ( + Handler::new_unattached(ContainerType::Tree), + ContainerType::Tree, + ), + ]; + + for (handler, expected_type) in handlers { + assert!(!handler.is_attached()); + assert!(handler.attached_handler().is_none()); + assert!(handler.doc().is_none()); + assert!(handler.get_attached().is_none()); + assert_eq!(handler.kind(), expected_type); + assert_eq!(handler.c_type(), expected_type); + assert_eq!(handler.id().container_type(), expected_type); + assert_eq!(handler.idx().get_type(), expected_type); + assert_eq!( + Handler::from_handler(handler.clone()).unwrap().c_type(), + expected_type + ); + } + } + + #[test] + fn attaching_detached_handlers_sets_parent_and_attached_back_reference() { + let loro = LoroDoc::new_auto_commit(); + + let map = loro.get_map("map"); + let detached_text = TextHandler::new_detached(); + detached_text + .insert(0, "detached", PosType::Unicode) + .unwrap(); + let attached_text = map.insert_container("text", detached_text.clone()).unwrap(); + assert!(attached_text.is_attached()); + assert_eq!(attached_text.to_string(), "detached"); + assert_eq!(attached_text.parent().unwrap().c_type(), ContainerType::Map); + assert_eq!( + detached_text.get_attached().unwrap().id(), + attached_text.id() + ); + + let list = loro.get_list("list"); + let detached_map = MapHandler::new_detached(); + detached_map.insert("k", 1_i64).unwrap(); + let attached_map = list.insert_container(0, detached_map.clone()).unwrap(); + assert!(attached_map.is_attached()); + assert_eq!(attached_map.parent().unwrap().c_type(), ContainerType::List); + assert_eq!(detached_map.get_attached().unwrap().id(), attached_map.id()); + + let movable = loro.get_movable_list("movable"); + let detached_list = ListHandler::new_detached(); + detached_list.push("item").unwrap(); + let attached_list = movable.insert_container(0, detached_list.clone()).unwrap(); + assert!(attached_list.is_attached()); + assert_eq!( + attached_list.parent().unwrap().c_type(), + ContainerType::MovableList + ); + assert_eq!( + detached_list.get_attached().unwrap().id(), + attached_list.id() + ); + + let nested = attached_map + .insert_container("movable", MovableListHandler::new_detached()) + .unwrap(); + assert_eq!(nested.parent().unwrap().id(), attached_map.id()); + } + + #[test] + fn unknown_handler_reports_identity_without_materializing_value() { + let loro = LoroDoc::new_auto_commit(); + let id = ContainerID::Root { + name: "unknown".into(), + container_type: ContainerType::Unknown(7), + }; + let handler = Handler::new_attached(id.clone(), loro.clone()); + let unknown = handler.as_unknown().unwrap(); + + assert!(unknown.is_attached()); + assert_eq!(unknown.kind(), ContainerType::Unknown(7)); + assert_eq!(unknown.id(), id); + assert_eq!(unknown.to_handler().c_type(), ContainerType::Unknown(7)); + assert!(unknown.doc().is_some()); + assert!(!unknown.is_deleted()); + assert_eq!(format!("{unknown:?}"), "UnknownHandler"); + assert!(unknown.get_attached().is_some()); + assert!(super::UnknownHandler::from_handler(handler).is_some()); + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/handler/movable_list_apply_delta.rs b/3rdparty/loro/crates/loro-internal/src/handler/movable_list_apply_delta.rs new file mode 100644 index 0000000000..5e678af8a2 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/handler/movable_list_apply_delta.rs @@ -0,0 +1,375 @@ +use super::*; + +#[derive(Debug)] +struct ReplacementContext<'a> { + index: &'a mut usize, + index_shift: &'a mut usize, + to_delete: &'a mut FxHashMap, + container_remap: &'a mut FxHashMap, + deleted_indices: &'a mut Vec, + next_deleted: &'a mut BinaryHeap>, +} + +impl MovableListHandler { + /// Applies a delta to the movable list handler. + /// + /// This function processes the given delta, performing the necessary insertions, + /// deletions, and moves to update the list accordingly. It handles container elements, + /// maintains a map for remapping container IDs, and ensures proper indexing throughout + /// the operation. + /// + /// # Arguments + /// + /// * `delta` - A delta representing the changes to apply. + /// * `container_remap` - A map used to remap container IDs during the operation. + /// + /// # Returns + /// + /// * `LoroResult<()>` - Returns `Ok(())` if successful, or an error if something goes wrong. + #[tracing::instrument(level = "debug", skip_all)] + pub fn apply_delta( + &self, + delta: loro_delta::DeltaRope< + loro_delta::array_vec::ArrayVec, + crate::event::ListDeltaMeta, + >, + container_remap: &mut FxHashMap, + ) -> LoroResult<()> { + { + // Test whether the delta is valid + let len = self.len(); + let mut index = 0; + for delta_item in delta.iter() { + match delta_item { + loro_delta::DeltaItem::Retain { len, .. } => { + index += *len; + } + loro_delta::DeltaItem::Replace { delete, .. } => { + index += *delete; + } + } + + if index > len { + return Err(LoroError::OutOfBound { + pos: index, + len, + info: "apply_delta".into(), + }); + } + } + } + + match &self.inner { + MaybeDetached::Detached(_) => { + unimplemented!(); + } + MaybeDetached::Attached(_) => { + // use tracing::debug; + // debug!( + // "Movable list value before apply_delta: {:#?}", + // self.get_deep_value_with_id() + // ); + // debug!("Applying delta: {:#?}", &delta); + + // Preprocess deletions to build a map of containers to delete. + let mut to_delete = self.preprocess_deletions(&delta); + // Process insertions and moves. + let mut index = 0; + let mut index_shift = 0; + let mut deleted_indices = Vec::new(); + let mut next_deleted = BinaryHeap::new(); + // - positive values are retain + // - negative values are deletions + let mut delta_change: Vec = Vec::new(); + + for delta_item in delta.iter() { + match delta_item { + loro_delta::DeltaItem::Retain { len, .. } => { + index += len; + delta_change.push(*len as isize); + } + loro_delta::DeltaItem::Replace { + value, + delete, + attr, + } => { + // Handle deletions in the current replace operation. + let old_index = index; + self.handle_deletions_in_replace( + *delete, + &mut index, + index_shift, + &mut next_deleted, + ); + delta_change.push(-((index - old_index) as isize)); + + // Process the insertions and moves. + let mut context = ReplacementContext { + index: &mut index, + index_shift: &mut index_shift, + to_delete: &mut to_delete, + container_remap, + deleted_indices: &mut deleted_indices, + next_deleted: &mut next_deleted, + }; + + self.process_replacements(value, attr, &mut context) + .unwrap(); + delta_change.push(value.len() as isize); + } + } + } + + // Apply any remaining deletions. + self.apply_remaining_deletions(delta_change, &mut deleted_indices)?; + Ok(()) + } + } + } + + /// Preprocess deletions to build a map of containers to delete. + /// + /// # Arguments + /// + /// * `delta` - The delta containing the deletions. + /// + /// # Returns + /// + /// * `FxHashMap` - A map of containers to their indices that need to be deleted. + fn preprocess_deletions( + &self, + delta: &loro_delta::DeltaRope< + loro_delta::array_vec::ArrayVec, + crate::event::ListDeltaMeta, + >, + ) -> FxHashMap { + let mut index = 0; + let mut to_delete = FxHashMap::default(); + + for delta_item in delta.iter() { + match delta_item { + loro_delta::DeltaItem::Retain { len, .. } => { + index += len; + } + loro_delta::DeltaItem::Replace { delete, .. } => { + if *delete > 0 { + for i in index..index + *delete { + if let Some(LoroValue::Container(c)) = self.get(i) { + to_delete.insert(c, i); + } + } + index += *delete; + } + } + } + } + + to_delete + } + + /// Handles deletions' effect on the index within a replace operation. + /// It will not perform the deletions. + /// + /// # Arguments + /// + /// * `delete_len` - The number of deletions. + /// * `index` - The current index in the list. + /// * `index_shift` - The current index shift due to previous operations. + /// * `next_deleted` - A heap of indices scheduled for deletion. + fn handle_deletions_in_replace( + &self, + delete_len: usize, + index: &mut usize, + index_shift: usize, + next_deleted: &mut BinaryHeap>, + ) { + if delete_len > 0 { + let mut remaining_deletes = delete_len; + while let Some(Reverse(old_index)) = next_deleted.peek() { + if *old_index + index_shift < *index + remaining_deletes { + assert!(*index <= *old_index + index_shift); + assert!(remaining_deletes > 0); + next_deleted.pop(); + remaining_deletes -= 1; + } else { + break; + } + } + + // Increase the index by the number of deletions handled. + *index += remaining_deletes; + } + } + + /// Processes replacements, handling insertions and moves. + /// + /// # Arguments + /// + /// * `values` - The values to insert or move. + /// * `attr` - Additional attributes for the delta item. + /// * `context` - A context struct containing related parameters. + fn process_replacements( + &self, + values: &loro_delta::array_vec::ArrayVec, + attr: &crate::event::ListDeltaMeta, + context: &mut ReplacementContext, + ) -> LoroResult<()> { + for v in values.iter() { + match v { + ValueOrHandler::Value(LoroValue::Container(old_id)) => { + self.apply_insertion(attr, context, old_id.clone())?; + } + ValueOrHandler::Handler(handler) => { + let old_id = handler.id(); + self.apply_insertion(attr, context, old_id)?; + } + ValueOrHandler::Value(value) => { + self.insert(*context.index, value.clone())?; + Self::update_positions_on_insert(context.to_delete, *context.index, 1); + *context.index += 1; + *context.index_shift += 1; + } + } + } + + Ok(()) + } + + fn apply_insertion( + &self, + attr: &crate::event::ListDeltaMeta, + context: &mut ReplacementContext<'_>, + mut old_id: ContainerID, + ) -> Result<(), LoroError> { + if !context.to_delete.contains_key(&old_id) { + while let Some(new_id) = context.container_remap.get(&old_id) { + old_id = new_id.clone(); + if context.to_delete.contains_key(&old_id) { + break; + } + } + } + if let Some(old_index) = context.to_delete.remove(&old_id) { + if old_index > *context.index { + ensure_cov::notify_cov( + "loro_internal::handler::movable_list_apply_delta::process_replacements::mov_0", + ); + self.mov(old_index, *context.index)?; + context.next_deleted.push(Reverse(old_index)); + *context.index += 1; + *context.index_shift += 1; + } else { + ensure_cov::notify_cov( + "loro_internal::handler::movable_list_apply_delta::process_replacements::mov_1", + ); + self.mov(old_index, *context.index - 1)?; + } + context.deleted_indices.push(old_index); + Self::update_positions_on_delete(context.to_delete, old_index); + Self::update_positions_on_insert(context.to_delete, *context.index, 1); + } else if !attr.from_move || !self.contains_container(&old_id) { + // Insert a new container if not moved. + let new_handler = self.insert_container( + *context.index, + Handler::new_unattached(old_id.container_type()), + )?; + let new_id = new_handler.id(); + context.container_remap.insert(old_id, new_id); + Self::update_positions_on_insert(context.to_delete, *context.index, 1); + *context.index += 1; + *context.index_shift += 1; + } + Ok(()) + } + + fn contains_container(&self, id: &ContainerID) -> bool { + (0..self.len()).any(|index| { + matches!( + self.get(index), + Some(LoroValue::Container(container_id)) if container_id == *id + ) + }) + } + + /// Applies any remaining deletions after processing insertions and moves. + /// + /// # Arguments + /// + /// * `delta` - The delta containing the deletions. + /// * `deleted_indices` - A list of indices that have been deleted. + fn apply_remaining_deletions( + &self, + delta: Vec, + deleted_indices: &mut Vec, + ) -> LoroResult<()> { + // Sort deleted indices from largest to smallest. + deleted_indices.sort_by_key(|&x| std::cmp::Reverse(x)); + + let mut index: usize = 0; + for delta_item in delta.iter() { + match *delta_item { + x if x > 0 => { + index += x as usize; + } + neg_delete => { + let delete = neg_delete.unsigned_abs(); + let mut remaining_deletes = delete; + while let Some(&last) = deleted_indices.last() { + if last < index { + deleted_indices.pop(); + continue; + } + + if last < index + remaining_deletes { + deleted_indices.pop(); + remaining_deletes -= 1; + } else { + break; + } + } + + self.delete(index, remaining_deletes)?; + } + } + } + + Ok(()) + } + + /// Updates positions in the map after an insertion. + /// + /// Increments positions that are greater than or equal to the insertion index. + /// + /// # Arguments + /// + /// * `positions` - The map of positions to update. + /// * `index` - The index where the insertion occurred. + /// * `len` - The length of the insertion. + fn update_positions_on_insert( + positions: &mut FxHashMap, + index: usize, + len: usize, + ) { + for pos in positions.values_mut() { + if *pos >= index { + *pos += len; + } + } + } + + /// Updates positions in the map after a deletion. + /// + /// Decrements positions that are greater than or equal to the deletion index. + /// + /// # Arguments + /// + /// * `positions` - The map of positions to update. + /// * `index` - The index where the deletion occurred. + fn update_positions_on_delete(positions: &mut FxHashMap, index: usize) { + for pos in positions.values_mut() { + if *pos >= index { + *pos -= 1; + } + } + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/handler/text_update.rs b/3rdparty/loro/crates/loro-internal/src/handler/text_update.rs new file mode 100644 index 0000000000..6202262d4b --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/handler/text_update.rs @@ -0,0 +1,149 @@ +use std::{char, sync::Arc}; + +use itertools::Itertools; +use rustc_hash::FxHashMap; + +use crate::diff::DiffHandler; + +use super::TextHandler; + +pub(super) struct DiffHook<'a> { + text: &'a TextHandler, + new: &'a [u32], + last_old_index: usize, + current_index: usize, +} + +impl<'a> DiffHook<'a> { + pub(crate) fn new(text: &'a TextHandler, new: &'a [u32]) -> Self { + Self { + text, + new, + last_old_index: 0, + current_index: 0, + } + } +} + +impl DiffHandler for DiffHook<'_> { + fn insert(&mut self, old_index: usize, new_index: usize, new_len: usize) { + if old_index > self.last_old_index { + self.current_index += old_index - self.last_old_index; + self.last_old_index = old_index; + } + + self.text + .insert_unicode( + self.current_index, + &self.new[new_index..new_index + new_len] + .iter() + .map(|x| char::from_u32(*x).unwrap()) + .collect::(), + ) + .unwrap(); + self.current_index += new_len; + } + + fn delete(&mut self, old_index: usize, old_len: usize) { + self.current_index += old_index - self.last_old_index; + self.text + .delete_unicode(self.current_index, old_len) + .unwrap(); + self.last_old_index = old_index + old_len; + } +} + +pub(super) struct DiffHookForLine<'a> { + text: &'a TextHandler, + old: Vec, + new: Vec, + lines: Vec>, + lines_lookup: FxHashMap, usize>, + + last_old_index: usize, + current_index: usize, +} + +impl<'a> DiffHookForLine<'a> { + pub(crate) fn new(text: &'a TextHandler, new_str: &str) -> Self { + let mut this = Self { + text, + old: Vec::new(), + new: Vec::new(), + lines: Vec::new(), + lines_lookup: FxHashMap::default(), + last_old_index: 0, + current_index: 0, + }; + + let text_str = text.to_string(); + for line in text_str.split_inclusive('\n') { + let line: Arc = Arc::from(line); + let id = this.register_line(line); + this.old.push(id as u32); + } + + for line in new_str.split_inclusive('\n') { + let line: Arc = Arc::from(line); + let id = this.register_line(line); + this.new.push(id as u32); + } + + this + } + + fn register_line(&mut self, line: Arc) -> usize { + if let Some(&index) = self.lines_lookup.get(&line) { + return index; + } + + self.lines.push(line.clone()); + self.lines_lookup.insert(line, self.lines.len() - 1); + self.lines.len() - 1 + } + + pub fn get_old_arr(&self) -> &[u32] { + &self.old + } + + pub fn get_new_arr(&self) -> &[u32] { + &self.new + } +} + +impl DiffHandler for DiffHookForLine<'_> { + fn insert(&mut self, old_index: usize, new_index: usize, new_len: usize) { + if self.last_old_index < old_index { + assert!(self.last_old_index < old_index); + self.current_index += (self.last_old_index..old_index) + .map(|x| self.lines[self.old[x] as usize].chars().count()) + .sum::(); + self.last_old_index = old_index; + } + + let s = self.new[new_index..new_index + new_len] + .iter() + .map(|x| self.lines[*x as usize].clone()) + .join(""); + self.text.insert_unicode(self.current_index, &s).unwrap(); + self.current_index += s.chars().count(); + } + + fn delete(&mut self, old_index: usize, old_len: usize) { + if self.last_old_index != old_index { + assert!(self.last_old_index < old_index); + self.current_index += (self.last_old_index..old_index) + .map(|x| self.lines[self.old[x] as usize].chars().count()) + .sum::(); + } + + self.last_old_index = old_index + old_len; + let delete_len = (old_index..old_index + old_len) + .map(|x| self.lines[self.old[x] as usize].chars().count()) + .sum::(); + + self.text + .delete_unicode(self.current_index, delete_len) + .unwrap(); + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/handler/tree.rs b/3rdparty/loro/crates/loro-internal/src/handler/tree.rs new file mode 100644 index 0000000000..ca2853bf68 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/handler/tree.rs @@ -0,0 +1,1184 @@ +use std::{collections::VecDeque, sync::Arc}; + +use fractional_index::FractionalIndex; +use loro_common::{ + ContainerID, ContainerType, Counter, IdFull, IdLp, LoroError, LoroResult, LoroTreeError, + LoroValue, PeerID, TreeID, ID, +}; +use rustc_hash::FxHashMap; +use smallvec::smallvec; + +use crate::{ + container::tree::tree_op::TreeOp, + delta::{TreeDiffItem, TreeExternalDiff}, + state::{ + FiIfNotConfigured, FractionalIndexGenResult, NodePosition, TreeNode, TreeNodeWithChildren, + TreeParentId, + }, + txn::{EventHint, Transaction}, + BasicHandler, HandlerTrait, MapHandler, +}; + +use super::{create_handler, Handler, MaybeDetached}; + +#[derive(Clone)] +pub struct TreeHandler { + pub(super) inner: MaybeDetached, +} + +#[derive(Clone)] +pub(super) struct TreeInner { + next_counter: Counter, + map: FxHashMap, + parent_links: FxHashMap>, + children_links: FxHashMap, Vec>, +} + +impl TreeInner { + fn new() -> Self { + TreeInner { + next_counter: 0, + map: FxHashMap::default(), + parent_links: FxHashMap::default(), + children_links: FxHashMap::default(), + } + } + + fn create(&mut self, parent: Option, index: usize) -> TreeID { + let id = TreeID::new(PeerID::MAX, self.next_counter); + self.next_counter += 1; + self.map.insert(id, MapHandler::new_detached()); + self.parent_links.insert(id, parent); + let children = self.children_links.entry(parent).or_default(); + children.insert(index, id); + id + } + + fn mov(&mut self, target: TreeID, new_parent: Option, index: usize) -> LoroResult<()> { + let old_parent = self + .parent_links + .get_mut(&target) + .ok_or(LoroTreeError::TreeNodeNotExist(target))?; + let children = self.children_links.get_mut(old_parent).unwrap(); + children.retain(|x| x != &target); + self.parent_links.insert(target, new_parent); + let children = self.children_links.entry(new_parent).or_default(); + children.insert(index, target); + Ok(()) + } + + fn delete(&mut self, id: TreeID) -> LoroResult<()> { + self.map.remove(&id); + let parent = self + .parent_links + .remove(&id) + .ok_or(LoroTreeError::TreeNodeNotExist(id))?; + let children = self.children_links.get_mut(&parent).unwrap(); + children.retain(|x| x != &id); + self.children_links.remove(&Some(id)); + Ok(()) + } + + fn get_id_by_index(&self, parent: &Option, index: usize) -> Option { + self.children_links + .get(parent) + .and_then(|x| x.get(index).cloned()) + } + + fn get_parent(&self, id: &TreeID) -> Option> { + self.parent_links.get(id).cloned() + } + + fn get_children(&self, parent: Option) -> Option> { + self.children_links.get(&parent).cloned() + } + + fn children_num(&self, parent: Option) -> Option { + self.children_links.get(&parent).map(|x| x.len()) + } + + fn is_parent(&self, target: &TreeID, parent: &Option) -> bool { + self.parent_links.get(target) == Some(parent) + } + + fn get_index_by_tree_id(&self, target: &TreeID) -> Option { + self.parent_links + .get(target) + .and_then(|parent| self.children_links.get(parent)) + .and_then(|children| children.iter().position(|x| x == target)) + } + + fn get_value(&self, deep: bool) -> LoroValue { + let mut ans = vec![]; + + let empty_vec = vec![]; + let mut q = VecDeque::from_iter( + self.children_links + .get(&None) + .unwrap_or(&empty_vec) + .iter() + .enumerate() + .zip(std::iter::repeat(None::)), + ); + + while let Some(((idx, target), parent)) = q.pop_front() { + let map = self.map.get(target).unwrap(); + let mut loro_map_value = FxHashMap::default(); + loro_map_value.insert("id".to_string(), target.to_string().into()); + let parent = parent + .map(|p| p.to_string().into()) + .unwrap_or(LoroValue::Null); + loro_map_value.insert("parent".to_string(), parent); + loro_map_value.insert( + "meta".to_string(), + if deep { + map.get_deep_value() + } else { + String::from("UnResolved").into() + }, + ); + loro_map_value.insert("index".to_string(), (idx as i64).into()); + ans.push(loro_map_value); + if let Some(children) = self.children_links.get(&Some(*target)) { + for (idx, child) in children.iter().enumerate() { + q.push_back(((idx, child), Some(*target))); + } + } + } + ans.into() + } + + fn get_nodes_under(&self, root: TreeParentId) -> Vec { + let root_id = root.tree_id(); + let mut ans = vec![]; + let empty_vec = vec![]; + let mut q = VecDeque::from_iter( + self.children_links + .get(&root_id) + .unwrap_or(&empty_vec) + .iter() + .enumerate() + .zip(std::iter::repeat(root)), + ); + + while let Some(((index, target), parent)) = q.pop_front() { + ans.push(TreeNode { + id: *target, + parent, + fractional_index: FractionalIndex::default(), + index, + last_move_op: IdFull { + peer: target.peer, + lamport: 0, + counter: target.counter, + }, + }); + if let Some(children) = self.children_links.get(&Some(*target)) { + let parent = TreeParentId::Node(*target); + q.extend( + children + .iter() + .enumerate() + .map(|(index, target)| ((index, target), parent)), + ); + } + } + + ans + } +} + +impl HandlerTrait for TreeHandler { + fn to_handler(&self) -> Handler { + Handler::Tree(self.clone()) + } + + fn attach( + &self, + txn: &mut Transaction, + parent: &BasicHandler, + self_id: ContainerID, + ) -> LoroResult { + match &self.inner { + MaybeDetached::Detached(t) => { + let mut t = t.lock(); + let inner = create_handler(parent, self_id); + let tree = inner.into_tree().unwrap(); + + let children = t.value.children_links.get(&None); + let mut q = children + .map(|c| { + VecDeque::from_iter( + c.iter() + .enumerate() + .zip(std::iter::repeat(TreeParentId::Root)), + ) + }) + .unwrap_or_default(); + while let Some(((idx, target), parent)) = q.pop_front() { + let real_id = + tree.create_with_txn(txn, parent, idx, FiIfNotConfigured::UseJitterZero)?; + let map = t.value.map.get(target).unwrap(); + map.attach( + txn, + tree.inner.try_attached_state()?, + real_id.associated_meta_container(), + )?; + + if let Some(children) = t.value.children_links.get(&Some(*target)) { + for (idx, child) in children.iter().enumerate() { + q.push_back(((idx, child), TreeParentId::Node(real_id))); + } + } + } + t.attached = tree.attached_handler().cloned(); + Ok(tree) + } + MaybeDetached::Attached(a) => { + let new_inner = create_handler(a, self_id); + let ans = new_inner.into_tree().unwrap(); + + fn attach_nodes( + source: &TreeHandler, + target: &TreeHandler, + txn: &mut Transaction, + parent: TreeParentId, + nodes: Vec, + ) -> LoroResult<()> { + for node in nodes { + let real_id = target.create_with_txn( + txn, + parent, + node.index, + FiIfNotConfigured::UseJitterZero, + )?; + source.get_meta(node.id)?.attach( + txn, + target.inner.try_attached_state()?, + real_id.associated_meta_container(), + )?; + attach_nodes( + source, + target, + txn, + TreeParentId::Node(real_id), + node.children, + )?; + } + + Ok(()) + } + + let tree_nodes = self.get_all_hierarchy_nodes_under(TreeParentId::Root); + attach_nodes(self, &ans, txn, TreeParentId::Root, tree_nodes)?; + Ok(ans) + } + } + } + + fn is_attached(&self) -> bool { + self.inner.is_attached() + } + + fn attached_handler(&self) -> Option<&BasicHandler> { + self.inner.attached_handler() + } + + fn get_value(&self) -> LoroValue { + match &self.inner { + MaybeDetached::Detached(t) => { + let t = t.lock(); + t.value.get_value(false) + } + MaybeDetached::Attached(a) => a.get_value(), + } + } + + fn get_deep_value(&self) -> LoroValue { + match &self.inner { + MaybeDetached::Detached(t) => { + let t = t.lock(); + t.value.get_value(true) + } + MaybeDetached::Attached(a) => a.get_deep_value(), + } + } + + fn kind(&self) -> ContainerType { + ContainerType::Tree + } + + fn get_attached(&self) -> Option { + match &self.inner { + MaybeDetached::Detached(d) => d.lock().attached.clone().map(|x| Self { + inner: MaybeDetached::Attached(x), + }), + MaybeDetached::Attached(_a) => Some(self.clone()), + } + } + + fn from_handler(h: Handler) -> Option { + match h { + Handler::Tree(x) => Some(x), + _ => None, + } + } + + fn doc(&self) -> Option { + match &self.inner { + MaybeDetached::Detached(_) => None, + MaybeDetached::Attached(a) => Some(a.doc()), + } + } +} + +impl std::fmt::Debug for TreeHandler { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match &self.inner { + MaybeDetached::Detached(_) => write!(f, "TreeHandler Detached"), + MaybeDetached::Attached(a) => write!(f, "TreeHandler {}", a.id), + } + } +} + +impl TreeHandler { + /// Create a new container that is detached from the document. + /// + /// The edits on a detached container will not be persisted/synced. + /// To attach the container to the document, please insert it into an attached + /// container. + pub fn new_detached() -> Self { + Self { + inner: MaybeDetached::new_detached(TreeInner::new()), + } + } + + pub fn delete(&self, target: TreeID) -> LoroResult<()> { + match &self.inner { + MaybeDetached::Detached(t) => { + let mut t = t.lock(); + t.value.delete(target)?; + Ok(()) + } + MaybeDetached::Attached(a) => a.with_txn(|txn| self.delete_with_txn(txn, target)), + } + } + + pub(crate) fn delete_with_txn(&self, txn: &mut Transaction, target: TreeID) -> LoroResult<()> { + let inner = self.inner.try_attached_state()?; + let index = match self.get_index_by_tree_id(&target) { + Some(i) => i, + None => { + return Err(LoroTreeError::TreeNodeDeletedOrNotExist(target).into()); + } + }; + txn.apply_local_op( + inner.container_idx, + crate::op::RawOpContent::Tree(Arc::new(TreeOp::Delete { target })), + EventHint::Tree(smallvec![TreeDiffItem { + target, + action: TreeExternalDiff::Delete { + old_parent: self + .get_node_parent(&target) + .ok_or(LoroTreeError::TreeNodeDeletedOrNotExist(target))?, + old_index: index + }, + }]), + &inner.doc, + ) + } + + pub fn create(&self, parent: TreeParentId) -> LoroResult { + match parent { + TreeParentId::Deleted | TreeParentId::Unexist => { + return Err(LoroTreeError::InvalidParent.into()); + } + _ => {} + } + let index: usize = self.children_num(&parent).unwrap_or(0); + match &self.inner { + MaybeDetached::Detached(t) => { + let t = &mut t.lock().value; + Ok(t.create(parent.tree_id(), index)) + } + MaybeDetached::Attached(a) => { + a.with_txn(|txn| self.create_with_txn(txn, parent, index, FiIfNotConfigured::Zero)) + } + } + } + + pub fn create_at(&self, parent: TreeParentId, index: usize) -> LoroResult { + match parent { + TreeParentId::Deleted | TreeParentId::Unexist => { + return Err(LoroTreeError::InvalidParent.into()); + } + _ => {} + } + let children_len = self.children_num(&parent).unwrap_or(0); + if index > children_len { + return Err(LoroTreeError::IndexOutOfBound { + len: children_len, + index, + } + .into()); + } + match &self.inner { + MaybeDetached::Detached(t) => { + let t = &mut t.lock().value; + Ok(t.create(parent.tree_id(), index)) + } + MaybeDetached::Attached(a) => { + a.with_txn(|txn| self.create_with_txn(txn, parent, index, FiIfNotConfigured::Throw)) + } + } + } + + /// For undo/redo, Specify the TreeID of the created node + pub(crate) fn create_at_with_target_for_apply_diff( + &self, + parent: TreeParentId, + position: FractionalIndex, + target: TreeID, + ) -> LoroResult { + let MaybeDetached::Attached(a) = &self.inner else { + unreachable!(); + }; + + if let Some(p) = self.get_node_parent(&target) { + if p == parent { + return Ok(false); + // If parent is deleted, we need to create the node, so this op from move_apply_diff + } + match p { + TreeParentId::Node(p) => { + if !self.is_node_unexist(&target) && !self.is_node_deleted(&p)? { + return self.move_at_with_target_for_apply_diff(parent, position, target); + } + } + TreeParentId::Root => { + return self.move_at_with_target_for_apply_diff(parent, position, target); + } + TreeParentId::Deleted | TreeParentId::Unexist => {} + } + } + + let with_event = !parent + .tree_id() + .is_some_and(|p| self.is_node_deleted(&p).unwrap()); + if !with_event { + return Ok(false); + } + + // println!( + // "create_at_with_target_for_apply_diff: {:?} {:?}", + // target, parent + // ); + + let index = self + .get_index_by_fractional_index( + &parent, + &NodePosition { + position: position.clone(), + idlp: self.next_idlp(), + }, + ) + // TODO: parent has deleted? + .unwrap_or(0); + + let children = a.with_txn(|txn| { + let inner = self.inner.try_attached_state()?; + + txn.apply_local_op( + inner.container_idx, + crate::op::RawOpContent::Tree(Arc::new(TreeOp::Create { + target, + parent: parent.tree_id(), + position: position.clone(), + })), + EventHint::Tree(smallvec![TreeDiffItem { + target, + action: TreeExternalDiff::Create { + parent, + index, + position: position.clone(), + }, + }]), + &inner.doc, + )?; + + Ok(self + .children(&TreeParentId::Node(target)) + .unwrap_or_default()) + })?; + for child in children { + let position = self.get_position_by_tree_id(&child).unwrap(); + self.create_at_with_target_for_apply_diff(TreeParentId::Node(target), position, child)?; + } + Ok(true) + } + + /// For undo/redo, Specify the TreeID of the created node + pub(crate) fn move_at_with_target_for_apply_diff( + &self, + parent: TreeParentId, + position: FractionalIndex, + target: TreeID, + ) -> LoroResult { + let MaybeDetached::Attached(a) = &self.inner else { + unreachable!(); + }; + + // // the move node does not exist, create it + // if self.is_node_unexist(&target) || self.is_node_deleted(&target).unwrap() { + // return self.create_at_with_target_for_apply_diff(parent, position, target); + // } + + if let (Some(p), Some(fi)) = ( + self.get_node_parent(&target), + self.get_position_by_tree_id(&target), + ) { + if p == parent && position == fi { + return Ok(false); + } + } + + let old_parent = self.get_node_parent(&target).unwrap(); + let old_index = self.get_index_by_tree_id(&target).unwrap(); + let mut index = self + .get_index_by_fractional_index( + &parent, + &NodePosition { + position: position.clone(), + idlp: self.next_idlp(), + }, + ) + .unwrap_or(0); + if old_parent == parent && old_index < index { + index -= 1; + } + let with_event = match parent.tree_id() { + Some(p) => !self.is_node_deleted(&p)?, + None => true, + }; + + if !with_event { + return Ok(false); + } + + // println!( + // "move_at_with_target_for_apply_diff: {:?} {:?}", + // target, parent + // ); + + a.with_txn(|txn| { + let inner = self.inner.try_attached_state()?; + txn.apply_local_op( + inner.container_idx, + crate::op::RawOpContent::Tree(Arc::new(TreeOp::Move { + target, + parent: parent.tree_id(), + position: position.clone(), + })), + EventHint::Tree(smallvec![TreeDiffItem { + target, + action: TreeExternalDiff::Move { + parent, + index, + position: position.clone(), + // the old parent should be exist, so we can unwrap + old_parent, + old_index, + }, + }]), + &inner.doc, + ) + })?; + Ok(true) + } + + pub(crate) fn create_with_txn( + &self, + txn: &mut Transaction, + parent: TreeParentId, + index: usize, + cfg: FiIfNotConfigured, + ) -> LoroResult { + let inner = self.inner.try_attached_state()?; + let target = TreeID::from_id(txn.next_id()); + + match self.generate_position_at(&target, &parent, index, cfg) { + FractionalIndexGenResult::Ok(position) => { + self.create_with_position(inner, txn, target, parent, index, position) + } + FractionalIndexGenResult::Rearrange(ids) => { + for (i, (id, position)) in ids.into_iter().enumerate() { + if i == 0 { + self.create_with_position(inner, txn, id, parent, index, position)?; + continue; + } + self.mov_with_position(inner, txn, id, parent, index + i, position, index + i)?; + } + Ok(target) + } + FractionalIndexGenResult::NotConfigured => { + Err(LoroTreeError::FractionalIndexNotEnabled.into()) + } + } + } + + pub fn mov(&self, target: TreeID, parent: TreeParentId) -> LoroResult<()> { + match &self.inner { + MaybeDetached::Detached(_) => { + let mut index: usize = self.children_num(&parent).unwrap_or(0); + if self.is_parent(&target, &parent) { + index -= 1; + } + self.move_to(target, parent, index) + } + MaybeDetached::Attached(a) => { + let mut index: usize = self.children_num(&parent).unwrap_or(0); + if self.is_parent(&target, &parent) { + index -= 1; + } + a.with_txn(|txn| { + self.mov_with_txn(txn, target, parent, index, FiIfNotConfigured::Zero) + }) + } + } + } + + pub fn mov_after(&self, target: TreeID, other: TreeID) -> LoroResult<()> { + let parent = self + .get_node_parent(&other) + .ok_or(LoroTreeError::TreeNodeNotExist(other))?; + let mut index = self + .get_index_by_tree_id(&other) + .ok_or(LoroTreeError::TreeNodeDeletedOrNotExist(other))? + + 1; + if self.is_parent(&target, &parent) { + if let Some(target_index) = self.get_index_by_tree_id(&target) { + if target_index < index { + index -= 1; + } + } + } + self.move_to(target, parent, index) + } + + pub fn mov_before(&self, target: TreeID, other: TreeID) -> LoroResult<()> { + let parent = self + .get_node_parent(&other) + .ok_or(LoroTreeError::TreeNodeNotExist(other))?; + let mut index = self + .get_index_by_tree_id(&other) + .ok_or(LoroTreeError::TreeNodeDeletedOrNotExist(other))?; + if self.is_parent(&target, &parent) && index >= 1 { + if let Some(target_index) = self.get_index_by_tree_id(&target) { + if target_index < index { + index -= 1; + } + } + } + self.move_to(target, parent, index) + } + + pub fn move_to(&self, target: TreeID, parent: TreeParentId, index: usize) -> LoroResult<()> { + match &self.inner { + MaybeDetached::Detached(t) => { + let mut t = t.lock(); + t.value.mov(target, parent.tree_id(), index) + } + MaybeDetached::Attached(a) => a.with_txn(|txn| { + self.mov_with_txn(txn, target, parent, index, FiIfNotConfigured::Throw) + }), + } + } + + pub(crate) fn mov_with_txn( + &self, + txn: &mut Transaction, + target: TreeID, + parent: TreeParentId, + index: usize, + cfg: FiIfNotConfigured, + ) -> LoroResult<()> { + let inner = self.inner.try_attached_state()?; + let mut children_len = self.children_num(&parent).unwrap_or(0); + let mut already_in_parent = false; + // check the input is valid + if self.is_parent(&target, &parent) { + // If the position after moving is same as the current position , do nothing + if let Some(current_index) = self.get_index_by_tree_id(&target) { + if current_index == index { + return Ok(()); + } + // move out first, we cannot delete the position here + // If throw error, the tree will be in a inconsistent state + children_len -= 1; + already_in_parent = true; + } + }; + if index > children_len { + return Err(LoroTreeError::IndexOutOfBound { + len: children_len, + index, + } + .into()); + } + let Some(old_index) = self.get_index_by_tree_id(&target) else { + return Err(LoroError::TreeError( + LoroTreeError::TreeNodeDeletedOrNotExist(target), + )); + }; + + if already_in_parent { + self.delete_position(&parent, &target); + } + + match self.generate_position_at(&target, &parent, index, cfg) { + FractionalIndexGenResult::Ok(position) => { + self.mov_with_position(inner, txn, target, parent, index, position, old_index) + } + FractionalIndexGenResult::Rearrange(ids) => { + for (i, (id, position)) in ids.into_iter().enumerate() { + self.mov_with_position(inner, txn, id, parent, index + i, position, old_index)?; + } + Ok(()) + } + FractionalIndexGenResult::NotConfigured => { + Err(LoroTreeError::FractionalIndexNotEnabled.into()) + } + } + } + + #[allow(clippy::too_many_arguments)] + fn create_with_position( + &self, + inner: &BasicHandler, + txn: &mut Transaction, + tree_id: TreeID, + parent: TreeParentId, + index: usize, + position: FractionalIndex, + ) -> LoroResult { + txn.apply_local_op( + inner.container_idx, + crate::op::RawOpContent::Tree(Arc::new(TreeOp::Create { + target: tree_id, + parent: parent.tree_id(), + position: position.clone(), + })), + EventHint::Tree(smallvec![TreeDiffItem { + target: tree_id, + action: TreeExternalDiff::Create { + parent, + index, + position, + }, + }]), + &inner.doc, + )?; + Ok(tree_id) + } + + #[allow(clippy::too_many_arguments)] + fn mov_with_position( + &self, + inner: &BasicHandler, + txn: &mut Transaction, + target: TreeID, + parent: TreeParentId, + index: usize, + position: FractionalIndex, + old_index: usize, + ) -> LoroResult<()> { + txn.apply_local_op( + inner.container_idx, + crate::op::RawOpContent::Tree(Arc::new(TreeOp::Move { + target, + parent: parent.tree_id(), + position: position.clone(), + })), + EventHint::Tree(smallvec![TreeDiffItem { + target, + action: TreeExternalDiff::Move { + parent, + index, + position, + old_parent: self + .get_node_parent(&target) + .ok_or(LoroTreeError::TreeNodeDeletedOrNotExist(target))?, + old_index, + }, + }]), + &inner.doc, + ) + } + + pub fn get_meta(&self, target: TreeID) -> LoroResult { + match &self.inner { + MaybeDetached::Detached(d) => { + let d = d.lock(); + d.value + .map + .get(&target) + .cloned() + .ok_or(LoroTreeError::TreeNodeNotExist(target).into()) + } + MaybeDetached::Attached(a) => { + if self.is_node_unexist(&target) { + return Err(LoroTreeError::TreeNodeNotExist(target).into()); + } + let map_container_id = target.associated_meta_container(); + let handler = create_handler(a, map_container_id); + handler.into_map().map_err(|_| { + LoroError::DecodeError( + "Tree node's associated meta container is not a map".into(), + ) + }) + } + } + } + + pub fn is_node_unexist(&self, target: &TreeID) -> bool { + match &self.inner { + MaybeDetached::Detached(d) => { + let d = d.lock(); + !d.value.map.contains_key(target) + } + MaybeDetached::Attached(a) => a.with_state(|state| { + let a = state.as_tree_state().unwrap(); + a.is_node_unexist(target) + }), + } + } + + pub fn is_node_deleted(&self, target: &TreeID) -> LoroResult { + match &self.inner { + MaybeDetached::Detached(t) => { + let t = t.lock(); + t.value + .map + .get(target) + .and(Some(true)) + .ok_or(LoroTreeError::TreeNodeNotExist(*target).into()) + } + MaybeDetached::Attached(a) => a.with_state(|state| { + let a = state.as_tree_state().unwrap(); + a.is_node_deleted(target) + .ok_or(LoroTreeError::TreeNodeNotExist(*target).into()) + }), + } + } + + /// Get the parent of the node, if the node does not exist, return None + pub fn get_node_parent(&self, target: &TreeID) -> Option { + match &self.inner { + MaybeDetached::Detached(t) => { + let t = t.lock(); + t.value.get_parent(target).map(TreeParentId::from) + } + MaybeDetached::Attached(a) => a.with_state(|state| { + let a = state.as_tree_state().unwrap(); + a.parent(target) + }), + } + } + + // TODO: iterator + pub fn children(&self, parent: &TreeParentId) -> Option> { + match &self.inner { + MaybeDetached::Detached(t) => { + let t = t.lock(); + t.value.get_children(parent.tree_id()) + } + MaybeDetached::Attached(a) => a.with_state(|state| { + let a = state.as_tree_state().unwrap(); + a.get_children(parent).map(|x| x.collect()) + }), + } + } + + pub fn children_num(&self, parent: &TreeParentId) -> Option { + match &self.inner { + MaybeDetached::Detached(t) => { + let t = t.lock(); + t.value.children_num(parent.tree_id()) + } + MaybeDetached::Attached(a) => a.with_state(|state| { + let a = state.as_tree_state().unwrap(); + a.children_num(parent) + }), + } + } + + /// Check if the node is exist. include deleted node. + pub fn contains(&self, target: TreeID) -> bool { + match &self.inner { + MaybeDetached::Detached(t) => { + let t = t.lock(); + t.value.map.contains_key(&target) + } + MaybeDetached::Attached(a) => a.with_state(|state| { + let a = state.as_tree_state().unwrap(); + !a.is_node_unexist(&target) + }), + } + } + + pub fn get_child_at(&self, parent: &TreeParentId, index: usize) -> Option { + match &self.inner { + MaybeDetached::Detached(t) => { + let t = t.lock(); + t.value.get_id_by_index(&parent.tree_id(), index) + } + MaybeDetached::Attached(a) => a.with_state(|state| { + let a = state.as_tree_state().unwrap(); + a.get_id_by_index(parent, index) + }), + } + } + + pub fn is_parent(&self, target: &TreeID, parent: &TreeParentId) -> bool { + match &self.inner { + MaybeDetached::Detached(t) => { + let t = t.lock(); + t.value.is_parent(target, &parent.tree_id()) + } + MaybeDetached::Attached(a) => a.with_state(|state| { + let a = state.as_tree_state().unwrap(); + a.is_parent(target, parent) + }), + } + } + + /// Get all nodes in the tree, including deleted nodes + pub fn nodes(&self) -> Vec { + match &self.inner { + MaybeDetached::Detached(t) => { + let t = t.lock(); + t.value.map.keys().cloned().collect() + } + MaybeDetached::Attached(a) => a.with_state(|state| { + let a = state.as_tree_state().unwrap(); + a.nodes() + }), + } + } + + pub fn get_nodes_under(&self, parent: TreeParentId) -> Vec { + match &self.inner { + MaybeDetached::Detached(t) => t.lock().value.get_nodes_under(parent), + MaybeDetached::Attached(a) => a.with_state(|state| { + let a = state.as_tree_state().unwrap(); + a.get_all_tree_nodes_under(parent) + }), + } + } + pub fn roots(&self) -> Vec { + self.children(&TreeParentId::Root).unwrap_or_default() + } + + pub fn get_all_hierarchy_nodes_under(&self, parent: TreeParentId) -> Vec { + match &self.inner { + MaybeDetached::Detached(_t) => { + unreachable!() + } + MaybeDetached::Attached(a) => a.with_state(|state| { + let a = state.as_tree_state().unwrap(); + a.get_all_hierarchy_nodes_under(parent) + }), + } + } + + #[allow(non_snake_case)] + pub fn __internal__next_tree_id(&self) -> TreeID { + match &self.inner { + MaybeDetached::Detached(d) => { + let d = d.lock(); + TreeID::new(PeerID::MAX, d.value.next_counter) + } + MaybeDetached::Attached(a) => a + .with_txn(|txn| Ok(TreeID::from_id(txn.next_id()))) + .unwrap(), + } + } + + fn generate_position_at( + &self, + target: &TreeID, + parent: &TreeParentId, + index: usize, + cfg: FiIfNotConfigured, + ) -> FractionalIndexGenResult { + let MaybeDetached::Attached(a) = &self.inner else { + unreachable!() + }; + a.with_state(|state| { + let a = state.as_tree_state_mut().unwrap(); + a.generate_position_at(target, parent, index, cfg) + }) + } + + /// Get the index of the target node in the parent node + /// + /// O(logN) + pub fn get_index_by_tree_id(&self, target: &TreeID) -> Option { + match &self.inner { + MaybeDetached::Detached(t) => { + let t = t.lock(); + t.value.get_index_by_tree_id(target) + } + MaybeDetached::Attached(a) => a.with_state(|state| { + let a = state.as_tree_state().unwrap(); + a.get_index_by_tree_id(target) + }), + } + } + + pub fn get_position_by_tree_id(&self, target: &TreeID) -> Option { + match &self.inner { + MaybeDetached::Detached(t) => t + .lock() + .value + .parent_links + .contains_key(target) + .then(FractionalIndex::default), + MaybeDetached::Attached(a) => a.with_state(|state| { + let a = state.as_tree_state().unwrap(); + a.get_position(target) + }), + } + } + + fn delete_position(&self, parent: &TreeParentId, target: &TreeID) { + let MaybeDetached::Attached(a) = &self.inner else { + unreachable!() + }; + a.with_state(|state| { + let a = state.as_tree_state_mut().unwrap(); + a.try_delete_position_cache(parent, target) + }) + } + + // use for apply diff + pub(crate) fn get_index_by_fractional_index( + &self, + parent: &TreeParentId, + node_position: &NodePosition, + ) -> Option { + match &self.inner { + MaybeDetached::Detached(_) => { + unreachable!(); + } + MaybeDetached::Attached(a) => a.with_state(|state| { + let a = state.as_tree_state().unwrap(); + a.get_index_by_position(parent, node_position) + }), + } + } + + pub(crate) fn next_idlp(&self) -> IdLp { + match &self.inner { + MaybeDetached::Detached(_) => { + unreachable!() + } + MaybeDetached::Attached(a) => a.with_txn(|txn| Ok(txn.next_idlp())).unwrap(), + } + } + + pub fn is_fractional_index_enabled(&self) -> bool { + match &self.inner { + MaybeDetached::Detached(_) => true, + MaybeDetached::Attached(a) => a.with_state(|state| { + let a = state.as_tree_state().unwrap(); + a.is_fractional_index_enabled() + }), + } + } + + /// Set whether to generate fractional index for Tree Position. The LoroDoc is set to use jitter 0 by default. + /// + /// The jitter is used to avoid conflicts when multiple users are creating the node at the same position. + /// value 0 is default, which means no jitter, any value larger than 0 will enable jitter. + /// + /// Generally speaking, jitter will affect the growth rate of document size. + /// [Read more about it](https://www.loro.dev/blog/movable-tree#implementation-and-encoding-size) + pub fn enable_fractional_index(&self, jitter: u8) { + match &self.inner { + MaybeDetached::Detached(_) => { + // No-op on detached trees + let _ = jitter; + } + MaybeDetached::Attached(a) => a.with_state(|state| { + let a = state.as_tree_state_mut().unwrap(); + a.enable_generate_fractional_index(jitter); + }), + } + } + + /// Disable the fractional index generation for Tree Position when + /// you don't need the Tree's siblings to be sorted. The fractional index will be always default. + /// + /// The LoroDoc is set to disable fractional index by default. + pub fn disable_fractional_index(&self) { + match &self.inner { + MaybeDetached::Detached(_) => { + // No-op on detached trees + } + MaybeDetached::Attached(a) => a.with_state(|state| { + let a = state.as_tree_state_mut().unwrap(); + a.disable_generate_fractional_index(); + }), + } + } + + pub fn is_deleted(&self) -> bool { + match &self.inner { + MaybeDetached::Detached(_) => false, + MaybeDetached::Attached(a) => a.is_deleted(), + } + } + + pub fn is_empty(&self) -> bool { + match &self.inner { + MaybeDetached::Detached(t) => { + let t = t.lock(); + t.value.map.is_empty() + } + MaybeDetached::Attached(a) => a.with_state(|state| { + let a = state.as_tree_state().unwrap(); + a.is_empty() + }), + } + } + + pub fn get_last_move_id(&self, target: &TreeID) -> Option { + match &self.inner { + MaybeDetached::Detached(_) => None, + MaybeDetached::Attached(a) => a.with_state(|state| { + let a = state.as_tree_state().unwrap(); + a.get_last_move_id(target) + }), + } + } + + pub fn clear(&self) -> LoroResult<()> { + match &self.inner { + MaybeDetached::Detached(t) => { + let mut t = t.lock(); + t.value.map.clear(); + t.value.children_links.clear(); + t.value.parent_links.clear(); + Ok(()) + } + MaybeDetached::Attached(_) => { + let nodes = self.get_nodes_under(TreeParentId::Root); + for node in nodes { + self.delete(node.id)?; + } + Ok(()) + } + } + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/history_cache.rs b/3rdparty/loro/crates/loro-internal/src/history_cache.rs new file mode 100644 index 0000000000..b229739d4c --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/history_cache.rs @@ -0,0 +1,991 @@ +use std::{ + collections::{BTreeMap, BTreeSet}, + marker::PhantomData, + ops::Bound, + sync::Arc, +}; + +use crate::sync::Mutex; +use either::Either; +use enum_as_inner::EnumAsInner; +use enum_dispatch::enum_dispatch; +use generic_btree::rle::{HasLength as _, Sliceable}; +use loro_common::{ + ContainerType, Counter, HasLamport, IdFull, IdLp, InternalString, LoroValue, PeerID, ID, +}; +use rle::HasLength; +use rustc_hash::FxHashMap; + +use crate::{ + change::{Change, Lamport}, + container::{ + idx::ContainerIdx, + list::list_op::InnerListOp, + richtext::{richtext_state::RichtextStateChunk, AnchorType, StyleOp}, + tree::tree_op::TreeOp, + }, + delta::MapValue, + diff_calc::tree::{MoveLamportAndID, TreeCacheForDiff}, + encoding::value_register::ValueRegister, + op::{InnerContent, RichOp, SliceWithId}, + oplog::ChangeStore, + state::{ContainerCreationContext, GcStore}, + OpLog, VersionVector, +}; + +/// A cache for the history of a container. +/// +/// There are cases where a container needs a view of the history of the container. +/// For example, when switching to an older version, a map container may need to know the value of a key at a certain version. +/// This cache provides a faster way to do that than scanning the oplog. +#[derive(Debug)] +pub(crate) struct ContainerHistoryCache { + change_store: ChangeStore, + shallow_root_state: Option>, + for_checkout: Option, + for_importing: Option>, +} + +#[derive(Debug, Default)] +pub(crate) struct ForCheckout { + pub(crate) map: MapHistoryCache, + pub(crate) movable_list: MovableListHistoryCache, +} + +#[derive(Clone, Copy)] +pub(crate) struct HasImportingCacheMark { + _private: PhantomData<()>, +} + +impl HistoryCacheTrait for ForCheckout { + fn insert(&mut self, op: &RichOp) { + match op.raw_op().container.get_type() { + ContainerType::Map => self.map.insert(op), + ContainerType::MovableList => self.movable_list.insert(op), + _ => {} + } + } +} + +impl ContainerHistoryCache { + pub(crate) fn new(change_store: ChangeStore, gc: Option>) -> Self { + Self { + change_store, + for_checkout: Default::default(), + for_importing: Default::default(), + shallow_root_state: gc, + } + } + + pub(crate) fn insert_by_new_change( + &mut self, + change: &Change, + for_checkout: bool, + for_importing: bool, + ) { + if self.for_checkout.is_none() && self.for_importing.is_none() { + return; + } + + for op in change.ops.iter() { + match op.container.get_type() { + ContainerType::Map | ContainerType::MovableList + if self.for_checkout.is_some() && for_checkout => + { + let rich_op = RichOp::new_by_change(change, op); + self.for_checkout.as_mut().unwrap().insert(&rich_op) + } + ContainerType::Tree if self.for_importing.is_some() && for_importing => { + let container_idx = op.container; + let rich_op = RichOp::new_by_change(change, op); + let manager = self + .for_importing + .as_mut() + .unwrap() + .entry(container_idx) + .or_insert_with(|| match op.container.get_type() { + ContainerType::Tree => { + HistoryCacheForImporting::Tree(TreeOpGroup::default()) + } + _ => unreachable!(), + }); + manager.insert(&rich_op) + } + _ => continue, + } + } + } + + pub(crate) fn get_checkout_index(&mut self) -> &ForCheckout { + self.ensure_all_caches_exist(); + self.for_checkout.as_ref().unwrap() + } + + pub(crate) fn ensure_all_caches_exist(&mut self) { + let mut record_for_checkout = false; + let mut record_for_importing = false; + if self.for_checkout.is_none() { + self.for_checkout = Some(ForCheckout::default()); + record_for_checkout = true; + } + + if self.for_importing.is_none() { + self.for_importing = Some(FxHashMap::default()); + record_for_importing = true; + } + + if !record_for_checkout && !record_for_importing { + return; + } + + self.init_cache_by_visit_all_change_slow(record_for_checkout, record_for_importing); + } + + pub(crate) fn ensure_importing_caches_exist(&mut self) -> HasImportingCacheMark { + if self.for_importing.is_some() { + return HasImportingCacheMark { + _private: PhantomData, + }; + } + + self.for_importing = Some(FxHashMap::default()); + self.init_cache_by_visit_all_change_slow(false, true); + HasImportingCacheMark { + _private: PhantomData, + } + } + + fn init_cache_by_visit_all_change_slow(&mut self, for_checkout: bool, for_importing: bool) { + if self.for_checkout.is_none() && self.for_importing.is_none() { + return; + } + + if !for_checkout && !for_importing { + return; + } + + self.change_store.visit_all_changes(&mut |c| { + for op in c.ops.iter() { + match op.container.get_type() { + ContainerType::Map | ContainerType::MovableList + if self.for_checkout.is_some() && for_checkout => + { + let rich_op = RichOp::new_by_change(c, op); + self.for_checkout.as_mut().unwrap().insert(&rich_op) + } + ContainerType::Tree if self.for_importing.is_some() && for_importing => { + let container_idx = op.container; + let rich_op = RichOp::new_by_change(c, op); + let manager = self + .for_importing + .as_mut() + .unwrap() + .entry(container_idx) + .or_insert_with(|| match op.container.get_type() { + ContainerType::Tree => { + HistoryCacheForImporting::Tree(TreeOpGroup::default()) + } + _ => unreachable!(), + }); + manager.insert(&rich_op) + } + _ => continue, + } + } + }); + + let default_ctx = ContainerCreationContext { + configure: &Default::default(), + peer: 0, + }; + + if let Some(state) = self.shallow_root_state.as_ref() { + ensure_cov::notify_cov( + "loro_internal::history_cache::init_cache_by_visit_all_change_slow::visit_gc", + ); + let mut store = state.store.lock(); + for (idx, c) in store.iter_all_containers_mut() { + match idx.get_type() { + ContainerType::Text | ContainerType::List | ContainerType::Unknown(_) => { + continue + } + #[cfg(feature = "counter")] + ContainerType::Counter => continue, + ContainerType::Map => {} + ContainerType::MovableList => {} + ContainerType::Tree => {} + } + + let state = c.get_state_mut(idx, default_ctx); + + match state { + crate::state::State::MapState(m) => { + if for_checkout { + let c = self.for_checkout.as_mut().unwrap(); + for (k, v) in m.iter() { + c.map.record_shallow_root_state_entry(idx, k, v); + } + } + } + crate::state::State::MovableListState(l) => { + for (idlp, elem) in l.elements() { + if for_checkout { + let c = self.for_checkout.as_mut().unwrap(); + let item = l.get_list_item(elem.pos).unwrap(); + c.movable_list.record_shallow_root_state( + item.id, + idlp.peer, + idlp.lamport.into(), + elem.value.clone(), + ); + } + } + } + crate::state::State::TreeState(t) => { + if for_importing { + let c = self.for_importing.as_mut().unwrap(); + let tree = c.entry(idx).or_insert_with(|| { + HistoryCacheForImporting::Tree(Default::default()) + }); + tree.as_tree_mut().unwrap().record_shallow_root_state( + t.tree_nodes() + .into_iter() + .map(|node| MoveLamportAndID { + id: node.last_move_op, + op: Arc::new(TreeOp::Create { + target: node.id, + parent: node.parent.tree_id(), + position: node.fractional_index.clone(), + }), + effected: true, + }) + .collect(), + ); + } + } + _ => unreachable!(), + } + } + } + } + + pub(crate) fn get_importing_cache( + &self, + container_idx: &ContainerIdx, + _: HasImportingCacheMark, + ) -> Option<&HistoryCacheForImporting> { + self.for_importing.as_ref().unwrap().get(container_idx) + } + + pub(crate) fn get_tree( + &self, + container_idx: &ContainerIdx, + _: HasImportingCacheMark, + ) -> Option<&TreeOpGroup> { + self.for_importing + .as_ref() + .unwrap() + .get(container_idx) + .map(|group| match group { + HistoryCacheForImporting::Tree(tree) => tree, + }) + } + + pub(crate) fn has_cache(&self) -> bool { + self.for_checkout.is_some() + } + + pub(crate) fn free(&mut self) { + self.for_checkout = None; + } + + pub(crate) fn free_all(&mut self) { + self.for_checkout = None; + self.for_importing = None; + } + + pub(crate) fn set_shallow_root_store(&mut self, shallow_root_store: Option>) { + self.shallow_root_state = shallow_root_store; + } + + pub(crate) fn find_text_chunks_in( + &self, + idx: ContainerIdx, + target_span: loro_common::IdSpan, + ) -> Vec { + ensure_cov::notify_cov("loro_internal::history_cache::find_text_chunks_in"); + let Some(state) = self.shallow_root_state.as_ref() else { + return Vec::new(); + }; + + let mut binding = state.store.lock(); + let Some(text) = binding.get_mut(idx) else { + return Vec::new(); + }; + + let text_state = text + .get_state( + idx, + ContainerCreationContext { + configure: &Default::default(), + peer: 0, + }, + ) + .as_richtext_state() + .unwrap(); + + // TODO: FIXME: This could be super slow in some cases + let mut ans = Vec::new(); + text_state.iter_raw(&mut |chunk| { + let c_span = chunk.get_id_span(); + if let Some(range) = target_span.get_slice_range_on(&c_span) { + let c = chunk.slice(range.0..range.1); + ans.push(c); + } + }); + + ans.sort_unstable_by_key(|x| x.counter()); + let mut normalized = Vec::with_capacity(ans.len()); + let mut covered_end = target_span.counter.start; + for chunk in ans { + let span = chunk.get_id_span(); + let start = span.counter.start.max(covered_end); + let end = span.counter.end.min(target_span.counter.end); + if start >= end { + continue; + } + + normalized.push( + chunk.slice( + (start - span.counter.start) as usize..(end - span.counter.start) as usize, + ), + ); + covered_end = end; + } + + normalized + } + + pub(crate) fn find_text_chunks_in_shallow_root_order( + &self, + idx: ContainerIdx, + ) -> Vec { + ensure_cov::notify_cov( + "loro_internal::history_cache::find_text_chunks_in_shallow_root_order", + ); + let Some(state) = self.shallow_root_state.as_ref() else { + return Vec::new(); + }; + + let mut binding = state.store.lock(); + let Some(text) = binding.get_mut(idx) else { + return Vec::new(); + }; + + let text_state = text + .get_state( + idx, + ContainerCreationContext { + configure: &Default::default(), + peer: 0, + }, + ) + .as_richtext_state() + .unwrap(); + + let mut ans = Vec::new(); + text_state.iter_raw(&mut |chunk| { + ans.push(chunk.slice(0..chunk.rle_len())); + }); + + ans + } + + pub(crate) fn find_text_style_end_in_shallow_root( + &self, + idx: ContainerIdx, + style_start_id: ID, + ) -> Option<(StyleOp, usize)> { + ensure_cov::notify_cov("loro_internal::history_cache::find_text_style_end_in_shallow_root"); + let state = self.shallow_root_state.as_ref()?; + + let mut binding = state.store.lock(); + let text = binding.get_mut(idx)?; + + let text_state = text + .get_state( + idx, + ContainerCreationContext { + configure: &Default::default(), + peer: 0, + }, + ) + .as_richtext_state() + .unwrap(); + + let mut entity_index = 0; + let mut ans = None; + text_state.iter_raw(&mut |chunk| { + if ans.is_some() { + return; + } + + match chunk { + RichtextStateChunk::Text(text) => { + entity_index += text.unicode_len() as usize; + } + RichtextStateChunk::Style { style, anchor_type } => { + if style.id() == style_start_id && *anchor_type == AnchorType::End { + ans = Some((style.as_ref().clone(), entity_index.saturating_sub(1))); + } + entity_index += 1; + } + } + }); + + ans + } + + pub(crate) fn find_list_chunks_in( + &self, + idx: ContainerIdx, + target_span: loro_common::IdSpan, + ) -> Vec { + ensure_cov::notify_cov("loro_internal::history_cache::find_list_chunks_in"); + let Some(state) = self.shallow_root_state.as_ref() else { + return Vec::new(); + }; + + let mut binding = state.store.lock(); + let Some(list) = binding.get_mut(idx) else { + return Vec::new(); + }; + + let list_state = list.get_state( + idx, + ContainerCreationContext { + configure: &Default::default(), + peer: 0, + }, + ); + + let mut ans = Vec::with_capacity(target_span.atom_len()); + match list_state { + crate::state::State::ListState(list) => { + for v in list.iter_with_id() { + if target_span.contains(v.id.id()) { + ans.push(SliceWithId { + values: Either::Right(v.v.clone()), + id: v.id, + elem_id: None, + }) + } + } + } + crate::state::State::MovableListState(list) => { + for (move_id, elem_id, v) in list.iter_with_last_move_id_and_elem_id() { + if target_span.contains(move_id.id()) { + ans.push(SliceWithId { + values: Either::Right(v.clone()), + id: move_id, + elem_id: Some(elem_id), + }) + } + } + } + _ => unreachable!(), + } + + ans.sort_unstable_by_key(|x| x.id.counter); + ans + } +} + +#[enum_dispatch(OpGroupTrait)] +#[derive(Debug, EnumAsInner)] +pub(crate) enum HistoryCacheForImporting { + Tree(TreeOpGroup), +} + +impl HistoryCacheForImporting { + fn insert(&mut self, op: &RichOp) { + match self { + HistoryCacheForImporting::Tree(t) => t.insert(op), + } + } +} + +#[enum_dispatch] +trait HistoryCacheTrait { + fn insert(&mut self, op: &RichOp); +} + +#[derive(Debug, Clone)] +pub(crate) struct GroupedMapOpInfo> { + pub(crate) value: T, + pub(crate) lamport: Lamport, + pub(crate) peer: PeerID, +} + +impl PartialEq for GroupedMapOpInfo { + fn eq(&self, other: &Self) -> bool { + self.lamport == other.lamport && self.peer == other.peer + } +} + +impl Eq for GroupedMapOpInfo {} + +impl PartialOrd for GroupedMapOpInfo { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for GroupedMapOpInfo { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.lamport + .cmp(&other.lamport) + .then_with(|| self.peer.cmp(&other.peer)) + } +} + +#[derive(Debug, Clone)] +struct MapHistoryCacheEntry { + container: ContainerIdx, + key: u32, + lamport: Lamport, + peer: PeerID, + counter_or_value: Either>>, +} + +impl PartialEq for MapHistoryCacheEntry { + fn eq(&self, other: &Self) -> bool { + self.container == other.container + && self.key == other.key + && self.lamport == other.lamport + && self.peer == other.peer + } +} + +impl Eq for MapHistoryCacheEntry {} + +impl PartialOrd for MapHistoryCacheEntry { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for MapHistoryCacheEntry { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.container + .cmp(&other.container) + .then_with(|| self.key.cmp(&other.key)) + .then_with(|| self.lamport.cmp(&other.lamport)) + .then_with(|| self.peer.cmp(&other.peer)) + } +} + +#[derive(Debug, Default)] +pub(crate) struct MapHistoryCache { + keys: ValueRegister, + map: BTreeSet, +} + +impl HistoryCacheTrait for MapHistoryCache { + fn insert(&mut self, op: &RichOp) { + let container = op.raw_op().container; + let key = match &op.raw_op().content { + InnerContent::Map(map) => map.key.clone(), + _ => unreachable!(), + }; + + let key_idx = self.keys.register(&key); + self.map.insert(MapHistoryCacheEntry { + container, + key: key_idx as u32, + lamport: op.lamport(), + peer: op.peer, + counter_or_value: Either::Left(op.counter()), + }); + } +} + +impl MapHistoryCache { + fn record_shallow_root_state_entry( + &mut self, + idx: ContainerIdx, + k: &InternalString, + v: &MapValue, + ) { + let key_idx = self.keys.register(k); + self.map.insert(MapHistoryCacheEntry { + container: idx, + key: key_idx as u32, + lamport: v.lamport(), + peer: v.peer, + counter_or_value: Either::Right(Box::new(v.value.clone())), + }); + } + + pub fn get_container_latest_op_at_vv( + &self, + container: ContainerIdx, + vv: &VersionVector, + // PERF: utilize this lamport + _max_lamport: Lamport, + oplog: &OpLog, + ) -> FxHashMap { + let mut ans = FxHashMap::default(); + let mut last_key = u32::MAX; + + 'outer: loop { + let range = ( + Bound::Included(MapHistoryCacheEntry { + container, + key: 0, + lamport: 0, + peer: 0, + counter_or_value: Either::Left(0), + }), + Bound::Excluded(MapHistoryCacheEntry { + container, + key: last_key, + lamport: 0, + peer: 0, + counter_or_value: Either::Left(0), + }), + ); + + for entry in self.map.range(range).rev() { + match &entry.counter_or_value { + Either::Left(cnt) => { + if vv.get(&entry.peer).copied().unwrap_or(0) > *cnt { + let id = ID::new(entry.peer, *cnt); + let op = oplog.get_op_that_includes(id).unwrap(); + debug_assert_eq!(op.atom_len(), 1); + match &op.content { + InnerContent::Map(map) => { + ans.insert( + self.keys.get_value(entry.key as usize).unwrap().clone(), + GroupedMapOpInfo { + value: map.value.clone(), + lamport: entry.lamport, + peer: entry.peer, + }, + ); + } + _ => unreachable!(), + } + last_key = entry.key; + continue 'outer; + } + } + Either::Right(v) => { + let k = self.keys.get_value(entry.key as usize).unwrap().clone(); + ans.insert( + k, + GroupedMapOpInfo { + value: (**v).clone(), + lamport: entry.lamport, + peer: entry.peer, + }, + ); + last_key = entry.key; + continue 'outer; + } + } + } + + break; + } + + ans + } +} + +#[derive(Debug, Clone)] +pub(crate) struct GroupedTreeOpInfo { + pub(crate) counter: Counter, + pub(crate) value: Arc, +} + +#[derive(Debug, Default)] +pub(crate) struct TreeOpGroup { + ops: BTreeMap, + tree_for_diff: Mutex, +} + +impl HistoryCacheTrait for TreeOpGroup { + fn insert(&mut self, op: &RichOp) { + let tree_op = op.raw_op().content.as_tree().unwrap(); + self.ops.insert( + op.idlp(), + GroupedTreeOpInfo { + value: tree_op.clone(), + counter: op.raw_op().counter, + }, + ); + } +} + +impl TreeOpGroup { + pub fn ops(&self) -> &BTreeMap { + &self.ops + } + + pub fn tree(&self) -> &Mutex { + &self.tree_for_diff + } + + pub(crate) fn record_shallow_root_state(&mut self, nodes: Vec) { + let mut tree = self.tree_for_diff.lock(); + for node in nodes.iter() { + self.ops.insert( + node.id.idlp(), + GroupedTreeOpInfo { + counter: node.id.counter, + value: node.op.clone(), + }, + ); + } + tree.init_tree_with_shallow_root_version(nodes); + } +} + +#[derive(Debug, Default)] +pub(crate) struct MovableListHistoryCache { + move_set: BTreeSet, + set_set: BTreeSet, +} + +#[derive(Debug, Eq, PartialEq, Clone)] +struct MovableListInnerDeltaEntry { + element_lamport: Lamport, + element_peer: PeerID, + lamport: Lamport, + peer: PeerID, + counter: Counter, +} + +#[derive(Debug, Eq, PartialEq, Clone)] +struct MovableListSetDeltaEntry { + element_lamport: Lamport, + element_peer: PeerID, + lamport: Lamport, + peer: PeerID, + counter_or_value: Either>, +} + +impl Ord for MovableListInnerDeltaEntry { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.element_lamport + .cmp(&other.element_lamport) + .then_with(|| self.element_peer.cmp(&other.element_peer)) + .then_with(|| self.lamport.cmp(&other.lamport)) + .then_with(|| self.peer.cmp(&other.peer)) + } +} + +impl PartialOrd for MovableListInnerDeltaEntry { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl PartialOrd for MovableListSetDeltaEntry { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for MovableListSetDeltaEntry { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.element_lamport + .cmp(&other.element_lamport) + .then_with(|| self.element_peer.cmp(&other.element_peer)) + .then_with(|| self.lamport.cmp(&other.lamport)) + .then_with(|| self.peer.cmp(&other.peer)) + } +} + +impl HistoryCacheTrait for MovableListHistoryCache { + fn insert(&mut self, op: &RichOp) { + let cur_id = op.id_full(); + match &op.op().content { + InnerContent::List(l) => match l { + crate::container::list::list_op::InnerListOp::Move { elem_id, .. } => { + self.move_set.insert(MovableListInnerDeltaEntry { + element_lamport: elem_id.lamport, + element_peer: elem_id.peer, + lamport: cur_id.lamport, + peer: cur_id.peer, + counter: cur_id.counter, + }); + } + crate::container::list::list_op::InnerListOp::Set { elem_id, .. } => { + self.set_set.insert(MovableListSetDeltaEntry { + element_lamport: elem_id.lamport, + element_peer: elem_id.peer, + lamport: cur_id.lamport, + peer: cur_id.peer, + counter_or_value: Either::Left(cur_id.counter), + }); + } + _ => {} + }, + _ => unreachable!(), + } + } +} + +impl MovableListHistoryCache { + pub(crate) fn record_shallow_root_state( + &mut self, + id: IdFull, + elem_peer: PeerID, + elem_lamport: Lamport, + value: LoroValue, + ) { + self.set_set.insert(MovableListSetDeltaEntry { + element_lamport: elem_lamport, + element_peer: elem_peer, + lamport: id.lamport, + peer: id.peer, + counter_or_value: Either::Right(Box::new(value.clone())), + }); + self.move_set.insert(MovableListInnerDeltaEntry { + element_lamport: elem_lamport, + element_peer: elem_peer, + lamport: id.lamport, + peer: id.peer, + counter: id.counter, + }); + } + + pub(crate) fn last_value( + &self, + key: IdLp, + vv: &VersionVector, + max_lamport: Lamport, + oplog: &OpLog, + ) -> Option> { + self.set_set + .range(( + Bound::Included(MovableListSetDeltaEntry { + element_lamport: key.lamport, + element_peer: key.peer, + lamport: 0, + peer: 0, + counter_or_value: Either::Left(0), + }), + Bound::Excluded(MovableListSetDeltaEntry { + element_lamport: key.lamport, + element_peer: key.peer, + lamport: max_lamport, + peer: PeerID::MAX, + counter_or_value: Either::Left(Counter::MAX), + }), + )) + .rev() + .find(|e| { + let counter = match &e.counter_or_value { + Either::Left(c) => *c, + Either::Right(_) => -1, + }; + vv.get(&e.peer).copied().unwrap_or(0) > counter + }) + .map_or_else( + || { + let id = oplog.idlp_to_id(key).unwrap(); + if vv.get(&id.peer).copied().unwrap_or(0) <= id.counter { + return None; + } + let op = oplog.get_op_that_includes(id).unwrap(); + let offset = id.counter - op.counter; + match &op.content { + InnerContent::List(InnerListOp::Insert { slice, .. }) => { + let value = oplog + .arena + .get_value(slice.0.start as usize + offset as usize) + .unwrap(); + Some(GroupedMapOpInfo { + value, + lamport: key.lamport, + peer: id.peer, + }) + } + _ => { + unreachable!() + } + } + }, + |e| { + let (lamport, value) = match &e.counter_or_value { + Either::Left(c) => { + let id = ID::new(e.peer, *c); + let op = oplog.get_op_that_includes(id).unwrap(); + debug_assert_eq!(op.atom_len(), 1); + let lamport = op.lamport(); + match &op.content { + InnerContent::List(InnerListOp::Set { value, .. }) => { + (lamport, value.clone()) + } + _ => { + unreachable!() + } + } + } + Either::Right(v) => (e.lamport, v.as_ref().clone()), + }; + Some(GroupedMapOpInfo { + value: value.clone(), + lamport, + peer: e.peer, + }) + }, + ) + } + + pub(crate) fn last_pos( + &self, + key: IdLp, + vv: &VersionVector, + max_lamport: Lamport, + oplog: &OpLog, + ) -> Option { + self.move_set + .range(( + Bound::Included(MovableListInnerDeltaEntry { + element_lamport: key.lamport, + element_peer: key.peer, + lamport: 0, + peer: 0, + counter: 0, + }), + Bound::Excluded(MovableListInnerDeltaEntry { + element_lamport: key.lamport, + element_peer: key.peer, + lamport: max_lamport, + peer: PeerID::MAX, + counter: Counter::MAX, + }), + )) + .rev() + .find(|e| vv.get(&e.peer).copied().unwrap_or(0) > e.counter) + .map_or_else( + || { + let id = oplog.idlp_to_id(key).unwrap(); + if vv.get(&id.peer).copied().unwrap_or(0) > id.counter { + Some(IdFull::new(id.peer, id.counter, key.lamport)) + } else { + None + } + }, + |e| { + let id = ID::new(e.peer, e.counter); + let lamport = oplog.get_lamport_at(id).unwrap_or(e.lamport); + Some(IdFull::new(e.peer, e.counter, lamport)) + }, + ) + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/id.rs b/3rdparty/loro/crates/loro-internal/src/id.rs new file mode 100644 index 0000000000..1c8882be9f --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/id.rs @@ -0,0 +1,4 @@ +pub use loro_common::Counter; +pub use loro_common::PeerID; +pub use loro_common::TreeID; +pub use loro_common::ID; diff --git a/3rdparty/loro/crates/loro-internal/src/jsonpath/ast.rs b/3rdparty/loro/crates/loro-internal/src/jsonpath/ast.rs new file mode 100644 index 0000000000..cd06c317e2 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/jsonpath/ast.rs @@ -0,0 +1,458 @@ +use crate::jsonpath::errors::JSONPathError; +use crate::jsonpath::JSONPathParser; +use std::fmt::{self, Write}; + +#[derive(Debug, Clone)] +pub struct Query { + pub segments: Segment, +} + +#[derive(Debug, Clone)] +pub enum Segment { + Root {}, + Child { + left: Box, + selectors: Vec, + }, + Recursive { + left: Box, + selectors: Vec, + }, +} + +#[derive(Debug, Clone)] +pub enum Selector { + Name { + name: String, + }, + Index { + index: i64, + }, + Slice { + start: Option, + stop: Option, + step: Option, + }, + Wild {}, + Filter { + expression: Box, + }, +} + +#[derive(Debug, Clone)] +pub enum FilterExpression { + True_ {}, + False_ {}, + Null {}, + Array { + values: Vec, + }, + StringLiteral { + value: String, + }, + Int { + value: i64, + }, + Float { + value: f64, + }, + Not { + expression: Box, + }, + Logical { + left: Box, + operator: LogicalOperator, + right: Box, + }, + Comparison { + left: Box, + operator: ComparisonOperator, + right: Box, + }, + RelativeQuery { + query: Box, + }, + RootQuery { + query: Box, + }, + Function { + name: String, + args: Vec, + }, +} + +#[derive(Debug, Clone)] +pub enum LogicalOperator { + And, + Or, +} + +#[derive(Debug, Clone)] +pub enum ComparisonOperator { + Eq, + Ne, + Ge, + Gt, + Le, + Lt, + Contains, + In, +} + +impl Query { + pub fn standard(expr: &str) -> Result { + JSONPathParser::new().parse(expr) + } + + pub fn is_singular(&self) -> bool { + self.segments.is_singular() + } +} + +impl Segment { + // Returns `true` if this query can resolve to at most one node, or `false` otherwise. + pub fn is_singular(&self) -> bool { + match self { + Segment::Child { left, selectors } => { + selectors.len() == 1 + && selectors.first().is_some_and(|selector| { + matches!(selector, Selector::Name { .. } | Selector::Index { .. }) + }) + && left.is_singular() + } + Segment::Recursive { .. } => false, + Segment::Root {} => true, + } + } +} + +impl FilterExpression { + pub fn is_literal(&self) -> bool { + matches!( + self, + FilterExpression::True_ { .. } + | FilterExpression::False_ { .. } + | FilterExpression::Null { .. } + | FilterExpression::StringLiteral { .. } + | FilterExpression::Int { .. } + | FilterExpression::Float { .. } + | FilterExpression::Array { .. } + ) + } +} + +impl fmt::Display for Query { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "${}", self.segments) + } +} + +impl fmt::Display for Segment { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Segment::Child { left, selectors } => write!( + f, + "{}[{}]", + left, + selectors + .iter() + .map(|s| s.to_string()) + .collect::>() + .join(", ") + ), + Segment::Recursive { left, selectors } => write!( + f, + "{}..[{}]", + left, + selectors + .iter() + .map(|s| s.to_string()) + .collect::>() + .join(", ") + ), + Segment::Root {} => Ok(()), + } + } +} + +impl fmt::Display for Selector { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Selector::Name { name, .. } => write!(f, "'{name}'"), + Selector::Index { + index: array_index, .. + } => write!(f, "{array_index}"), + Selector::Slice { + start, stop, step, .. + } => { + write!( + f, + "{}:{}:{}", + start + .and_then(|i| Some(i.to_string())) + .unwrap_or(String::from("")), + stop.and_then(|i| Some(i.to_string())) + .unwrap_or(String::from("")), + step.and_then(|i| Some(i.to_string())) + .unwrap_or(String::from("1")), + ) + } + Selector::Wild { .. } => f.write_char('*'), + Selector::Filter { expression, .. } => write!(f, "?{expression}"), + } + } +} + +impl fmt::Display for FilterExpression { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + FilterExpression::True_ { .. } => f.write_str("true"), + FilterExpression::False_ { .. } => f.write_str("false"), + FilterExpression::Null { .. } => f.write_str("null"), + FilterExpression::StringLiteral { value, .. } => write!(f, "'{value}'"), + FilterExpression::Array { values, .. } => { + write!( + f, + "[{}]", + values + .iter() + .map(|e| e.to_string()) + .collect::>() + .join(", ") + ) + } + FilterExpression::Int { value, .. } => write!(f, "{value}"), + FilterExpression::Float { value, .. } => write!(f, "{value}"), + FilterExpression::Not { expression, .. } => write!(f, "!{expression}"), + FilterExpression::Logical { + left, + operator, + right, + .. + } => write!(f, "({left} {operator} {right})"), + FilterExpression::Comparison { + left, + operator, + right, + .. + } => write!(f, "{left} {operator} {right}"), + FilterExpression::RelativeQuery { query, .. } => { + write!(f, "@{}", query.segments) + } + FilterExpression::RootQuery { query, .. } => { + write!(f, "${}", query.segments) + } + FilterExpression::Function { name, args, .. } => { + write!( + f, + "{}({})", + name, + args.iter() + .map(|a| a.to_string()) + .collect::>() + .join(", ") + ) + } + } + } +} + +impl fmt::Display for LogicalOperator { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + LogicalOperator::And => f.write_str("&&"), + LogicalOperator::Or => f.write_str("||"), + } + } +} + +impl fmt::Display for ComparisonOperator { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + ComparisonOperator::Eq => f.write_str("=="), + ComparisonOperator::Ne => f.write_str("!="), + ComparisonOperator::Ge => f.write_str(">="), + ComparisonOperator::Gt => f.write_str(">"), + ComparisonOperator::Le => f.write_str("<="), + ComparisonOperator::Lt => f.write_str("<"), + ComparisonOperator::Contains => f.write_str("contains"), + ComparisonOperator::In => f.write_str("in"), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn parse(expr: &str) -> Query { + Query::standard(expr).unwrap() + } + + fn relative_query(selector: Selector) -> Query { + Query { + segments: Segment::Child { + left: Box::new(Segment::Root {}), + selectors: vec![selector], + }, + } + } + + #[test] + fn query_standard_reports_singular_queries_correctly() { + let cases = [ + ("$", true), + ("$.store", true), + ("$['store']", true), + ("$.store.books[0]", true), + ("$.store.books[0][1]", true), + ("$.store.books[0, 1]", false), + ("$.store.books[*]", false), + ("$.store.books[?(@.available == true)]", false), + ("$.store..title", false), + ]; + + for (expr, expected) in cases { + assert_eq!( + parse(expr).is_singular(), + expected, + "{expr} should have singular={expected}" + ); + } + } + + #[test] + fn filter_expression_is_literal_matches_jsonpath_contract() { + let literal_cases = [ + FilterExpression::True_ {}, + FilterExpression::False_ {}, + FilterExpression::Null {}, + FilterExpression::StringLiteral { + value: "hello".to_owned(), + }, + FilterExpression::Int { value: 42 }, + FilterExpression::Float { value: 3.5 }, + FilterExpression::Array { + values: vec![ + FilterExpression::Int { value: 1 }, + FilterExpression::StringLiteral { + value: "two".to_owned(), + }, + FilterExpression::Array { + values: vec![FilterExpression::Null {}], + }, + ], + }, + ]; + + for expr in literal_cases { + assert!(expr.is_literal(), "{expr:?} should be literal"); + } + + let non_literal_cases = [ + FilterExpression::RelativeQuery { + query: Box::new(relative_query(Selector::Name { + name: "title".to_owned(), + })), + }, + FilterExpression::RootQuery { + query: Box::new(parse("$.store.books[0]")), + }, + FilterExpression::Function { + name: "count".to_owned(), + args: vec![FilterExpression::RelativeQuery { + query: Box::new(relative_query(Selector::Name { + name: "title".to_owned(), + })), + }], + }, + FilterExpression::Not { + expression: Box::new(FilterExpression::True_ {}), + }, + FilterExpression::Logical { + left: Box::new(FilterExpression::True_ {}), + operator: LogicalOperator::And, + right: Box::new(FilterExpression::False_ {}), + }, + FilterExpression::Comparison { + left: Box::new(FilterExpression::Int { value: 1 }), + operator: ComparisonOperator::Eq, + right: Box::new(FilterExpression::Int { value: 2 }), + }, + ]; + + for expr in non_literal_cases { + assert!(!expr.is_literal(), "{expr:?} should not be literal"); + } + } + + #[test] + fn query_and_ast_display_are_stable() { + let query = parse("$.store.books[0]"); + assert_eq!(query.to_string(), "$['store']['books'][0]"); + assert_eq!( + parse("$.store.books[0, 1]").to_string(), + "$['store']['books'][0, 1]" + ); + assert_eq!(parse("$.store..title").to_string(), "$['store']..['title']"); + + let selector_display = [ + ( + Selector::Name { + name: "field".into(), + }, + "'field'", + ), + (Selector::Index { index: -2 }, "-2"), + ( + Selector::Slice { + start: Some(1), + stop: Some(3), + step: None, + }, + "1:3:1", + ), + (Selector::Wild {}, "*"), + ]; + + for (selector, expected) in selector_display { + assert_eq!(selector.to_string(), expected); + } + + let filter_display = FilterExpression::Logical { + left: Box::new(FilterExpression::Comparison { + left: Box::new(FilterExpression::RelativeQuery { + query: Box::new(relative_query(Selector::Name { + name: "available".to_owned(), + })), + }), + operator: ComparisonOperator::Eq, + right: Box::new(FilterExpression::True_ {}), + }), + operator: LogicalOperator::And, + right: Box::new(FilterExpression::Not { + expression: Box::new(FilterExpression::Function { + name: "count".to_owned(), + args: vec![FilterExpression::RootQuery { + query: Box::new(parse("$.store.books[*]")), + }], + }), + }), + }; + assert_eq!( + filter_display.to_string(), + "(@['available'] == true && !count($['store']['books'][*]))" + ); + + assert_eq!(LogicalOperator::And.to_string(), "&&"); + assert_eq!(LogicalOperator::Or.to_string(), "||"); + assert_eq!(ComparisonOperator::Eq.to_string(), "=="); + assert_eq!(ComparisonOperator::Ne.to_string(), "!="); + assert_eq!(ComparisonOperator::Ge.to_string(), ">="); + assert_eq!(ComparisonOperator::Gt.to_string(), ">"); + assert_eq!(ComparisonOperator::Le.to_string(), "<="); + assert_eq!(ComparisonOperator::Lt.to_string(), "<"); + assert_eq!(ComparisonOperator::Contains.to_string(), "contains"); + assert_eq!(ComparisonOperator::In.to_string(), "in"); + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/jsonpath/errors.rs b/3rdparty/loro/crates/loro-internal/src/jsonpath/errors.rs new file mode 100644 index 0000000000..65e13119cb --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/jsonpath/errors.rs @@ -0,0 +1,61 @@ +use std::fmt; + +#[derive(Debug)] +pub enum JSONPathErrorType { + LexerError, + SyntaxError, + TypeError, + NameError, +} + +#[derive(Debug)] +pub struct JSONPathError { + pub kind: JSONPathErrorType, + pub msg: String, +} + +impl JSONPathError { + pub fn new(error: JSONPathErrorType, msg: String) -> Self { + Self { kind: error, msg } + } + + pub fn syntax(msg: String) -> Self { + Self { + kind: JSONPathErrorType::SyntaxError, + msg, + } + } + + pub fn typ(msg: String) -> Self { + Self { + kind: JSONPathErrorType::TypeError, + msg, + } + } + + pub fn name(msg: String) -> Self { + Self { + kind: JSONPathErrorType::NameError, + msg, + } + } +} + +impl fmt::Display for JSONPathErrorType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + JSONPathErrorType::LexerError => f.write_str("lexer error:"), + JSONPathErrorType::SyntaxError => f.write_str("syntax error:"), + JSONPathErrorType::TypeError => f.write_str("type error:"), + JSONPathErrorType::NameError => f.write_str("name error:"), + } + } +} + +impl std::error::Error for JSONPathError {} + +impl fmt::Display for JSONPathError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.msg) + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/jsonpath/jsonpath.pest b/3rdparty/loro/crates/loro-internal/src/jsonpath/jsonpath.pest new file mode 100644 index 0000000000..6a34d33c26 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/jsonpath/jsonpath.pest @@ -0,0 +1,110 @@ +// JSONPath Parser +// This grammar defines the structure for parsing JSONPath expressions. +// It follows a structured approach with grouped rules for clarity and maintainability. + +// Entry Point +// Defines the overall structure of a JSONPath expression, starting with SOI (start of input) and ending with EOI (end of input). +jsonpath = _{ SOI ~ jsonpath_query ~ EOI } + +// Core Query Structure +jsonpath_query = _{ root_identifier ~ segments } + +// Segments +segments = _{ (S ~ segment)* } + +// Whitespace Handling +B = _{ "\x20" | "\x09" | "\x0A" | "\x0D" } // Basic whitespace characters: space, tab, newline, carriage return +S = _{ B* } // Optional whitespace + +// Identifiers +root_identifier = _{ "$" } // Root node identifier +current_node_identifier = _{ "@" } // Current node identifier + +// Segment Types +segment = _{ child_segment | descendant_segment } +child_segment = { bracketed_selection | ("." ~ (wildcard_selector | member_name_shorthand)) } +descendant_segment = { ".." ~ (bracketed_selection | wildcard_selector | member_name_shorthand) } + +// Bracketed Selection +bracketed_selection = { "[" ~ S ~ selector ~ (S ~ "," ~ S ~ selector)* ~ S ~ "]" } + +// Selectors +selector = _{ name_selector | wildcard_selector | slice_selector | index_selector | filter_selector } +wildcard_selector = { "*" } // Matches any element +name_selector = _{ string_literal } // Matches a specific name +index_selector = @{ int } // Matches a specific index +slice_selector = { (start ~ S)? ~ ":" ~ S ~ (stop ~ S)? ~ (":" ~ (S ~ step)?)? } // Array slicing +filter_selector = { "?" ~ S ~ logical_expr } // Filter based on logical expression + +// Integer and Slice Components +int = { "0" | ("-"? ~ ASCII_NONZERO_DIGIT ~ ASCII_DIGIT*) } +start = @{ int } +stop = @{ int } +step = @{ int } + +// String Literals +string_literal = _{ ("\x22" ~ double_quoted ~ "\x22") | ("\x27" ~ single_quoted ~ "\x27") } +double_quoted = { double_quoted_char* } +single_quoted = { single_quoted_char* } +double_quoted_char = _{ unescaped | "\x27" | (ESC ~ "\x22") | (ESC ~ escapable) } +single_quoted_char = _{ unescaped | "\x22" | (ESC ~ "\x27") | (ESC ~ escapable) } +ESC = _{ "\x5C" } // Escape character (backslash) + +// Escapable Characters +escapable = _{ "\x62" | "\x66" | "\x6E" | "\x72" | "\x74" | "/" | "\\" | ("\x75" ~ hexchar) } +hexchar = _{ non_surrogate | (high_surrogate ~ "\\u" ~ low_surrogate) } +non_surrogate = _{ ((ASCII_DIGIT | ^"A" | ^"B" | ^"C" | ^"E" | ^"F") ~ ASCII_HEX_DIGIT{3}) | (^"D" ~ '\u{30}'..'\u{37}' ~ ASCII_HEX_DIGIT{2}) } +high_surrogate = _{ ^"D" ~ ("8" | "9" | ^"A" | ^"B") ~ ASCII_HEX_DIGIT{2} } +low_surrogate = _{ ^"D" ~ (^"C" | ^"D" | ^"E" | ^"F") ~ ASCII_HEX_DIGIT{2} } + +// Unescaped Characters +unescaped = _{ '\u{20}'..'\u{21}' | '\u{23}'..'\u{26}' | '\u{28}'..'\u{5B}' | '\u{5D}'..'\u{D7FF}' | '\u{E000}'..'\u{10FFFF}' } + +// Logical Expressions +logical_expr = _{ logical_or_expr } +logical_or_expr = { logical_and_expr ~ (S ~ "||" ~ S ~ logical_and_expr)* } +logical_and_expr = { basic_expr ~ (S ~ "&&" ~ S ~ basic_expr)* } +basic_expr = _{ paren_expr | comparison_expr | test_expr } +paren_expr = { (logical_not_op ~ S)? ~ "(" ~ S ~ logical_expr ~ S ~ ")" } +logical_not_op = { "!" } +test_expr = { (logical_not_op ~ S)? ~ (filter_query | function_expr) } + +// Queries in Filters +filter_query = _{ rel_query | root_query } +root_query = { jsonpath_query } +rel_query = { current_node_identifier ~ segments } + +// Comparison Expressions +comparison_expr = { comparable ~ S ~ comparison_op ~ S ~ comparable } +comparison_op = { "==" | "!=" | "<=" | ">=" | "<" | ">" | "contains" | "in" } +comparable = _{ literal | singular_query | function_expr } + +// Literals +literal = _{ number | string_literal | array_literal | true_literal | false_literal | null } +true_literal = { "true" } +false_literal = { "false" } +null = { "null" } +number = { (int | "-0") ~ frac? ~ exp? } +frac = { "." ~ ASCII_DIGIT+ } +exp = { ^"e" ~ ("-" | "+")? ~ ASCII_DIGIT+ } +array_literal = { "[" ~ S ~ (literal ~ (S ~ "," ~ S ~ literal)*)? ~ S ~ "]" } + +// Singular Queries +singular_query = _{ rel_singular_query | abs_singular_query } +rel_singular_query = { current_node_identifier ~ singular_query_segments } +abs_singular_query = { root_identifier ~ singular_query_segments } +singular_query_segments = _{ (S ~ (name_segment | index_segment))* } +name_segment = { ("[" ~ name_selector ~ "]") | ("." ~ member_name_shorthand) } +index_segment = { "[" ~ index_selector ~ "]" } + +// Function Expressions +function_expr = { function_name ~ "(" ~ S ~ (function_argument ~ (S ~ "," ~ S ~ function_argument)*)? ~ S ~ ")" } +function_name = { function_name_first ~ function_name_char* } +function_name_first = _{ ASCII_ALPHA_LOWER } +function_name_char = _{ function_name_first | "_" | ASCII_DIGIT } +function_argument = _{ filter_query | logical_expr | function_expr | literal } + +// Member Names +member_name_shorthand = { name_first ~ name_char* } +name_first = _{ ASCII_ALPHA | "_" | '\u{80}'..'\u{D7FF}' | '\u{E000}'..'\u{10FFFF}' } +name_char = _{ name_first | ASCII_DIGIT } \ No newline at end of file diff --git a/3rdparty/loro/crates/loro-internal/src/jsonpath/jsonpath_impl.rs b/3rdparty/loro/crates/loro-internal/src/jsonpath/jsonpath_impl.rs new file mode 100644 index 0000000000..9a9aed7894 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/jsonpath/jsonpath_impl.rs @@ -0,0 +1,1635 @@ +use crate::handler::{ + Handler, ListHandler, MapHandler, MovableListHandler, TextHandler, TreeHandler, ValueOrHandler, +}; +use crate::jsonpath::ast::{ + ComparisonOperator, FilterExpression, LogicalOperator, Segment, Selector, +}; +use crate::jsonpath::JSONPathParser; +use crate::{HandlerTrait, LoroDoc}; +use loro_common::{ContainerID, LoroValue}; +use std::ops::ControlFlow; +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum JsonPathError { + #[error("Invalid JSONPath: {0}")] + InvalidJsonPath(String), + #[error("JSONPath evaluation error: {0}")] + EvaluationError(String), +} +impl LoroDoc { + #[inline] + pub fn jsonpath(&self, jsonpath: &str) -> Result, JsonPathError> { + evaluate_jsonpath(self, jsonpath) + } +} + +enum ExprValue { + Bool(bool), + Value(LoroValue), + Nodes(Vec), +} + +pub fn evaluate_jsonpath( + root: &dyn PathValue, + jsonpath: &str, +) -> Result, JsonPathError> { + let parser = JSONPathParser::new(); + let query = parser + .parse(jsonpath) + .map_err(|e| JsonPathError::InvalidJsonPath(e.to_string()))?; + + let mut results = Vec::new(); + evaluate_segment(root, &query.segments, &mut results); + Ok(results) +} + +fn evaluate_segment(root: &dyn PathValue, segment: &Segment, results: &mut Vec) { + match segment { + Segment::Root {} => { + if let Ok(cloned) = root.clone_this() { + results.push(cloned); + } + } + Segment::Child { left, selectors } => { + if left.is_singular() && matches!(**left, Segment::Root {}) { + apply_selectors(root, root, selectors, results); + } else { + let mut intermediate = Vec::new(); + evaluate_segment(root, left, &mut intermediate); + for node in intermediate { + apply_selectors(root, &node, selectors, results); + } + } + } + Segment::Recursive { left, selectors } => { + if left.is_singular() && matches!(**left, Segment::Root {}) { + recursive_descent(root, root, selectors, results); + } else { + let mut intermediate = Vec::new(); + evaluate_segment(root, left, &mut intermediate); + for node in intermediate { + recursive_descent(root, &node, selectors, results); + } + } + } + } +} + +fn apply_selectors( + root: &dyn PathValue, + node: &dyn PathValue, + selectors: &[Selector], + results: &mut Vec, +) { + for sel in selectors { + match sel { + Selector::Name { name } => { + if let Some(child) = node.get_by_key(name) { + results.push(child); + } + } + Selector::Index { index } => { + let mut idx = *index; + if idx < 0 { + idx += node.length_for_path() as i64; + } + if idx >= 0 { + if let Some(child) = node.get_by_index(idx as isize) { + results.push(child); + } + } + } + Selector::Slice { start, stop, step } => { + let len = node.length_for_path() as i64; + let eff_start = start.unwrap_or(if step.unwrap_or(1) > 0 { 0 } else { len - 1 }); + let eff_stop = stop.unwrap_or(if step.unwrap_or(1) > 0 { len } else { -len - 1 }); + let eff_step = step.unwrap_or(1); + + if eff_step == 0 { + continue; + } + + let mut start_idx = if eff_start < 0 { + eff_start + len + } else { + eff_start + }; + let mut stop_idx = if eff_stop < 0 { + eff_stop + len + } else { + eff_stop + }; + start_idx = start_idx.max(0).min(len); + stop_idx = stop_idx.max(-1).min(len); + + // Pre-allocate if positive step for better perf + if eff_step > 0 { + let approx_count = ((stop_idx - start_idx) / eff_step).max(0) as usize; + results.reserve(approx_count); + let mut i = start_idx; + while i < stop_idx { + if let Some(child) = node.get_by_index(i as isize) { + results.push(child); + } + i += eff_step; + } + } else { + let approx_count = ((start_idx - stop_idx) / -eff_step).max(0) as usize; + results.reserve(approx_count); + let mut i = start_idx; + while i > stop_idx { + if let Some(child) = node.get_by_index(i as isize) { + results.push(child); + } + i += eff_step; + } + } + } + Selector::Wild {} => { + node.for_each_for_path(&mut |child| { + results.push(child); + ControlFlow::Continue(()) + }); + } + Selector::Filter { expression } => { + node.for_each_for_path(&mut |child| { + if eval_filter_expr(root, &child, expression) { + results.push(child); + } + ControlFlow::Continue(()) + }); + } + } + } +} + +fn recursive_descent( + root: &dyn PathValue, + node: &dyn PathValue, + selectors: &[Selector], + results: &mut Vec, +) { + // 1. apply selectors to the *current* node + apply_selectors(root, node, selectors, results); + + // 2. recurse into children + node.for_each_for_path(&mut |child| { + recursive_descent(root, &child, selectors, results); + ControlFlow::Continue(()) + }); +} + +fn eval_filter_expr( + root: &dyn PathValue, + current: &dyn PathValue, + expr: &FilterExpression, +) -> bool { + to_logical(eval_expr(root, current, expr)) +} + +fn eval_expr(root: &dyn PathValue, current: &dyn PathValue, expr: &FilterExpression) -> ExprValue { + match expr { + FilterExpression::True_ {} => ExprValue::Bool(true), + FilterExpression::False_ {} => ExprValue::Bool(false), + FilterExpression::Null {} => ExprValue::Value(LoroValue::Null), + FilterExpression::StringLiteral { value } => { + ExprValue::Value(LoroValue::String(value.clone().into())) + } + FilterExpression::Int { value } => ExprValue::Value(LoroValue::I64(*value)), + FilterExpression::Float { value } => ExprValue::Value(LoroValue::Double(*value)), + FilterExpression::Array { values } => { + let mut list = Vec::with_capacity(values.len()); + for val in values { + match eval_expr(root, current, val) { + ExprValue::Value(v) => list.push(v), + ExprValue::Bool(b) => list.push(LoroValue::Bool(b)), + ExprValue::Nodes(_) => return ExprValue::Value(LoroValue::Null), + } + } + ExprValue::Value(LoroValue::List(list.into())) + } + FilterExpression::Not { expression } => { + ExprValue::Bool(!to_logical(eval_expr(root, current, expression))) + } + FilterExpression::Logical { + left, + operator, + right, + } => { + // Short-circuit evaluation + let l = to_logical(eval_expr(root, current, left)); + match operator { + LogicalOperator::And => { + if !l { + return ExprValue::Bool(false); + } + ExprValue::Bool(to_logical(eval_expr(root, current, right))) + } + LogicalOperator::Or => { + if l { + return ExprValue::Bool(true); + } + ExprValue::Bool(to_logical(eval_expr(root, current, right))) + } + } + } + FilterExpression::Comparison { + left, + operator, + right, + } => { + let l = eval_expr(root, current, left); + let r = eval_expr(root, current, right); + ExprValue::Bool(compare_expr(l, operator, r)) + } + FilterExpression::RelativeQuery { query } => { + let mut query_results = Vec::new(); + evaluate_segment(current, &query.segments, &mut query_results); + ExprValue::Nodes(query_results) + } + FilterExpression::RootQuery { query } => { + let mut query_results = Vec::new(); + evaluate_segment(root, &query.segments, &mut query_results); + ExprValue::Nodes(query_results) + } + FilterExpression::Function { name, args } => eval_function(root, current, name, args), + } +} + +#[inline] +fn eval_function( + root: &dyn PathValue, + current: &dyn PathValue, + name: &str, + args: &[FilterExpression], +) -> ExprValue { + match name { + "count" if args.len() == 1 => { + let arg = eval_expr(root, current, &args[0]); + if let ExprValue::Nodes(ns) = arg { + ExprValue::Value(LoroValue::I64(ns.len() as i64)) + } else { + ExprValue::Value(LoroValue::I64(0)) + } + } + "length" if args.len() == 1 => { + let arg = eval_expr(root, current, &args[0]); + if let ExprValue::Value(v) = arg { + let len = match v { + LoroValue::List(l) => l.len(), + LoroValue::Map(m) => m.len(), + LoroValue::String(s) => s.len(), + _ => 0, + }; + ExprValue::Value(LoroValue::I64(len as i64)) + } else { + ExprValue::Value(LoroValue::I64(0)) + } + } + "value" if args.len() == 1 => { + let arg = eval_expr(root, current, &args[0]); + if let ExprValue::Nodes(ns) = arg { + if ns.len() == 1 { + match &ns[0] { + ValueOrHandler::Value(v) => ExprValue::Value(v.clone()), + ValueOrHandler::Handler(h) => ExprValue::Value(h.get_value()), + } + } else { + ExprValue::Value(LoroValue::Null) + } + } else { + ExprValue::Value(LoroValue::Null) + } + } + "match" => { + unimplemented!("JSONPath function `match()` is declared but not implemented"); + } + "search" => { + unimplemented!("JSONPath function `search()` is declared but not implemented"); + } + other => { + unimplemented!( + "JSONPath function `{}` is declared but does not have an evaluator", + other + ); + } + } +} + +#[inline(always)] +fn to_logical(v: ExprValue) -> bool { + match v { + ExprValue::Bool(b) => b, + ExprValue::Value(v) => match v { + LoroValue::Bool(b) => b, + LoroValue::Null => false, + LoroValue::String(s) => !s.is_empty(), + LoroValue::I64(0) => false, + LoroValue::I64(_) => true, + LoroValue::Double(f) => f != 0.0 && !f.is_nan(), + LoroValue::List(l) => !l.is_empty(), + LoroValue::Map(m) => !m.is_empty(), + _ => false, + }, + ExprValue::Nodes(ns) => !ns.is_empty(), + } +} + +#[inline] +fn compare_expr(l: ExprValue, op: &ComparisonOperator, r: ExprValue) -> bool { + match (l, r) { + (ExprValue::Value(l), ExprValue::Value(r)) => compare_values(&l, op, &r), + (ExprValue::Value(l), ExprValue::Bool(r)) => compare_values(&l, op, &LoroValue::Bool(r)), + (ExprValue::Bool(l), ExprValue::Value(r)) => compare_values(&LoroValue::Bool(l), op, &r), + (ExprValue::Bool(l), ExprValue::Bool(r)) => compare_bools(l, op, r), + (ExprValue::Nodes(ls), ExprValue::Nodes(rs)) => { + // Optimized: check if either is empty first + if ls.is_empty() || rs.is_empty() { + return false; + } + ls.iter().any(|a| { + let a_val = get_value(a); + rs.iter().any(|b| compare_values(&a_val, op, &get_value(b))) + }) + } + (ExprValue::Nodes(ls), v) => { + if ls.is_empty() { + return false; + } + let r_val = get_other_value(&v); + ls.iter().any(|a| compare_values(&get_value(a), op, &r_val)) + } + (v, ExprValue::Nodes(rs)) => { + if rs.is_empty() { + return false; + } + let l_val = get_other_value(&v); + rs.iter().any(|b| compare_values(&l_val, op, &get_value(b))) + } + } +} + +#[inline(always)] +fn compare_values(l: &LoroValue, op: &ComparisonOperator, r: &LoroValue) -> bool { + match op { + ComparisonOperator::In => { + if let LoroValue::List(list) = r { + list.iter() + .any(|item| compare_values(l, &ComparisonOperator::Eq, item)) + } else { + false + } + } + _ => match (l, r) { + (LoroValue::Double(a), LoroValue::Double(b)) => compare_nums(*a, op, *b), + (LoroValue::I64(a), LoroValue::I64(b)) => compare_i64(*a, op, *b), + (LoroValue::Double(a), LoroValue::I64(b)) => compare_nums(*a, op, *b as f64), + (LoroValue::I64(a), LoroValue::Double(b)) => compare_nums(*a as f64, op, *b), + (LoroValue::String(a), LoroValue::String(b)) => { + compare_strs(a.as_ref(), op, b.as_ref()) + } + (LoroValue::Bool(a), LoroValue::Bool(b)) => compare_bools(*a, op, *b), + (LoroValue::Null, LoroValue::Null) => matches!(op, ComparisonOperator::Eq), + _ => false, + }, + } +} + +#[inline(always)] +fn compare_i64(a: i64, op: &ComparisonOperator, b: i64) -> bool { + match op { + ComparisonOperator::Eq => a == b, + ComparisonOperator::Ne => a != b, + ComparisonOperator::Lt => a < b, + ComparisonOperator::Le => a <= b, + ComparisonOperator::Gt => a > b, + ComparisonOperator::Ge => a >= b, + _ => false, + } +} + +#[inline(always)] +fn compare_nums(a: f64, op: &ComparisonOperator, b: f64) -> bool { + match op { + ComparisonOperator::Eq => (a - b).abs() < f64::EPSILON, + ComparisonOperator::Ne => (a - b).abs() >= f64::EPSILON, + ComparisonOperator::Lt => a < b, + ComparisonOperator::Le => a <= b, + ComparisonOperator::Gt => a > b, + ComparisonOperator::Ge => a >= b, + _ => false, + } +} + +#[inline(always)] +fn compare_strs(a: &str, op: &ComparisonOperator, b: &str) -> bool { + match op { + ComparisonOperator::Eq => a == b, + ComparisonOperator::Ne => a != b, + ComparisonOperator::Lt => a < b, + ComparisonOperator::Le => a <= b, + ComparisonOperator::Gt => a > b, + ComparisonOperator::Ge => a >= b, + ComparisonOperator::Contains => a.contains(b), + _ => false, + } +} + +#[inline(always)] +fn compare_bools(a: bool, op: &ComparisonOperator, b: bool) -> bool { + match op { + ComparisonOperator::Eq => a == b, + ComparisonOperator::Ne => a != b, + _ => false, + } +} + +#[inline(always)] +fn get_value(node: &ValueOrHandler) -> LoroValue { + match node { + ValueOrHandler::Value(v) => v.clone(), + ValueOrHandler::Handler(h) => h.get_value(), + } +} + +#[inline(always)] +fn get_other_value(v: &ExprValue) -> LoroValue { + match v { + ExprValue::Value(v) => v.clone(), + ExprValue::Bool(b) => LoroValue::Bool(*b), + _ => LoroValue::Null, + } +} + +// Implement necessary trait bounds for PathValue +pub trait PathValue { + fn get_by_key(&self, key: &str) -> Option; + fn get_by_index(&self, index: isize) -> Option; + fn for_each_for_path(&self, f: &mut dyn FnMut(ValueOrHandler) -> ControlFlow<()>); + fn length_for_path(&self) -> usize; + fn get_child_by_id(&self, id: ContainerID) -> Option; + fn clone_this(&self) -> Result; +} + +// Implement PathValue for ValueOrHandler +impl PathValue for ValueOrHandler { + fn get_by_key(&self, key: &str) -> Option { + match self { + ValueOrHandler::Value(v) => v.get_by_key(key).cloned().map(ValueOrHandler::Value), + ValueOrHandler::Handler(h) => h.get_by_key(key), + } + } + + fn get_by_index(&self, index: isize) -> Option { + match self { + ValueOrHandler::Value(v) => v.get_by_index(index).cloned().map(ValueOrHandler::Value), + ValueOrHandler::Handler(h) => h.get_by_index(index), + } + } + + fn for_each_for_path(&self, f: &mut dyn FnMut(ValueOrHandler) -> ControlFlow<()>) { + match self { + ValueOrHandler::Value(v) => v.for_each_for_path(f), + ValueOrHandler::Handler(h) => h.for_each_for_path(f), + } + } + + fn length_for_path(&self) -> usize { + match self { + ValueOrHandler::Value(v) => v.length_for_path(), + ValueOrHandler::Handler(h) => h.length_for_path(), + } + } + + fn get_child_by_id(&self, id: ContainerID) -> Option { + match self { + ValueOrHandler::Handler(h) => h.get_child_by_id(id), + _ => None, + } + } + + fn clone_this(&self) -> Result { + match self { + ValueOrHandler::Value(v) => Ok(ValueOrHandler::Value(v.clone())), + ValueOrHandler::Handler(h) => Ok(ValueOrHandler::Handler(h.clone())), + } + } +} + +// Implement PathValue for LoroDoc +impl PathValue for LoroDoc { + fn get_by_key(&self, key: &str) -> Option { + self.get_by_str_path(key) + } + + fn get_by_index(&self, _index: isize) -> Option { + None // LoroDoc doesn't support index-based access + } + + fn for_each_for_path(&self, f: &mut dyn FnMut(ValueOrHandler) -> ControlFlow<()>) { + let roots = self.state.lock().preferred_root_containers(); + let arena = self.arena(); + for c in roots { + let cid = arena.idx_to_id(c).unwrap(); + let h = self.get_handler(cid).unwrap(); + if f(ValueOrHandler::Handler(h)) == ControlFlow::Break(()) { + break; + } + } + } + + fn length_for_path(&self) -> usize { + self.state.lock().preferred_root_containers().len() + } + + fn get_child_by_id(&self, id: ContainerID) -> Option { + self.get_handler(id) + } + + fn clone_this(&self) -> Result { + Ok(ValueOrHandler::Value(self.get_deep_value())) + } +} + +// Implement PathValue for Handler +impl PathValue for Handler { + fn get_by_key(&self, key: &str) -> Option { + match self { + Handler::Map(h) => h.get_by_key(key), + Handler::Tree(h) => h.get_by_key(key), + _ => None, + } + } + + fn get_by_index(&self, index: isize) -> Option { + match self { + Handler::List(h) => h.get_by_index(index), + Handler::MovableList(h) => h.get_by_index(index), + _ => None, + } + } + + fn for_each_for_path(&self, f: &mut dyn FnMut(ValueOrHandler) -> ControlFlow<()>) { + match self { + Handler::Map(h) => h.for_each_for_path(f), + Handler::List(h) => h.for_each_for_path(f), + Handler::MovableList(h) => h.for_each_for_path(f), + Handler::Tree(h) => h.for_each_for_path(f), + _ => {} + } + } + + fn length_for_path(&self) -> usize { + match self { + Handler::Map(h) => h.length_for_path(), + Handler::List(h) => h.length_for_path(), + Handler::MovableList(h) => h.length_for_path(), + Handler::Tree(h) => h.length_for_path(), + Handler::Text(h) => h.length_for_path(), + _ => 0, + } + } + + fn get_child_by_id(&self, id: ContainerID) -> Option { + match self { + Handler::Map(h) => h.get_child_by_id(id), + Handler::List(h) => h.get_child_by_id(id), + Handler::MovableList(h) => h.get_child_by_id(id), + Handler::Tree(h) => h.get_child_by_id(id), + _ => None, + } + } + + fn clone_this(&self) -> Result { + Ok(ValueOrHandler::Handler(self.clone())) + } +} + +// Implementations for specific handlers +impl PathValue for MapHandler { + fn get_by_key(&self, key: &str) -> Option { + self.get_(key) + } + + fn get_by_index(&self, _index: isize) -> Option { + None + } + + fn for_each_for_path(&self, f: &mut dyn FnMut(ValueOrHandler) -> ControlFlow<()>) { + let mut done = false; + self.for_each(|_, v| { + if done { + return; + } + + if let ControlFlow::Break(_) = f(v) { + done = true; + } + }); + } + + fn length_for_path(&self) -> usize { + self.len() + } + + fn get_child_by_id(&self, id: ContainerID) -> Option { + self.get_child_handler(id.to_string().as_str()).ok() + } + + fn clone_this(&self) -> Result { + Ok(ValueOrHandler::Handler(Handler::Map(self.clone()))) + } +} + +impl PathValue for ListHandler { + fn get_by_key(&self, _key: &str) -> Option { + None + } + + fn get_by_index(&self, index: isize) -> Option { + if index < 0 { + self.get_(self.len() - (-index) as usize) + } else { + self.get_(index as usize) + } + } + + fn for_each_for_path(&self, f: &mut dyn FnMut(ValueOrHandler) -> ControlFlow<()>) { + let mut done = false; + self.for_each(|v| { + if done { + return; + } + + if let ControlFlow::Break(_) = f(v) { + done = true; + } + }); + } + + fn length_for_path(&self) -> usize { + self.len() + } + + fn get_child_by_id(&self, _id: ContainerID) -> Option { + unimplemented!() + } + + fn clone_this(&self) -> Result { + Ok(ValueOrHandler::Handler(Handler::List(self.clone()))) + } +} + +impl PathValue for MovableListHandler { + fn get_by_key(&self, _key: &str) -> Option { + None + } + + fn get_by_index(&self, index: isize) -> Option { + if index < 0 { + if self.len() > (-index) as usize { + self.get_(self.len() - (-index) as usize) + } else { + None + } + } else { + self.get_(index as usize) + } + } + + fn for_each_for_path(&self, f: &mut dyn FnMut(ValueOrHandler) -> ControlFlow<()>) { + let mut done = false; + self.for_each(|v| { + if done { + return; + } + + if let ControlFlow::Break(_) = f(v) { + done = true; + } + }) + } + + fn length_for_path(&self) -> usize { + self.len() + } + + fn get_child_by_id(&self, _id: ContainerID) -> Option { + unimplemented!() + } + + fn clone_this(&self) -> Result { + Ok(ValueOrHandler::Handler(Handler::MovableList(self.clone()))) + } +} + +impl PathValue for TextHandler { + fn get_by_key(&self, _key: &str) -> Option { + None + } + + fn get_by_index(&self, _index: isize) -> Option { + None + } + + fn for_each_for_path(&self, _f: &mut dyn FnMut(ValueOrHandler) -> ControlFlow<()>) { + // TextHandler doesn't have children to iterate over + } + + fn length_for_path(&self) -> usize { + self.len_unicode() + } + + fn get_child_by_id(&self, _id: ContainerID) -> Option { + None + } + + fn clone_this(&self) -> Result { + Ok(ValueOrHandler::Handler(Handler::Text(self.clone()))) + } +} + +impl PathValue for TreeHandler { + fn get_by_key(&self, _key: &str) -> Option { + None + } + + fn get_by_index(&self, _index: isize) -> Option { + None + } + + fn for_each_for_path(&self, _f: &mut dyn FnMut(ValueOrHandler) -> ControlFlow<()>) { + unimplemented!() + } + + fn length_for_path(&self) -> usize { + unimplemented!() + } + + fn get_child_by_id(&self, _id: ContainerID) -> Option { + unimplemented!() + } + + fn clone_this(&self) -> Result { + Ok(ValueOrHandler::Handler(Handler::Tree(self.clone()))) + } +} + +impl PathValue for LoroValue { + fn get_by_key(&self, key: &str) -> Option { + match self { + LoroValue::Map(map) => map.get(key).map(|v| ValueOrHandler::Value(v.clone())), + _ => None, + } + } + + fn get_by_index(&self, index: isize) -> Option { + match self { + LoroValue::List(list) => { + let index = if index < 0 { + if list.len() > (-index) as usize { + list.len() - (-index) as usize + } else { + return None; + } + } else { + index as usize + }; + list.get(index).map(|v| ValueOrHandler::Value(v.clone())) + } + _ => None, + } + } + + fn for_each_for_path(&self, f: &mut dyn FnMut(ValueOrHandler) -> ControlFlow<()>) { + match self { + LoroValue::List(list) => { + for item in list.iter() { + if let ControlFlow::Break(_) = f(ValueOrHandler::Value(item.clone())) { + break; + } + } + } + LoroValue::Map(map) => { + for (_, value) in map.iter() { + if let ControlFlow::Break(_) = f(ValueOrHandler::Value(value.clone())) { + break; + } + } + } + _ => {} + } + } + + fn length_for_path(&self) -> usize { + match self { + LoroValue::List(list) => list.len(), + LoroValue::Map(map) => map.len(), + LoroValue::String(s) => s.len(), + _ => 0, + } + } + + fn get_child_by_id(&self, _id: ContainerID) -> Option { + None + } + + fn clone_this(&self) -> Result { + Ok(ValueOrHandler::Value(self.clone())) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ListHandler, LoroDoc, LoroValue, MapHandler}; + + fn setup_test_doc() -> LoroDoc { + let doc = LoroDoc::new(); + doc.start_auto_commit(); + let store = doc.get_map("store"); + let books = store + .insert_container("books", ListHandler::new_detached()) + .unwrap(); + + // Book 1: 1984 + let book = books + .insert_container(0, MapHandler::new_detached()) + .unwrap(); + book.insert("title", "1984").unwrap(); + book.insert("author", "George Orwell").unwrap(); + book.insert("price", 10).unwrap(); + book.insert("available", true).unwrap(); + book.insert("isbn", "978-0451524935").unwrap(); + + // Book 2: Animal Farm + let book = books + .insert_container(1, MapHandler::new_detached()) + .unwrap(); + book.insert("title", "Animal Farm").unwrap(); + book.insert("author", "George Orwell").unwrap(); + book.insert("price", 8).unwrap(); + book.insert("available", true).unwrap(); + + // Book 3: Brave New World + let book = books + .insert_container(2, MapHandler::new_detached()) + .unwrap(); + book.insert("title", "Brave New World").unwrap(); + book.insert("author", "Aldous Huxley").unwrap(); + book.insert("price", 12).unwrap(); + book.insert("available", false).unwrap(); + book.insert("isbn", "978-0060850524").unwrap(); + + // Book 4: Fahrenheit 451 + let book = books + .insert_container(3, MapHandler::new_detached()) + .unwrap(); + book.insert("title", "Fahrenheit 451").unwrap(); + book.insert("author", "Ray Bradbury").unwrap(); + book.insert("price", 9).unwrap(); + book.insert("available", true).unwrap(); + book.insert("isbn", "978-1451673319").unwrap(); + + // Book 5: The Great Gatsby + let book = books + .insert_container(4, MapHandler::new_detached()) + .unwrap(); + book.insert("title", "The Great Gatsby").unwrap(); + book.insert("author", "F. Scott Fitzgerald").unwrap(); + book.insert("price", LoroValue::Null).unwrap(); + book.insert("available", true).unwrap(); + + // Book 6: To Kill a Mockingbird + let book = books + .insert_container(5, MapHandler::new_detached()) + .unwrap(); + book.insert("title", "To Kill a Mockingbird").unwrap(); + book.insert("author", "Harper Lee").unwrap(); + book.insert("price", 11).unwrap(); + book.insert("available", true).unwrap(); + + // Book 7: The Catcher in the Rye + let book = books + .insert_container(6, MapHandler::new_detached()) + .unwrap(); + book.insert("title", "The Catcher in the Rye").unwrap(); + book.insert("author", "J.D. Salinger").unwrap(); + book.insert("price", 10).unwrap(); + book.insert("available", false).unwrap(); + + // Book 8: Lord of the Flies + let book = books + .insert_container(7, MapHandler::new_detached()) + .unwrap(); + book.insert("title", "Lord of the Flies").unwrap(); + book.insert("author", "William Golding").unwrap(); + book.insert("price", 9).unwrap(); + book.insert("available", true).unwrap(); + + // Book 9: Pride and Prejudice + let book = books + .insert_container(8, MapHandler::new_detached()) + .unwrap(); + book.insert("title", "Pride and Prejudice").unwrap(); + book.insert("author", "Jane Austen").unwrap(); + book.insert("price", 7).unwrap(); + book.insert("available", true).unwrap(); + + // Book 10: The Hobbit + let book = books + .insert_container(9, MapHandler::new_detached()) + .unwrap(); + book.insert("title", "The Hobbit").unwrap(); + book.insert("author", "J.R.R. Tolkien").unwrap(); + book.insert("price", 14).unwrap(); + book.insert("available", true).unwrap(); + + // Additional metadata + store.insert("featured_author", "George Orwell").unwrap(); + let featured_authors = store + .insert_container("featured_authors", ListHandler::new_detached()) + .unwrap(); + featured_authors.push("George Orwell").unwrap(); + featured_authors.push("Aldous Huxley").unwrap(); + featured_authors.push("Ray Bradbury").unwrap(); + store.insert("min_price", 10).unwrap(); + doc + } + + mod jsonpath_selectors { + use super::*; + + #[test] + fn handles_child_selectors() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[0].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 1); + assert_eq!( + &**result[0].as_value().unwrap().as_string().unwrap(), + "1984" + ); + Ok(()) + } + + #[test] + fn handles_wildcard_selector() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[*].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 10); + let mut titles: Vec<&str> = result + .iter() + .map(|v| v.as_value().unwrap().as_string().unwrap().as_str()) + .collect(); + titles.sort(); + let mut expected = vec![ + "1984", + "Animal Farm", + "Brave New World", + "Fahrenheit 451", + "The Great Gatsby", + "To Kill a Mockingbird", + "The Catcher in the Rye", + "Lord of the Flies", + "Pride and Prejudice", + "The Hobbit", + ]; + expected.sort(); + assert_eq!(titles, expected); + Ok(()) + } + + #[test] + fn handles_recursive_descent() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$..title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 10); + Ok(()) + } + + #[test] + fn handles_quoted_keys() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store['books'][0]['title']"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 1); + assert_eq!( + &**result[0].as_value().unwrap().as_string().unwrap(), + "1984" + ); + Ok(()) + } + } + + mod string_filters { + use super::*; + + #[test] + fn filters_by_exact_string_match() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[?(@.title == '1984')].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 1); + assert_eq!( + &**result[0].as_value().unwrap().as_string().unwrap(), + "1984" + ); + Ok(()) + } + + #[test] + fn filters_by_string_contains() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[?(@.title contains 'Farm')].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 1); + assert_eq!( + &**result[0].as_value().unwrap().as_string().unwrap(), + "Animal Farm" + ); + Ok(()) + } + + #[test] + fn filters_by_recursive_string_match() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$..[?(@.author contains 'Orwell')].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 2); + let mut titles: Vec<&str> = result + .iter() + .map(|v| v.as_value().unwrap().as_string().unwrap().as_str()) + .collect(); + titles.sort(); + let mut expected = vec!["1984", "Animal Farm"]; + expected.sort(); + assert_eq!(titles, expected); + Ok(()) + } + } + + mod logical_operators { + use super::*; + + #[test] + fn filters_with_and_operator() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[?(@.author == 'George Orwell' && @.price < 10)].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 1); + assert_eq!( + &**result[0].as_value().unwrap().as_string().unwrap(), + "Animal Farm" + ); + Ok(()) + } + + #[test] + fn filters_with_or_operator() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[?(@.author == 'George Orwell' || @.price >= 10)].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 6); + let mut titles: Vec<&str> = result + .iter() + .map(|v| v.as_value().unwrap().as_string().unwrap().as_str()) + .collect(); + titles.sort(); + let mut expected = vec![ + "1984", + "Animal Farm", + "Brave New World", + "To Kill a Mockingbird", + "The Catcher in the Rye", + "The Hobbit", + ]; + expected.sort(); + assert_eq!(titles, expected); + Ok(()) + } + + #[test] + fn filters_with_complex_and_or_combination() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[?(@.author == 'George Orwell' && (@.price < 10 || @.available == true))].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 2); + let mut titles: Vec<&str> = result + .iter() + .map(|v| v.as_value().unwrap().as_string().unwrap().as_str()) + .collect(); + titles.sort(); + let mut expected = vec!["1984", "Animal Farm"]; + expected.sort(); + assert_eq!(titles, expected); + Ok(()) + } + + #[test] + fn filters_with_not_operator() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[?(!(@.available == false))].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 8); + let mut titles: Vec<&str> = result + .iter() + .map(|v| v.as_value().unwrap().as_string().unwrap().as_str()) + .collect(); + titles.sort(); + let mut expected = vec![ + "1984", + "Animal Farm", + "Fahrenheit 451", + "The Great Gatsby", + "To Kill a Mockingbird", + "Lord of the Flies", + "Pride and Prejudice", + "The Hobbit", + ]; + expected.sort(); + assert_eq!(titles, expected); + Ok(()) + } + } + + mod in_operator { + use super::*; + + #[test] + fn filters_by_author_in_list() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[?(@.author in ['George Orwell', 'Jane Austen'])].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 3); + let mut titles: Vec<&str> = result + .iter() + .map(|v| v.as_value().unwrap().as_string().unwrap().as_str()) + .collect(); + titles.sort(); + let mut expected = vec!["1984", "Animal Farm", "Pride and Prejudice"]; + expected.sort(); + assert_eq!(titles, expected); + Ok(()) + } + + #[test] + fn filters_by_price_in_list() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[?(@.price in [7, 10, 14])].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 4); + let mut titles: Vec<&str> = result + .iter() + .map(|v| v.as_value().unwrap().as_string().unwrap().as_str()) + .collect(); + titles.sort(); + let mut expected = vec![ + "1984", + "Pride and Prejudice", + "The Catcher in the Rye", + "The Hobbit", + ]; + expected.sort(); + assert_eq!(titles, expected); + Ok(()) + } + + #[test] + fn filters_with_in_operator_and_null_values() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[?(@.price in [null, 9])].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 3); + let mut titles: Vec<&str> = result + .iter() + .map(|v| v.as_value().unwrap().as_string().unwrap().as_str()) + .collect(); + titles.sort(); + let mut expected = vec!["Fahrenheit 451", "Lord of the Flies", "The Great Gatsby"]; + expected.sort(); + assert_eq!(titles, expected); + Ok(()) + } + + #[test] + fn filters_with_in_operator_in_recursive_descent() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$..[?(@.author in ['George Orwell', 'Ray Bradbury'])].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 3); + let mut titles: Vec<&str> = result + .iter() + .map(|v| v.as_value().unwrap().as_string().unwrap().as_str()) + .collect(); + titles.sort(); + let mut expected = vec!["1984", "Animal Farm", "Fahrenheit 451"]; + expected.sort(); + assert_eq!(titles, expected); + Ok(()) + } + + #[test] + fn filters_with_root_list_in() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[?(@.author in $.store.featured_authors)].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 4); + let mut titles: Vec<&str> = result + .iter() + .map(|v| v.as_value().unwrap().as_string().unwrap().as_str()) + .collect(); + titles.sort(); + let mut expected = vec!["1984", "Animal Farm", "Brave New World", "Fahrenheit 451"]; + expected.sort(); + assert_eq!(titles, expected); + Ok(()) + } + } + + mod union_and_slice_operations { + use super::*; + + #[test] + fn handles_union_indexes() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[0,2].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 2); + assert_eq!( + &**result[0].as_value().unwrap().as_string().unwrap(), + "1984" + ); + assert_eq!( + &**result[1].as_value().unwrap().as_string().unwrap(), + "Brave New World" + ); + Ok(()) + } + + #[test] + fn handles_union_keys() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[0]['title','author']"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 2); + assert_eq!( + &**result[0].as_value().unwrap().as_string().unwrap(), + "1984" + ); + assert_eq!( + &**result[1].as_value().unwrap().as_string().unwrap(), + "George Orwell" + ); + Ok(()) + } + + #[test] + fn handles_union_with_negative_indexes() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[-2,-1].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 2); + assert_eq!( + &**result[0].as_value().unwrap().as_string().unwrap(), + "Pride and Prejudice" + ); + assert_eq!( + &**result[1].as_value().unwrap().as_string().unwrap(), + "The Hobbit" + ); + Ok(()) + } + + #[test] + fn handles_basic_slice() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[0:3].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 3); + let titles: Vec<&str> = result + .iter() + .map(|v| v.as_value().unwrap().as_string().unwrap().as_str()) + .collect(); + assert_eq!(titles, vec!["1984", "Animal Farm", "Brave New World"]); + Ok(()) + } + + #[test] + fn handles_slice_with_step() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[0:5:2].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 3); + let titles: Vec<&str> = result + .iter() + .map(|v| v.as_value().unwrap().as_string().unwrap().as_str()) + .collect(); + assert_eq!(titles, vec!["1984", "Brave New World", "The Great Gatsby"]); + Ok(()) + } + + #[test] + fn handles_negative_slice() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[-2:].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 2); + let titles: Vec<&str> = result + .iter() + .map(|v| v.as_value().unwrap().as_string().unwrap().as_str()) + .collect(); + assert_eq!(titles, vec!["Pride and Prejudice", "The Hobbit"]); + Ok(()) + } + } + + mod complex_and_recursive_filters { + use super::*; + + #[test] + fn filters_with_multiple_conditions() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[?(@.price >= 10 && @.available == true && @.title contains '1984')].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 1); + assert_eq!( + &**result[0].as_value().unwrap().as_string().unwrap(), + "1984" + ); + Ok(()) + } + + #[test] + fn filters_with_path_expressions() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[?(@.author == 'George Orwell' && @.title != '1984')].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 1); + assert_eq!( + &**result[0].as_value().unwrap().as_string().unwrap(), + "Animal Farm" + ); + Ok(()) + } + + #[test] + fn filters_with_null_checks() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[?(@.price == null || @.price < 10)].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 5); + let mut titles: Vec<&str> = result + .iter() + .map(|v| v.as_value().unwrap().as_string().unwrap().as_str()) + .collect(); + titles.sort(); + let mut expected = vec![ + "Animal Farm", + "Fahrenheit 451", + "The Great Gatsby", + "Pride and Prejudice", + "Lord of the Flies", + ]; + expected.sort(); + assert_eq!(titles, expected); + Ok(()) + } + + #[test] + fn handles_recursive_filter_with_price_condition() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$..[?(@.price > 10)].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 3); + let mut titles: Vec<&str> = result + .iter() + .map(|v| v.as_value().unwrap().as_string().unwrap().as_str()) + .collect(); + titles.sort(); + let mut expected = vec!["Brave New World", "To Kill a Mockingbird", "The Hobbit"]; + expected.sort(); + assert_eq!(titles, expected); + Ok(()) + } + + #[test] + fn handles_recursive_filter_with_logical_operators() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$..[?(@.author == 'George Orwell' || @.price > 10)].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 5); + let mut titles: Vec<&str> = result + .iter() + .map(|v| v.as_value().unwrap().as_string().unwrap().as_str()) + .collect(); + titles.sort(); + let mut expected = vec![ + "1984", + "Animal Farm", + "Brave New World", + "To Kill a Mockingbird", + "The Hobbit", + ]; + expected.sort(); + assert_eq!(titles, expected); + Ok(()) + } + } + + mod edge_cases_and_error_handling { + use super::*; + + #[test] + fn handles_quoted_keys_with_special_characters() -> Result<(), JsonPathError> { + let doc = LoroDoc::new(); + doc.start_auto_commit(); + let map = doc.get_map("store"); + let book = map + .insert_container("book-with-dash", MapHandler::new_detached()) + .unwrap(); + book.insert("price-$10", "cheap").unwrap(); + let path = "$['store']['book-with-dash']['price-$10']"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 1); + assert_eq!( + &**result[0].as_value().unwrap().as_string().unwrap(), + "cheap" + ); + Ok(()) + } + + #[test] + fn handles_quoted_keys_with_escaped_quotes() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[?(@.author == 'George Orwell')].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 2); + let mut titles: Vec<&str> = result + .iter() + .map(|v| v.as_value().unwrap().as_string().unwrap().as_str()) + .collect(); + titles.sort(); + let mut expected = vec!["1984", "Animal Farm"]; + expected.sort(); + assert_eq!(titles, expected); + Ok(()) + } + } + + mod root_filters { + use super::*; + + #[test] + fn filters_with_root_reference() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[?(@.author == $.store.featured_author)].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 2); + let mut titles: Vec<&str> = result + .iter() + .map(|v| v.as_value().unwrap().as_string().unwrap().as_str()) + .collect(); + titles.sort(); + let mut expected = vec!["1984", "Animal Farm"]; + expected.sort(); + assert_eq!(titles, expected); + Ok(()) + } + + #[test] + fn filters_with_root_numerical_comparison() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[?(@.price > $.store.min_price)].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 3); + let mut titles: Vec<&str> = result + .iter() + .map(|v| v.as_value().unwrap().as_string().unwrap().as_str()) + .collect(); + titles.sort(); + let mut expected = vec!["Brave New World", "The Hobbit", "To Kill a Mockingbird"]; + expected.sort(); + assert_eq!(titles, expected); + Ok(()) + } + + #[test] + fn filters_with_root_not_equal() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[?(@.author != $.store.featured_author)].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 8); + let mut titles: Vec<&str> = result + .iter() + .map(|v| v.as_value().unwrap().as_string().unwrap().as_str()) + .collect(); + titles.sort(); + let mut expected = vec![ + "Brave New World", + "Fahrenheit 451", + "The Great Gatsby", + "To Kill a Mockingbird", + "The Catcher in the Rye", + "Lord of the Flies", + "Pride and Prejudice", + "The Hobbit", + ]; + expected.sort(); + assert_eq!(titles, expected); + Ok(()) + } + + #[test] + fn filters_with_root_complex() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[?(@.author == $.store.featured_author && @.price <= $.store.min_price)].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 2); + let mut titles: Vec<&str> = result + .iter() + .map(|v| v.as_value().unwrap().as_string().unwrap().as_str()) + .collect(); + titles.sort(); + let mut expected = vec!["1984", "Animal Farm"]; + expected.sort(); + assert_eq!(titles, expected); + Ok(()) + } + } + + mod existence_filters { + use super::*; + #[test] + fn filters_by_existence_of_isbn() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[?(@.isbn)].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 3); + let mut titles: Vec<&str> = result + .iter() + .map(|v| v.as_value().unwrap().as_string().unwrap().as_str()) + .collect(); + titles.sort(); + let mut expected = vec!["1984", "Brave New World", "Fahrenheit 451"]; + expected.sort(); + assert_eq!(titles, expected); + Ok(()) + } + + #[test] + fn filters_by_non_existence_of_isbn() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[?(!@.isbn)].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 7); + let mut titles: Vec<&str> = result + .iter() + .map(|v| v.as_value().unwrap().as_string().unwrap().as_str()) + .collect(); + titles.sort(); + let mut expected = vec![ + "Animal Farm", + "The Great Gatsby", + "To Kill a Mockingbird", + "The Catcher in the Rye", + "Lord of the Flies", + "Pride and Prejudice", + "The Hobbit", + ]; + expected.sort(); + assert_eq!(titles, expected); + Ok(()) + } + + #[test] + fn filters_by_existence_of_price() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[?(@.price)].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 10); + Ok(()) + } + + #[test] + fn filters_by_non_existence_of_price() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[?(!@.price)].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 0); + Ok(()) + } + + #[test] + fn filters_by_non_existence_of_root_path() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[?($.store.nonexistent)].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 0); + Ok(()) + } + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/jsonpath/mod.rs b/3rdparty/loro/crates/loro-internal/src/jsonpath/mod.rs new file mode 100644 index 0000000000..9bbcdee9b2 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/jsonpath/mod.rs @@ -0,0 +1,10 @@ +pub mod ast; +pub mod errors; +pub mod jsonpath_impl; +pub mod subscription; + +pub use subscription::SubscribeJsonPathCallback; +pub mod parser; + +pub use ast::Query; +pub use parser::JSONPathParser; diff --git a/3rdparty/loro/crates/loro-internal/src/jsonpath/parser.rs b/3rdparty/loro/crates/loro-internal/src/jsonpath/parser.rs new file mode 100644 index 0000000000..60425fe492 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/jsonpath/parser.rs @@ -0,0 +1,708 @@ +//! A JSONPath parser using [pest]. +//! +//! Refer to `jsonpath.pest` and the [pest book]. +//! +//! [pest]: https://pest.rs/ +//! [pest book]: https://pest.rs/book/ + +use std::{collections::HashMap, ops::RangeInclusive}; + +use pest::{iterators::Pair, Parser}; +use pest_derive::Parser; + +use crate::jsonpath::ast::{ + ComparisonOperator, FilterExpression, LogicalOperator, Segment, Selector, +}; +use crate::jsonpath::errors::JSONPathError; +use crate::jsonpath::Query; + +#[derive(Parser)] +#[grammar = "jsonpath/jsonpath.pest"] +struct JSONPath; + +#[derive(Debug)] +pub enum ExpressionType { + Logical, + Nodes, + Value, +} + +#[derive(Debug)] +pub struct FunctionSignature { + pub param_types: Vec, + pub return_type: ExpressionType, +} + +pub fn standard_functions() -> HashMap { + let mut functions = HashMap::new(); + + functions.insert( + "count".to_owned(), + FunctionSignature { + param_types: vec![ExpressionType::Nodes], + return_type: ExpressionType::Value, + }, + ); + + functions.insert( + "length".to_owned(), + FunctionSignature { + param_types: vec![ExpressionType::Value], + return_type: ExpressionType::Value, + }, + ); + + functions.insert( + "match".to_owned(), + FunctionSignature { + param_types: vec![ExpressionType::Value, ExpressionType::Value], + return_type: ExpressionType::Logical, + }, + ); + + functions.insert( + "search".to_owned(), + FunctionSignature { + param_types: vec![ExpressionType::Value, ExpressionType::Value], + return_type: ExpressionType::Logical, + }, + ); + + functions.insert( + "value".to_owned(), + FunctionSignature { + param_types: vec![ExpressionType::Nodes], + return_type: ExpressionType::Value, + }, + ); + + functions +} + +#[derive(Debug)] +pub struct JSONPathParser { + pub index_range: RangeInclusive, + pub function_signatures: HashMap, +} + +impl Default for JSONPathParser { + fn default() -> Self { + Self::new() + } +} + +impl JSONPathParser { + pub fn new() -> Self { + JSONPathParser { + index_range: ((-2_i64).pow(53) + 1..=2_i64.pow(53) - 1), + function_signatures: standard_functions(), + } + } + + pub fn add_function( + &mut self, + name: &str, + params: Vec, + returns: ExpressionType, + ) { + self.function_signatures.insert( + name.to_owned(), + FunctionSignature { + param_types: params, + return_type: returns, + }, + ); + } + + pub fn parse(&self, query: &str) -> Result { + let segments = JSONPath::parse(Rule::jsonpath, query) + .map_err(|err| JSONPathError::syntax(err.to_string()))? + .try_fold(Segment::Root {}, |acc, segment| { + self.parse_segment(acc, segment) + })?; + Ok(Query { segments }) + } + + fn parse_segment(&self, acc: Segment, segment: Pair) -> Result { + Ok(match segment.as_rule() { + Rule::child_segment => Segment::Child { + left: Box::new(acc), + selectors: self.parse_segment_inner(segment.into_inner().next().unwrap())?, + }, + Rule::descendant_segment => Segment::Recursive { + left: Box::new(acc), + selectors: self.parse_segment_inner(segment.into_inner().next().unwrap())?, + }, + Rule::name_segment | Rule::index_segment => Segment::Child { + left: Box::new(acc), + selectors: vec![self.parse_selector(segment.into_inner().next().unwrap())?], + }, + Rule::EOI => acc, + _ => unreachable!(), + }) + } + + fn parse_segment_inner(&self, segment: Pair) -> Result, JSONPathError> { + Ok(match segment.as_rule() { + Rule::bracketed_selection => { + let seg: Result, _> = segment + .into_inner() + .map(|selector| self.parse_selector(selector)) + .collect(); + seg? + } + Rule::wildcard_selector => vec![Selector::Wild {}], + Rule::member_name_shorthand => vec![Selector::Name { + // for child_segment name: + name: segment.as_str().to_owned(), + }], + _ => unreachable!(), + }) + } + + fn parse_selector(&self, selector: Pair) -> Result { + Ok(match selector.as_rule() { + Rule::double_quoted => Selector::Name { + name: unescape_string(selector.as_str()), + }, + Rule::single_quoted => Selector::Name { + name: unescape_string(&selector.as_str().replace("\\'", "'")), + }, + Rule::wildcard_selector => Selector::Wild {}, + Rule::slice_selector => self.parse_slice_selector(selector)?, + Rule::index_selector => Selector::Index { + index: self.parse_i_json_int(selector.as_str())?, + }, + Rule::filter_selector => self.parse_filter_selector(selector)?, + Rule::member_name_shorthand => Selector::Name { + // for name_segment name: + name: selector.as_str().to_owned(), + }, + _ => unreachable!(), + }) + } + + fn parse_slice_selector(&self, selector: Pair) -> Result { + let mut start: Option = None; + let mut stop: Option = None; + let mut step: Option = None; + + for i in selector.into_inner() { + match i.as_rule() { + Rule::start => start = Some(self.parse_i_json_int(i.as_str())?), + Rule::stop => stop = Some(self.parse_i_json_int(i.as_str())?), + Rule::step => step = Some(self.parse_i_json_int(i.as_str())?), + _ => unreachable!(), + } + } + + Ok(Selector::Slice { start, stop, step }) + } + + fn parse_filter_selector(&self, selector: Pair) -> Result { + Ok(Selector::Filter { + expression: Box::new( + self.parse_logical_or_expression(selector.into_inner().next().unwrap(), true)?, + ), + }) + } + + fn parse_logical_or_expression( + &self, + expr: Pair, + assert_compared: bool, + ) -> Result { + let mut it = expr.into_inner(); + let mut or_expr = self.parse_logical_and_expression(it.next().unwrap(), assert_compared)?; + if assert_compared { + self.assert_compared(&or_expr)?; + } + for and_expr in it { + let right = self.parse_logical_and_expression(and_expr, assert_compared)?; + if assert_compared { + self.assert_compared(&right)?; + } + or_expr = FilterExpression::Logical { + left: Box::new(or_expr), + operator: LogicalOperator::Or, + right: Box::new(right), + }; + } + Ok(or_expr) + } + + fn parse_logical_and_expression( + &self, + expr: Pair, + assert_compared: bool, + ) -> Result { + let mut it = expr.into_inner(); + let mut and_expr = self.parse_basic_expression(it.next().unwrap())?; + if assert_compared { + self.assert_compared(&and_expr)?; + } + for basic_expr in it { + let right = self.parse_basic_expression(basic_expr)?; + if assert_compared { + self.assert_compared(&right)?; + } + and_expr = FilterExpression::Logical { + left: Box::new(and_expr), + operator: LogicalOperator::And, + right: Box::new(right), + }; + } + Ok(and_expr) + } + + fn parse_basic_expression(&self, expr: Pair) -> Result { + match expr.as_rule() { + Rule::paren_expr => self.parse_paren_expression(expr), + Rule::comparison_expr => self.parse_comparison_expression(expr), + Rule::test_expr => self.parse_test_expression(expr), + _ => unreachable!(), + } + } + + fn parse_paren_expression(&self, expr: Pair) -> Result { + let mut it = expr.into_inner(); + let p = it.next().unwrap(); + match p.as_rule() { + Rule::logical_not_op => Ok(FilterExpression::Not { + expression: Box::new(self.parse_logical_or_expression(it.next().unwrap(), true)?), + }), + Rule::logical_or_expr => self.parse_logical_or_expression(p, true), + _ => unreachable!(), + } + } + + fn parse_comparison_expression( + &self, + expr: Pair, + ) -> Result { + let mut it = expr.into_inner(); + let left = self.parse_comparable(it.next().unwrap())?; + let operator = match it.next().unwrap().as_str() { + "==" => ComparisonOperator::Eq, + "!=" => ComparisonOperator::Ne, + "<=" => ComparisonOperator::Le, + ">=" => ComparisonOperator::Ge, + "<" => ComparisonOperator::Lt, + ">" => ComparisonOperator::Gt, + "contains" => ComparisonOperator::Contains, + "in" => ComparisonOperator::In, + _ => unreachable!(), + }; + let right = self.parse_comparable(it.next().unwrap())?; + self.assert_comparable(&left)?; + self.assert_comparable(&right)?; + Ok(FilterExpression::Comparison { + left: Box::new(left), + operator, + right: Box::new(right), + }) + } + + fn parse_comparable(&self, expr: Pair) -> Result { + Ok(match expr.as_rule() { + Rule::number => self.parse_number(expr)?, + Rule::double_quoted => FilterExpression::StringLiteral { + value: unescape_string(expr.as_str()), + }, + Rule::single_quoted => FilterExpression::StringLiteral { + value: unescape_string(&expr.as_str().replace("\\'", "'")), + }, + Rule::true_literal => FilterExpression::True_ {}, + Rule::false_literal => FilterExpression::False_ {}, + Rule::null => FilterExpression::Null {}, + Rule::array_literal => self.parse_array_literal(expr)?, + Rule::rel_singular_query => { + let segments = expr + .into_inner() + .try_fold(Segment::Root {}, |acc, segment| { + self.parse_segment(acc, segment) + }); + FilterExpression::RelativeQuery { + query: Box::new(Query { + segments: segments?, + }), + } + } + Rule::abs_singular_query => { + let segments = expr + .into_inner() + .try_fold(Segment::Root {}, |acc, segment| { + self.parse_segment(acc, segment) + }); + FilterExpression::RootQuery { + query: Box::new(Query { + segments: segments?, + }), + } + } + Rule::function_expr => self.parse_function_expression(expr)?, + _ => unreachable!(), + }) + } + + fn parse_number(&self, expr: Pair) -> Result { + let raw = expr.as_str(); + if raw == "-0" { + return Ok(FilterExpression::Int { value: 0 }); + } + let is_float = raw.contains('.') || raw.contains('e') || raw.contains('E'); + if !is_float { + if let Ok(value) = raw.parse::() { + return Ok(FilterExpression::Int { value }); + } + } + Ok(FilterExpression::Float { + value: raw + .parse::() + .map_err(|_| JSONPathError::syntax(String::from("invalid float literal")))?, + }) + } + + fn parse_array_literal(&self, expr: Pair) -> Result { + let values: Result, _> = expr + .into_inner() + .map(|p| self.parse_comparable(p)) + .collect(); + Ok(FilterExpression::Array { values: values? }) + } + + fn parse_test_expression(&self, expr: Pair) -> Result { + let mut it = expr.into_inner(); + let pair = it.next().unwrap(); + Ok(match pair.as_rule() { + Rule::logical_not_op => FilterExpression::Not { + expression: Box::new(self.parse_test_expression_inner(it.next().unwrap())?), + }, + _ => self.parse_test_expression_inner(pair)?, + }) + } + + fn parse_test_expression_inner( + &self, + expr: Pair, + ) -> Result { + Ok(match expr.as_rule() { + Rule::rel_query => { + let segments = expr + .into_inner() + .try_fold(Segment::Root {}, |acc, segment| { + self.parse_segment(acc, segment) + }); + FilterExpression::RelativeQuery { + query: Box::new(Query { + segments: segments?, + }), + } + } + Rule::root_query => { + let segments = expr + .into_inner() + .try_fold(Segment::Root {}, |acc, segment| { + self.parse_segment(acc, segment) + }); + FilterExpression::RootQuery { + query: Box::new(Query { + segments: segments?, + }), + } + } + Rule::function_expr => self.parse_function_expression(expr)?, + _ => unreachable!(), + }) + } + + fn parse_function_expression( + &self, + expr: Pair, + ) -> Result { + let mut it = expr.into_inner(); + let name = it.next().unwrap().as_str(); + let args: Result, _> = it.map(|ex| self.parse_function_argument(ex)).collect(); + Ok(FilterExpression::Function { + name: name.to_string(), + args: self.assert_well_typed(name, args?)?, + }) + } + + fn parse_function_argument(&self, expr: Pair) -> Result { + Ok(match expr.as_rule() { + Rule::number => self.parse_number(expr)?, + Rule::double_quoted => FilterExpression::StringLiteral { + value: unescape_string(expr.as_str()), + }, + Rule::single_quoted => FilterExpression::StringLiteral { + value: unescape_string(&expr.as_str().replace("\\'", "'")), + }, + Rule::true_literal => FilterExpression::True_ {}, + Rule::false_literal => FilterExpression::False_ {}, + Rule::null => FilterExpression::Null {}, + Rule::array_literal => self.parse_array_literal(expr)?, + Rule::rel_query => { + let segments = expr + .into_inner() + .try_fold(Segment::Root {}, |acc, segment| { + self.parse_segment(acc, segment) + }); + FilterExpression::RelativeQuery { + query: Box::new(Query { + segments: segments?, + }), + } + } + Rule::root_query => { + let segments = expr + .into_inner() + .try_fold(Segment::Root {}, |acc, segment| { + self.parse_segment(acc, segment) + }); + FilterExpression::RootQuery { + query: Box::new(Query { + segments: segments?, + }), + } + } + Rule::logical_or_expr => self.parse_logical_or_expression(expr, false)?, + Rule::function_expr => self.parse_function_expression(expr)?, + _ => unreachable!(), + }) + } + + fn parse_i_json_int(&self, value: &str) -> Result { + let i = value + .parse::() + .map_err(|_| JSONPathError::syntax(format!("index out of range `{}`", value)))?; + if !self.index_range.contains(&i) { + return Err(JSONPathError::syntax(format!( + "index out of range `{}`", + value + ))); + } + Ok(i) + } + + fn assert_comparable(&self, expr: &FilterExpression) -> Result<(), JSONPathError> { + // TODO: accept span/position for better errors + match expr { + FilterExpression::RelativeQuery { query, .. } + | FilterExpression::RootQuery { query, .. } => { + if !query.is_singular() { + Err(JSONPathError::typ(String::from( + "non-singular query is not comparable", + ))) + } else { + Ok(()) + } + } + FilterExpression::Function { name, .. } => { + if let Some(FunctionSignature { + return_type: ExpressionType::Value, + .. + }) = self.function_signatures.get(name) + { + Ok(()) + } else { + Err(JSONPathError::typ(format!( + "result of {}() is not comparable", + name + ))) + } + } + _ => Ok(()), + } + } + + fn assert_compared(&self, expr: &FilterExpression) -> Result<(), JSONPathError> { + match expr { + FilterExpression::Function { name, .. } => { + if let Some(FunctionSignature { + return_type: ExpressionType::Value, + .. + }) = self.function_signatures.get(name) + { + Err(JSONPathError::typ(format!( + "result of {}() must be compared", + name + ))) + } else { + Ok(()) + } + } + _ => Ok(()), + } + } + + fn assert_well_typed( + &self, + func_name: &str, + args: Vec, + ) -> Result, JSONPathError> { + // TODO: accept span/position for better errors + let signature = self + .function_signatures + .get(func_name) + .ok_or_else(|| JSONPathError::name(format!("unknown function `{}`", func_name)))?; + + // correct number of arguments? + if args.len() != signature.param_types.len() { + return Err(JSONPathError::typ(format!( + "{}() takes {} argument{} but {} were given", + func_name, + signature.param_types.len(), + if signature.param_types.len() > 1 { + "s" + } else { + "" + }, + args.len() + ))); + } + + // correct argument types? + for (idx, typ) in signature.param_types.iter().enumerate() { + let arg = &args[idx]; + match typ { + ExpressionType::Value => { + if !self.is_value_type(arg) { + return Err(JSONPathError::typ(format!( + "argument {} of {}() must be of a 'Value' type", + idx + 1, + func_name + ))); + } + } + ExpressionType::Logical => { + if !matches!( + arg, + FilterExpression::RelativeQuery { .. } + | FilterExpression::RootQuery { .. } + | FilterExpression::Logical { .. } + | FilterExpression::Comparison { .. }, + ) { + return Err(JSONPathError::typ(format!( + "argument {} of {}() must be of a 'Logical' type", + idx + 1, + func_name + ))); + } + } + ExpressionType::Nodes => { + if !self.is_nodes_type(arg) { + return Err(JSONPathError::typ(format!( + "argument {} of {}() must be of a 'Nodes' type", + idx + 1, + func_name + ))); + } + } + } + } + + Ok(args) + } + + fn is_value_type(&self, expr: &FilterExpression) -> bool { + // literals are values + if expr.is_literal() { + return true; + } + + match expr { + FilterExpression::RelativeQuery { query, .. } + | FilterExpression::RootQuery { query, .. } => { + // singular queries will be coerced to a value + query.is_singular() + } + FilterExpression::Function { name, .. } => { + // some functions return a value + matches!( + self.function_signatures.get(name), + Some(FunctionSignature { + return_type: ExpressionType::Value, + .. + }) + ) + } + FilterExpression::Array { .. } => true, + _ => false, + } + } + + fn is_nodes_type(&self, expr: &FilterExpression) -> bool { + match expr { + FilterExpression::RelativeQuery { .. } | FilterExpression::RootQuery { .. } => true, + FilterExpression::Function { name, .. } => { + matches!( + self.function_signatures.get(name), + Some(FunctionSignature { + return_type: ExpressionType::Nodes, + .. + }) + ) + } + _ => false, + } + } +} + +fn unescape_string(value: &str) -> String { + let chars = value.chars().collect::>(); + let length = chars.len(); + let mut rv = String::new(); + let mut index: usize = 0; + + while index < length { + match chars[index] { + '\\' => { + index += 1; + match chars[index] { + '"' => rv.push('"'), + '\\' => rv.push('\\'), + '/' => rv.push('/'), + 'b' => rv.push('\x08'), + 'f' => rv.push('\x0C'), + 'n' => rv.push('\n'), + 'r' => rv.push('\r'), + 't' => rv.push('\t'), + 'u' => { + index += 1; + let digits = chars + .get(index..index + 4) + .unwrap() + .iter() + .collect::(); + let mut codepoint = u32::from_str_radix(&digits, 16).unwrap(); + + if index + 5 < length && chars[index + 4] == '\\' && chars[index + 5] == 'u' + { + let digits = &chars + .get(index + 6..index + 10) + .unwrap() + .iter() + .collect::(); + let low_surrogate = u32::from_str_radix(digits, 16).unwrap(); + codepoint = + 0x10000 + (((codepoint & 0x03FF) << 10) | (low_surrogate & 0x03FF)); + index += 6; + } + + let unescaped = char::from_u32(codepoint).unwrap(); + rv.push(unescaped); + index += 3; + } + _ => unreachable!(), + } + } + c => { + rv.push(c); + } + } + index += 1; + } + + rv +} diff --git a/3rdparty/loro/crates/loro-internal/src/jsonpath/subscription.rs b/3rdparty/loro/crates/loro-internal/src/jsonpath/subscription.rs new file mode 100644 index 0000000000..599bc18a1e --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/jsonpath/subscription.rs @@ -0,0 +1,551 @@ +//! # JSONPath Subscription Module +//! +//! This module provides a mechanism to subscribe to changes that may affect a JSONPath query +//! result, without re-evaluating the query on every change. +//! +//! ## Design Goals +//! +//! - **Conservative matching**: The matcher may produce false positives (triggering when +//! the query result hasn't actually changed) but must never produce false negatives +//! (missing a notification when the result did change). +//! - **Lightweight notifications**: Callbacks receive no payload; applications can +//! debounce/throttle and evaluate the JSONPath themselves. +//! - **Efficient matching**: Uses an NFA (non-deterministic finite automaton) approach +//! to match event paths against compiled JSONPath queries in O(path_len × steps) time. +//! +//! ## Algorithm Overview +//! +//! 1. **Compilation**: JSONPath is parsed and converted into a sequence of lightweight +//! `MatchSelector`s (no AST storage needed for matching). +//! +//! 2. **NFA Simulation**: When an event arrives, we simulate an NFA where: +//! - States are positions in the step sequence (0..steps.len()) +//! - Transitions occur when a selector matches the current path element +//! - Recursive steps allow staying at the same state while consuming input +//! - Acceptance occurs when any state reaches or exceeds `steps.len()` + +use std::sync::Arc; + +use loro_common::TreeID; +use smallvec::SmallVec; + +use crate::{ + event::{Diff, Index}, + jsonpath::{ + ast::{Query, Segment, Selector}, + JSONPathParser, + }, + utils::subscription::Subscription, + LoroDoc, LoroError, LoroResult, +}; + +/// Callback used by `subscribe_jsonpath`. +/// +/// Note: the callback does **not** carry the query result. It is intended as a +/// lightweight notification so applications can debounce/throttle and evaluate +/// JSONPath themselves if needed. +pub type SubscribeJsonPathCallback = Arc; + +/// Represents a single path segment used for matching against events. +/// +/// Path elements are derived from event paths and represent the location of a change +/// in the document tree. They form the "input alphabet" for our NFA simulation. +#[derive(Debug, Clone, PartialEq, Eq)] +enum PathElem { + /// A map key (e.g., "title" in `$.books[0].title`) + Key(Arc), + /// A sequence (list/array) index. + /// - `Some(i)` means a specific index changed + /// - `None` means "some index changed" (used for list/tree child mutations + /// where we don't know which specific indices are affected) + Seq(Option), + /// A tree node identifier (for Loro's tree CRDT) + Node(TreeID), +} + +impl From<&Index> for PathElem { + fn from(value: &Index) -> Self { + match value { + Index::Key(k) => PathElem::Key(k.as_ref().into()), + Index::Seq(i) => PathElem::Seq(Some(*i)), + Index::Node(n) => PathElem::Node(*n), + } + } +} + +// ============================================================================= +// Lightweight Matching Types +// ============================================================================= +// +// Instead of storing the full Selector AST (which includes heavyweight +// FilterExpression trees), we convert selectors to a simple matching-only enum. +// This reduces memory usage and code complexity. + +/// Simplified selector for matching only - no AST storage needed. +/// +/// All complex selectors (Slice, Wild, Filter, negative Index) become `Wildcard` +/// since they can match any element during conservative path matching. +#[derive(Debug, Clone)] +enum MatchSelector { + /// Exact key match (from `Selector::Name`) + Name(Arc), + /// Exact non-negative index match (from `Selector::Index` where index >= 0) + Index(usize), + /// Matches anything (from Wild, Slice, Filter, or negative Index) + Wildcard, +} + +/// A single step in the linearized JSONPath query. +#[derive(Debug, Clone)] +struct Step { + /// If true, this step uses recursive descent (`..`) and can match at any depth. + recursive: bool, + /// Simplified selectors for matching. + selectors: SmallVec<[MatchSelector; 2]>, +} + +/// A compiled matcher that conservatively checks whether a change at some path +/// can affect the JSONPath query. +#[derive(Debug, Clone)] +struct JsonPathMatcher { + /// Linearized sequence of matching steps + steps: Vec, +} + +impl JsonPathMatcher { + /// Compile a JSONPath query into a matcher. + fn new(query: &Query) -> Self { + let mut steps = Vec::new(); + build_steps(&query.segments, &mut steps); + JsonPathMatcher { steps } + } + + /// Returns true if the provided path (from root) could affect the query result. + fn may_match(&self, path: &[PathElem]) -> bool { + if self.steps.is_empty() { + return true; + } + let positions = self.positions_after(path); + positions.iter().any(|&p| p >= self.steps.len()) + } + + /// Check if any position was reached by passing through a Wildcard step. + /// Wildcard steps come from filter/wild/slice selectors - any key change + /// under them could affect the query result. + #[inline] + fn passed_through_wildcard(&self, positions: &[usize]) -> bool { + positions.iter().any(|&pos| { + pos > 0 + && self.steps[pos - 1] + .selectors + .iter() + .any(|s| matches!(s, MatchSelector::Wildcard)) + }) + } + + /// Simulate the NFA on a path and return all reachable positions after consuming it. + /// + /// - Position `== steps.len()`: Query fully matched + /// - Position `< steps.len()`: Partial match + /// - Empty result: No match possible + fn positions_after(&self, path: &[PathElem]) -> SmallVec<[usize; 8]> { + let mut positions = SmallVec::<[usize; 8]>::new(); + positions.push(0); + + for elem in path { + let mut next = SmallVec::<[usize; 8]>::new(); + + for &pos in positions.iter() { + if pos >= self.steps.len() { + next.push(pos); + continue; + } + + let step = &self.steps[pos]; + + // Recursive descent: can stay at same position + if step.recursive { + next.push(pos); + } + + // If selector matches, advance to next step + if selector_matches(&step.selectors, elem) { + next.push(pos + 1); + } + } + + dedup_positions(&mut next); + if next.is_empty() { + return next; + } + positions = next; + } + + positions + } +} + +/// Check if any selector matches the given path element. +fn selector_matches(selectors: &[MatchSelector], elem: &PathElem) -> bool { + selectors.iter().any(|sel| match sel { + MatchSelector::Name(name) => matches!(elem, PathElem::Key(k) if k == name), + MatchSelector::Index(idx) => match elem { + PathElem::Seq(Some(i)) => *i == *idx, + PathElem::Seq(None) => true, // Unknown index - conservative match + _ => false, + }, + MatchSelector::Wildcard => true, + }) +} + +/// Convert AST Selector to lightweight MatchSelector. +fn to_match_selector(sel: &Selector) -> MatchSelector { + match sel { + Selector::Name { name } => MatchSelector::Name(name.as_str().into()), + Selector::Index { index } if *index >= 0 => MatchSelector::Index(*index as usize), + // Negative index, Slice, Wild, Filter all become Wildcard + _ => MatchSelector::Wildcard, + } +} + +/// Linearize the JSONPath AST into a sequence of matching steps. +fn build_steps(segment: &Segment, steps: &mut Vec) { + match segment { + Segment::Root {} => {} + Segment::Child { left, selectors } => { + build_steps(left, steps); + steps.push(Step { + recursive: false, + selectors: selectors.iter().map(to_match_selector).collect(), + }); + } + Segment::Recursive { left, selectors } => { + build_steps(left, steps); + steps.push(Step { + recursive: true, + selectors: selectors.iter().map(to_match_selector).collect(), + }); + } + } +} + +/// Deduplicate positions in-place. +#[inline] +fn dedup_positions(v: &mut SmallVec<[usize; 8]>) { + v.sort_unstable(); + v.dedup(); +} + +// ============================================================================= +// Public API +// ============================================================================= + +impl LoroDoc { + /// Subscribe to updates that may affect the given JSONPath query. + /// + /// ## Behavior + /// + /// - **Conservative matching**: The callback may fire for changes that don't + /// actually affect the query result (false positives), but it will never + /// miss a change that does affect it (no false negatives). + /// - **Lightweight notifications**: The callback receives no payload. Applications + /// should debounce/throttle and re-evaluate the JSONPath if needed. + /// + /// ## Returns + /// + /// A `Subscription` handle. Drop it to unsubscribe. + #[cfg(feature = "jsonpath")] + pub fn subscribe_jsonpath( + &self, + jsonpath: &str, + callback: SubscribeJsonPathCallback, + ) -> LoroResult { + let query = JSONPathParser::new() + .parse(jsonpath) + .map_err(|e| LoroError::ArgErr(e.to_string().into_boxed_str()))?; + + let matcher = Arc::new(JsonPathMatcher::new(&query)); + + let sub = self.subscribe_root(Arc::new(move |event| { + if event.events.is_empty() { + return; + } + + let matcher = &matcher; + let mut fired = false; + + for container_diff in event.events.iter() { + if fired { + break; + } + + // Convert the container's path to PathElem representation + let base_path: SmallVec<[PathElem; 8]> = container_diff + .path + .iter() + .map(|(_, idx)| PathElem::from(idx)) + .collect(); + + // Check 1: Does the container path itself match the query? + if matcher.may_match(&base_path) { + fired = true; + break; + } + + // Check 2: Map-specific handling - check each changed key + if let Diff::Map(map) = &container_diff.diff { + let base_positions = matcher.positions_after(&base_path); + if base_positions.is_empty() { + continue; + } + + // If we passed through a Wildcard (filter/wild/slice), any key + // change could affect the result - be conservative. + let past_wildcard = matcher.passed_through_wildcard(&base_positions); + + for key in map.updated.keys() { + let mut extended: SmallVec<[PathElem; 8]> = base_path.clone(); + extended.push(PathElem::Key(key.as_ref().into())); + + let extended_positions = matcher.positions_after(&extended); + + // Trigger if key path could match OR we're inside a wildcard selection + if !extended_positions.is_empty() || past_wildcard { + fired = true; + break; + } + } + + if fired { + break; + } + } + + // Check 3: List/Tree/Counter child mutations + // These container types can have child changes at unknown indices + let has_child_changes = matches!( + &container_diff.diff, + Diff::List(_) | Diff::Tree(_) | Diff::Unknown + ); + #[cfg(feature = "counter")] + let has_child_changes = + has_child_changes || matches!(&container_diff.diff, Diff::Counter(_)); + + if has_child_changes { + let mut extended: SmallVec<[PathElem; 8]> = base_path.clone(); + extended.push(PathElem::Seq(None)); // "some child changed" + if !matcher.positions_after(&extended).is_empty() { + fired = true; + } + } + } + + if fired { + (callback)(); + } + })); + + Ok(sub) + } +} + +// ============================================================================= +// Tests +// ============================================================================= + +#[cfg(test)] +mod tests { + use super::*; + use crate::{LoroDoc, MapHandler}; + use std::sync::atomic::{AtomicUsize, Ordering}; + + fn make_book( + doc: &LoroDoc, + idx: usize, + title: &str, + available: bool, + price: i64, + ) -> MapHandler { + let books = doc.get_list("books"); + let book = books + .insert_container(idx, MapHandler::new_detached()) + .unwrap(); + book.insert("title", title).unwrap(); + book.insert("available", available).unwrap(); + book.insert("price", price).unwrap(); + book + } + + #[test] + fn jsonpath_subscribe_triggers_on_specific_key() { + let doc = LoroDoc::new_auto_commit(); + let first_book = make_book(&doc, 0, "Old", true, 10); + doc.commit_then_renew(); + + let hit = Arc::new(AtomicUsize::new(0)); + let hit_ref = hit.clone(); + let _sub = doc + .subscribe_jsonpath( + "$.books[0].title", + Arc::new(move || { + hit_ref.fetch_add(1, Ordering::SeqCst); + }), + ) + .unwrap(); + + first_book.insert("title", "New").unwrap(); + doc.commit_then_renew(); + + assert!(hit.load(Ordering::SeqCst) >= 1); + } + + #[test] + fn jsonpath_subscribe_wildcard_on_list() { + let doc = LoroDoc::new_auto_commit(); + make_book(&doc, 0, "A", true, 10); + let second_book = make_book(&doc, 1, "B", true, 20); + doc.commit_then_renew(); + + let hit = Arc::new(AtomicUsize::new(0)); + let hit_ref = hit.clone(); + let _sub = doc + .subscribe_jsonpath( + "$.books[*].price", + Arc::new(move || { + hit_ref.fetch_add(1, Ordering::SeqCst); + }), + ) + .unwrap(); + + second_book.insert("price", 25).unwrap(); + doc.commit_then_renew(); + + assert!(hit.load(Ordering::SeqCst) >= 1); + } + + #[test] + fn jsonpath_subscribe_negative_index() { + let doc = LoroDoc::new_auto_commit(); + make_book(&doc, 0, "A", true, 10); + make_book(&doc, 1, "B", true, 20); + doc.commit_then_renew(); + + let hit = Arc::new(AtomicUsize::new(0)); + let hit_ref = hit.clone(); + let _sub = doc + .subscribe_jsonpath( + "$.books[-1].title", + Arc::new(move || { + hit_ref.fetch_add(1, Ordering::SeqCst); + }), + ) + .unwrap(); + + let books = doc.get_list("books"); + let last = books.get_child_handler(1).unwrap(); + last.as_map().unwrap().insert("title", "B updated").unwrap(); + doc.commit_then_renew(); + + assert!(hit.load(Ordering::SeqCst) >= 1); + } + + #[test] + fn jsonpath_subscribe_slice_range() { + let doc = LoroDoc::new_auto_commit(); + make_book(&doc, 0, "A", true, 10); + let second_book = make_book(&doc, 1, "B", true, 20); + make_book(&doc, 2, "C", true, 30); + doc.commit_then_renew(); + + let hit = Arc::new(AtomicUsize::new(0)); + let hit_ref = hit.clone(); + let _sub = doc + .subscribe_jsonpath( + "$.books[0:2].title", + Arc::new(move || { + hit_ref.fetch_add(1, Ordering::SeqCst); + }), + ) + .unwrap(); + + second_book.insert("title", "B updated").unwrap(); + doc.commit_then_renew(); + + assert!(hit.load(Ordering::SeqCst) >= 1); + } + + #[test] + fn jsonpath_subscribe_recursive() { + let doc = LoroDoc::new_auto_commit(); + let store = doc.get_map("store"); + let nested = store + .insert_container("inventory", MapHandler::new_detached()) + .unwrap(); + nested.insert("total", 3).unwrap(); + doc.commit_then_renew(); + + let hit = Arc::new(AtomicUsize::new(0)); + let hit_ref = hit.clone(); + let _sub = doc + .subscribe_jsonpath( + "$..total", + Arc::new(move || { + hit_ref.fetch_add(1, Ordering::SeqCst); + }), + ) + .unwrap(); + + nested.insert("total", 4).unwrap(); + doc.commit_then_renew(); + + assert!(hit.load(Ordering::SeqCst) >= 1); + } + + #[test] + fn jsonpath_subscribe_filter_treated_as_wildcard() { + let doc = LoroDoc::new_auto_commit(); + make_book(&doc, 0, "A", true, 10); + let second_book = make_book(&doc, 1, "B", false, 20); + doc.commit_then_renew(); + + let hit = Arc::new(AtomicUsize::new(0)); + let hit_ref = hit.clone(); + let _sub = doc + .subscribe_jsonpath( + "$.books[?@.available].title", + Arc::new(move || { + hit_ref.fetch_add(1, Ordering::SeqCst); + }), + ) + .unwrap(); + + second_book.insert("available", true).unwrap(); + doc.commit_then_renew(); + + assert!(hit.load(Ordering::SeqCst) >= 1); + } + + #[test] + fn jsonpath_subscribe_triggers_once_per_commit() { + let doc = LoroDoc::new_auto_commit(); + let book = make_book(&doc, 0, "A", true, 10); + doc.commit_then_renew(); + + let hit = Arc::new(AtomicUsize::new(0)); + let hit_ref = hit.clone(); + let _sub = doc + .subscribe_jsonpath( + "$.books[0].title", + Arc::new(move || { + hit_ref.fetch_add(1, Ordering::SeqCst); + }), + ) + .unwrap(); + + book.insert("title", "X").unwrap(); + book.insert("title", "Y").unwrap(); + doc.commit_then_renew(); + + assert_eq!(hit.load(Ordering::SeqCst), 1); + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/kv_store.rs b/3rdparty/loro/crates/loro-internal/src/kv_store.rs new file mode 100644 index 0000000000..4598b4d61b --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/kv_store.rs @@ -0,0 +1,260 @@ +use crate::sync::Mutex; +use bytes::Bytes; +pub use loro_kv_store::compress::CompressionType; +pub use loro_kv_store::MemKvStore; +use std::sync::Arc; +use std::{collections::BTreeMap, ops::Bound}; + +pub trait KvStore: std::fmt::Debug + Send + Sync { + fn get(&self, key: &[u8]) -> Option; + fn set(&mut self, key: &[u8], value: Bytes); + fn compare_and_swap(&mut self, key: &[u8], old: Option, new: Bytes) -> bool; + fn remove(&mut self, key: &[u8]) -> Option; + fn contains_key(&self, key: &[u8]) -> bool; + fn scan( + &self, + start: Bound<&[u8]>, + end: Bound<&[u8]>, + ) -> Box + '_>; + fn len(&self) -> usize; + fn is_empty(&self) -> bool; + fn size(&self) -> usize; + fn export_all(&mut self) -> Bytes; + fn import_all(&mut self, bytes: Bytes) -> Result<(), String>; + /// Like [`Self::import_all`], but for callers that already guarantee the + /// blob's integrity via an outer checksum, so per-block checksums may be + /// skipped. The default falls back to the checked [`Self::import_all`]; only + /// override it where the fast path is safe (e.g. SsTable-backed stores). + fn import_all_unchecked(&mut self, bytes: Bytes) -> Result<(), String> { + self.import_all(bytes) + } + fn clone_store(&self) -> Arc>; +} + +fn get_common_prefix_len_and_strip<'a, T: AsRef<[u8]> + ?Sized>( + this: &'a T, + last: &T, +) -> (u8, &'a [u8]) { + let mut common_prefix_len = 0; + for (i, (a, b)) in this.as_ref().iter().zip(last.as_ref().iter()).enumerate() { + if a != b || i == 255 { + common_prefix_len = i; + break; + } + } + + let suffix = &this.as_ref()[common_prefix_len..]; + (common_prefix_len as u8, suffix) +} + +impl KvStore for MemKvStore { + fn get(&self, key: &[u8]) -> Option { + self.get(key) + } + + fn set(&mut self, key: &[u8], value: Bytes) { + self.set(key, value) + } + + fn compare_and_swap(&mut self, key: &[u8], old: Option, new: Bytes) -> bool { + self.compare_and_swap(key, old, new) + } + + fn remove(&mut self, key: &[u8]) -> Option { + let ans = self.get(key); + self.remove(key); + ans + } + + fn contains_key(&self, key: &[u8]) -> bool { + self.contains_key(key) + } + + fn scan( + &self, + start: Bound<&[u8]>, + end: Bound<&[u8]>, + ) -> Box + '_> { + self.scan(start, end) + } + + fn len(&self) -> usize { + self.len() + } + + fn is_empty(&self) -> bool { + self.is_empty() + } + + fn size(&self) -> usize { + self.size() + } + + fn export_all(&mut self) -> Bytes { + self.export_all() + } + + fn import_all(&mut self, bytes: Bytes) -> Result<(), String> { + self.import_all(bytes) + } + + fn import_all_unchecked(&mut self, bytes: Bytes) -> Result<(), String> { + self.import_all_unchecked(bytes) + } + + fn clone_store(&self) -> Arc> { + Arc::new(Mutex::new(self.clone())) + } +} + +mod default_binary_format { + //! Default binary format for the key-value store. + //! + //! It will compress the prefix of the keys that are common with the previous key. + + use bytes::Bytes; + + use super::get_common_prefix_len_and_strip; + + pub fn export_by_scan(store: &impl super::KvStore) -> bytes::Bytes { + let mut buf = Vec::new(); + let mut last_key: Option = None; + for (k, v) in store.scan(std::ops::Bound::Unbounded, std::ops::Bound::Unbounded) { + { + // Write the key + match last_key.take() { + None => { + leb128::write::unsigned(&mut buf, k.len() as u64).unwrap(); + buf.extend_from_slice(&k); + } + Some(last) => { + let (common, suffix) = get_common_prefix_len_and_strip(&k, &last); + buf.push(common); + leb128::write::unsigned(&mut buf, suffix.len() as u64).unwrap(); + buf.extend_from_slice(suffix); + } + }; + + last_key = Some(k); + } + + // Write the value + leb128::write::unsigned(&mut buf, v.len() as u64).unwrap(); + buf.extend_from_slice(&v); + } + + buf.into() + } + + pub fn import(store: &mut impl super::KvStore, bytes: bytes::Bytes) -> Result<(), String> { + let mut bytes: &[u8] = &bytes; + let mut last_key = Vec::new(); + + while !bytes.is_empty() { + // Read the key + let mut key = Vec::new(); + if last_key.is_empty() { + let key_len = leb128::read::unsigned(&mut bytes).map_err(|e| e.to_string())?; + let key_len = key_len as usize; + key.extend_from_slice(&bytes[..key_len]); + bytes = &bytes[key_len..]; + } else { + let common_prefix_len = bytes[0] as usize; + bytes = &bytes[1..]; + key.extend_from_slice(&last_key[..common_prefix_len]); + let suffix_len = leb128::read::unsigned(&mut bytes).map_err(|e| e.to_string())?; + let suffix_len = suffix_len as usize; + key.extend_from_slice(&bytes[..suffix_len]); + bytes = &bytes[suffix_len..]; + } + + // Read the value + let value_len = leb128::read::unsigned(&mut bytes).map_err(|e| e.to_string())?; + let value_len = value_len as usize; + let value = Bytes::copy_from_slice(&bytes[..value_len]); + bytes = &bytes[value_len..]; + + // Store the key-value pair + store.set(&key, value); + + last_key = key; + } + + Ok(()) + } +} + +impl KvStore for BTreeMap { + fn get(&self, key: &[u8]) -> Option { + self.get(key).cloned() + } + + fn set(&mut self, key: &[u8], value: Bytes) { + self.insert(Bytes::copy_from_slice(key), value); + } + + fn compare_and_swap(&mut self, key: &[u8], old: Option, new: Bytes) -> bool { + let key = Bytes::copy_from_slice(key); + match self.get_mut(&key) { + Some(v) => { + if old.as_ref() == Some(v) { + self.insert(key, new); + true + } else { + false + } + } + None => { + if old.is_none() { + self.insert(key, new); + true + } else { + false + } + } + } + } + + fn remove(&mut self, key: &[u8]) -> Option { + self.remove(key) + } + + fn contains_key(&self, key: &[u8]) -> bool { + self.contains_key(key) + } + + fn scan( + &self, + start: Bound<&[u8]>, + end: Bound<&[u8]>, + ) -> Box + '_> { + Box::new( + self.range::<[u8], _>((start, end)) + .map(|(k, v)| (k.clone(), v.clone())), + ) + } + + fn len(&self) -> usize { + self.len() + } + + fn is_empty(&self) -> bool { + self.is_empty() + } + + fn size(&self) -> usize { + self.iter().fold(0, |acc, (k, v)| acc + k.len() + v.len()) + } + + fn export_all(&mut self) -> Bytes { + default_binary_format::export_by_scan(self) + } + + fn import_all(&mut self, bytes: Bytes) -> Result<(), String> { + default_binary_format::import(self, bytes) + } + + fn clone_store(&self) -> Arc> { + Arc::new(Mutex::new(self.clone())) + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/lib.rs b/3rdparty/loro/crates/loro-internal/src/lib.rs new file mode 100644 index 0000000000..d802af3a19 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/lib.rs @@ -0,0 +1,183 @@ +//! loro-internal is a CRDT framework. +//! +//! +//! +//! +#![deny(clippy::undocumented_unsafe_blocks)] +#![allow(unused_assignments)] +#![allow(clippy::uninlined_format_args)] +#![warn(rustdoc::broken_intra_doc_links)] +#![warn(missing_debug_implementations)] + +pub mod arena; +pub mod diff; +pub mod diff_calc; +pub mod handler; +pub mod sync; +use crate::sync::{AtomicBool, AtomicUsize}; +use std::sync::Arc; +mod change_meta; +pub(crate) mod lock; +use arena::SharedArena; +use configure::Configure; +use diff_calc::DiffCalculator; +use lock::LoroMutex; + +pub use change_meta::ChangeMeta; +pub use event::{ContainerDiff, DiffEvent, DocDiff, ListDiff, ListDiffInsertItem, ListDiffItem}; +pub use handler::{ + BasicHandler, HandlerTrait, ListHandler, MapHandler, MovableListHandler, TextHandler, + TreeHandler, UnknownHandler, +}; +pub use loro_common; +pub use oplog::OpLog; +use pre_commit::{ + FirstCommitFromPeerCallback, FirstCommitFromPeerPayload, PreCommitCallback, + PreCommitCallbackPayload, +}; +pub use rustc_hash::FxHashMap; +pub use state::DocState; +pub use state::{TreeNode, TreeNodeWithChildren, TreeParentId}; +use subscription::{LocalUpdateCallback, Observer, PeerIdUpdateCallback}; +use txn::Transaction; +pub use undo::UndoManager; +pub use utils::subscription::SubscriberSetWithQueue; +pub use utils::subscription::Subscription; +pub mod allocation; +pub mod awareness; +pub mod change; +pub mod configure; +pub mod container; +pub mod cursor; +pub mod dag; +pub mod encoding; +pub(crate) mod fork; +pub mod id; +#[cfg(feature = "jsonpath")] +pub mod jsonpath; +pub mod kv_store; +pub mod loro; +pub mod op; +pub mod oplog; +pub mod subscription; +pub mod txn; +pub mod version; + +mod error; +#[cfg(feature = "test_utils")] +pub mod fuzz; +mod parent; +pub mod pre_commit; +mod span; +#[cfg(test)] +pub mod tests; +mod utils; +pub use utils::string_slice::StringSlice; + +pub mod delta; +pub use loro_delta; +pub mod event; + +pub mod estimated_size; +pub(crate) mod history_cache; +pub(crate) mod macros; +pub(crate) mod state; +pub mod undo; +pub(crate) mod value; + +// TODO: rename as Key? +pub(crate) use loro_common::InternalString; + +pub use container::ContainerType; +pub use encoding::json_schema::json; +pub use fractional_index::FractionalIndex; +pub use loro_common::{loro_value, to_value}; +pub use loro_common::{ + Counter, CounterSpan, IdLp, IdSpan, IdSpanVector, Lamport, LoroEncodeError, LoroError, + LoroResult, LoroTreeError, PeerID, TreeID, ID, +}; +pub use loro_common::{LoroBinaryValue, LoroListValue, LoroMapValue, LoroStringValue}; +#[cfg(feature = "wasm")] +pub use value::wasm; +pub use value::{ApplyDiff, LoroValue, ToJson}; +pub use version::VersionVector; + +/// [`LoroDoc`] serves as the library's primary entry point. +/// It's constituted by an [OpLog] and an [DocState]. +/// +/// - [OpLog] encompasses all operations, signifying the document history. +/// - [DocState] signifies the current document state. +/// +/// They will share a [super::arena::SharedArena] +/// +/// # Detached Mode +/// +/// This mode enables separate usage of [OpLog] and [DocState]. +/// It facilitates temporal navigation. [DocState] can be reverted to +/// any version contained within the [OpLog]. +/// +/// `LoroDoc::detach()` separates [DocState] from [OpLog]. In this mode, +/// updates to [OpLog] won't affect [DocState], while updates to [DocState] +/// will continue to affect [OpLog]. +#[derive(Debug, Clone)] +#[repr(transparent)] +pub struct LoroDoc { + inner: Arc, +} + +impl LoroDoc { + pub(crate) fn from_inner(inner: Arc) -> Self { + Self { inner } + } +} + +impl std::ops::Deref for LoroDoc { + type Target = LoroDocInner; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +pub struct LoroDocInner { + oplog: Arc>, + state: Arc>, + arena: SharedArena, + config: Configure, + visible_op_count: Arc, + observer: Arc, + diff_calculator: Arc>, + /// When dropping the doc, the txn will be committed + /// + /// # Internal Notes + /// + /// Txn can be accessed by different threads. But for certain methods we need to lock the txn and ensure it's empty: + /// + /// - `import` + /// - `export` + /// - `checkout` + /// - `checkout_to_latest` + /// - ... + /// + /// We need to lock txn and keep it None because otherwise the DocState may change due to a parallel edit on a new Txn, + /// which may break the invariants of `import`, `export` and `checkout`. + txn: Arc>>, + auto_commit: AtomicBool, + detached: AtomicBool, + local_update_subs: SubscriberSetWithQueue<(), LocalUpdateCallback, Vec>, + peer_id_change_subs: SubscriberSetWithQueue<(), PeerIdUpdateCallback, ID>, + first_commit_from_peer_subs: + SubscriberSetWithQueue<(), FirstCommitFromPeerCallback, FirstCommitFromPeerPayload>, + pre_commit_subs: SubscriberSetWithQueue<(), PreCommitCallback, PreCommitCallbackPayload>, +} + +/// The version of the loro crate +pub const LORO_VERSION: &str = include_str!("../VERSION"); + +impl Drop for LoroDoc { + fn drop(&mut self) { + if Arc::strong_count(&self.inner) == 1 { + let _ = self.implicit_commit_then_stop(); + } + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/lock.rs b/3rdparty/loro/crates/loro-internal/src/lock.rs new file mode 100644 index 0000000000..7ae65ca0ec --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/lock.rs @@ -0,0 +1,425 @@ +//! Lock-order-checked mutexes. +//! +//! This module provides a small utility to prevent deadlocks by enforcing a +//! strict, per-thread lock acquisition order across a set of related locks. +//! +//! Core ideas: +//! - Locks are created from a [`LoroLockGroup`]. Locks in the same group share +//! a per-thread stack that tracks the last acquired lock kind. +//! - Each lock has an associated [`LockKind`]. A thread may only acquire locks +//! in strictly increasing kind order (e.g. `Txn` → `OpLog` → `DocState` → `DiffCalculator`). +//! - Locks must be released in the reverse order they were acquired (LIFO). +//! - Violations are detected and reported with helpful panics that include the +//! callsite (via `#[track_caller]`) and a backtrace on release-order errors. +//! +//! The actual locking is backed by [`crate::sync::Mutex`], which resolves to +//! `std::sync::Mutex` in normal builds and `loom::sync::Mutex` under loom. This +//! keeps the code testable with loom while maintaining the same API. +//! +//! The per-thread order tracking is debug-only; in release builds those fields +//! and helpers are unused, hence the release-only `allow(dead_code)`. +#![cfg_attr(not(debug_assertions), allow(dead_code))] +use crate::sync::ThreadLocal; +use crate::sync::{Mutex, MutexGuard}; +#[cfg(debug_assertions)] +use std::backtrace::Backtrace; +use std::fmt::{Debug, Display}; +use std::ops::{Deref, DerefMut}; +use std::panic::Location; +use std::sync::Arc; + +/// A mutex that verifies lock acquisition and release order against a group-wide +/// strict ordering by [`LockKind`]. +/// +/// Create instances via [`LoroLockGroup::new_lock`]. Calling [`LoroMutex::lock`] +/// will panic if the current thread has already acquired a lock with a kind that +/// is greater than or equal to this lock’s kind, or if the underlying mutex was +/// poisoned by an earlier panic. Release order is also checked; dropping the +/// guard out of LIFO order results in a panic. +/// +/// This type wraps [`crate::sync::Mutex`], so it remains compatible with loom-based +/// concurrency testing. +#[derive(Debug)] +pub struct LoroMutex { + lock: Mutex, + kind: u8, + currently_locked_in_this_thread: Arc>>, +} + +/// Internal per-thread lock information used for diagnostics and order checks. +#[derive(Debug, Copy, Clone, Default)] +struct LockInfo { + kind: u8, + caller_location: Option<&'static Location<'static>>, +} + +impl Display for LockInfo { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self.caller_location { + Some(location) => write!( + f, + "LockInfo(kind: {}, location: {}:{}:{})", + self.kind, + location.file(), + location.line(), + location.column() + ), + None => write!(f, "LockInfo(kind: {}, location: None)", self.kind), + } + } +} + +#[derive(Debug)] +/// A group that defines a shared locking order domain. +/// +/// All [`LoroMutex`] created from the same group participate in a single, +/// per-thread lock-order stack. Locks from different groups are independent and +/// do not affect each other’s ordering checks. +/// +/// Use [`LoroLockGroup::new_lock`] to create locks of specific [`LockKind`]. +pub struct LoroLockGroup { + g: Arc>>, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +/// Logical lock kinds that define the allowed acquisition order. +/// +/// Kinds with smaller numeric values must be acquired before larger ones, and +/// they must be released in reverse order. The specific variants reflect the +/// high-level components in the system; extend this enum carefully to preserve +/// a consistent global ordering. +pub enum LockKind { + None = 0, + Txn = 1, + OpLog = 2, + DocState = 3, + DiffCalculator = 4, +} + +impl LoroLockGroup { + /// Create a new lock group. + /// + /// Cloning the returned group is cheap. All locks created from clones of + /// the same group still share the same ordering domain. + pub fn new() -> Self { + let g = Arc::new(ThreadLocal::new()); + LoroLockGroup { g } + } + + /// Create a new lock associated with this group and [`LockKind`]. + /// + /// The created lock participates in this group’s order checks. + pub fn new_lock(&self, value: T, kind: LockKind) -> LoroMutex { + LoroMutex { + lock: Mutex::new(value), + currently_locked_in_this_thread: self.g.clone(), + kind: kind as u8, + } + } +} + +impl Default for LoroLockGroup { + fn default() -> Self { + Self::new() + } +} + +impl LoroMutex { + #[track_caller] + /// Acquire the lock, enforcing strict increasing [`LockKind`] order. + /// + /// Returns a guard that unlocks when dropped and verifies that it is being + /// released in the reverse acquisition order (LIFO). The callsite is + /// recorded to improve panic diagnostics. + /// + /// Panics: + /// - If the underlying mutex was poisoned by an earlier panic. + /// - If the current thread already holds a lock with kind `>= self.kind`. + /// - If the guard is later dropped out of acquisition order. + pub fn lock(&self) -> LoroMutexGuard<'_, T> { + // Lock-order tracking is a debug-only deadlock-prevention aid. In release + // builds it is compiled out entirely: per-op locking is on the hot path + // (each local op takes the OpLog + DocState locks) and the per-thread + // bookkeeping (a ThreadLocal lookup plus an inner mutex acquired several + // times per lock/unlock) was measured at ~30% of B4 local-edit time. + #[cfg(debug_assertions)] + { + let caller = Location::caller(); + let v = self.currently_locked_in_this_thread.get_or_default(); + let last = *v.lock(); + let this = LockInfo { + kind: self.kind, + caller_location: Some(caller), + }; + if last.kind >= self.kind { + panic!( + "Locking order violation. Current lock: {}, New lock: {}", + last, this + ); + } + + let guard = self.lock.lock_with_kind("LoroMutex"); + *v.lock() = this; + LoroMutexGuard { + guard, + _inner: LoroMutexGuardInner { + inner: self, + this, + last, + }, + } + } + #[cfg(not(debug_assertions))] + { + let guard = self.lock.lock_with_kind("LoroMutex"); + LoroMutexGuard { + guard, + _inner: LoroMutexGuardInner { inner: self }, + } + } + } + + /// Returns whether the mutex appears locked at this instant. + /// + /// This is implemented via `try_lock().is_err()` and is intended only for + /// diagnostics. It is race-prone and should not be used to implement logic + /// that depends on the lock state. + pub fn is_locked(&self) -> bool { + self.lock.is_locked() + } + + /// Returns whether acquiring this lock would satisfy the current thread's + /// lock-order constraints. + /// + /// This only checks the order tracker for the current thread. It does not + /// guarantee that acquiring the underlying mutex will be non-blocking. + pub(crate) fn can_lock_in_this_thread(&self) -> bool { + #[cfg(debug_assertions)] + { + let v = self.currently_locked_in_this_thread.get_or_default(); + let last = *v.lock(); + last.kind < self.kind + } + // Without lock-order tracking (release) we cannot tell whether acquiring + // this lock would be reentrant/out-of-order, so we conservatively report + // "no" and let callers use their lock-free fallback (e.g. the cached + // `visible_op_count`, which is kept exact incrementally). + #[cfg(not(debug_assertions))] + { + false + } + } +} + +/// Guard returned by [`LoroMutex::lock`]. +/// +/// Dereferences to the protected data and enforces release-order checks on drop. +/// In most cases, you should keep using this guard type so order tracking remains +/// intact for the duration of the critical section. +pub struct LoroMutexGuard<'a, T> { + guard: MutexGuard<'a, T>, + _inner: LoroMutexGuardInner<'a, T>, +} + +/// RAII helper that updates the per-thread lock info on drop. +/// +/// This is an implementation detail of [`LoroMutexGuard`]. The order-tracking +/// fields exist only in debug builds; in release the inner is a thin wrapper. +struct LoroMutexGuardInner<'a, T> { + #[cfg_attr(not(debug_assertions), allow(dead_code))] + inner: &'a LoroMutex, + #[cfg(debug_assertions)] + this: LockInfo, + #[cfg(debug_assertions)] + last: LockInfo, +} + +impl Deref for LoroMutexGuard<'_, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.guard + } +} + +impl DerefMut for LoroMutexGuard<'_, T> { + fn deref_mut(&mut self) -> &mut T { + &mut self.guard + } +} + +impl std::fmt::Debug for LoroMutexGuard<'_, T> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("LoroMutex") + .field("data", &self.guard) + .finish() + } +} + +impl<'a, T> LoroMutexGuard<'a, T> { + /// Extract the underlying [`MutexGuard`], detaching order tracking. + /// + /// This consumes `self`, performs the release-order bookkeeping immediately + /// (making the thread-local lock info revert to the previous state), and + /// returns the raw guard. Subsequent lock acquisitions in this thread will no + /// longer consider this guard as held, which means order violations will not + /// be detected relative to it. + /// + /// Prefer to keep using [`LoroMutexGuard`] unless integrating with APIs that + /// require a plain [`MutexGuard`]. Misuse can lead to missing diagnostics. + pub fn take_guard(self) -> MutexGuard<'a, T> { + self.guard + } +} + +impl Drop for LoroMutexGuardInner<'_, T> { + #[cfg(not(debug_assertions))] + fn drop(&mut self) {} + + #[cfg(debug_assertions)] + fn drop(&mut self) { + let cur = self.inner.currently_locked_in_this_thread.get_or_default(); + let current_lock_info = *cur.lock(); + if current_lock_info.kind != self.this.kind { + let bt = Backtrace::capture(); + eprintln!("Locking release order violation callstack:\n{}", bt); + panic!( + "Locking release order violation. self.this: {}, self.last: {}, current: {}", + self.this, self.last, current_lock_info + ); + } + + *cur.lock() = self.last; + } +} + +// These tests exercise the lock-order instrumentation, which is only compiled +// in debug builds. Skip them under `cargo test --release`. +#[cfg(all(test, debug_assertions))] +mod tests { + use super::*; + + #[test] + #[should_panic(expected = "Locking order violation")] + fn test_locking_order_violation_shows_caller() { + let group = LoroLockGroup::new(); + let mutex1 = group.new_lock(1, LockKind::DocState); + let mutex2 = group.new_lock(2, LockKind::Txn); + + let _guard1 = mutex1.lock(); // Lock higher priority first + let _guard2 = mutex2.lock(); // This should panic with caller info + } + + #[test] + fn test_locking_order_when_dropped_in_order() { + let group = LoroLockGroup::new(); + let mutex1 = group.new_lock(1, LockKind::Txn); + let mutex2 = group.new_lock(2, LockKind::OpLog); + let mutex3 = group.new_lock(3, LockKind::DocState); + let _guard1 = mutex1.lock(); + drop(_guard1); + let _guard2 = mutex2.lock(); + drop(_guard2); + let _guard3 = mutex3.lock(); + } + + #[test] + #[should_panic] + fn test_locking_order_when_not_dropped_in_reverse_order() { + let group = LoroLockGroup::new(); + let mutex1 = group.new_lock(1, LockKind::Txn); + let mutex2 = group.new_lock(2, LockKind::OpLog); + let _guard1 = mutex1.lock(); + let _guard2 = mutex2.lock(); + drop(_guard1); + drop(_guard2); + } + + #[test] + fn test_dropping_should_restore_last_lock_info_0() { + let group = LoroLockGroup::new(); + let mutex1 = group.new_lock(1, LockKind::Txn); + let mutex2 = group.new_lock(2, LockKind::OpLog); + let mutex3 = group.new_lock(3, LockKind::DocState); + let _guard1 = mutex1.lock(); + let _guard3 = mutex3.lock(); + drop(_guard3); + let _guard2 = mutex2.lock(); + drop(_guard2); + } + + #[test] + #[should_panic] + fn test_dropping_should_restore_last_lock_info_1() { + let group = LoroLockGroup::new(); + let mutex1 = group.new_lock(1, LockKind::Txn); + let mutex2 = group.new_lock(2, LockKind::OpLog); + let mutex3 = group.new_lock(3, LockKind::DocState); + let _guard2 = mutex2.lock(); + let _guard3 = mutex3.lock(); + drop(_guard3); + let _guard1 = mutex1.lock(); + } + + #[test] + fn test_nested_locking_same_kind() { + let group = LoroLockGroup::new(); + let mutex1 = group.new_lock(1, LockKind::Txn); + let mutex2 = group.new_lock(2, LockKind::Txn); + + let guard1 = mutex1.lock(); + // Locking same kind should work (cur >= self.kind, so this would fail) + // Actually, let's test this properly - same kind should fail + drop(guard1); + + let _guard2 = mutex2.lock(); // This should work when guard1 is dropped + } + + #[test] + fn test_lock_kind_enum_values() { + assert_eq!(LockKind::None as u8, 0); + assert_eq!(LockKind::Txn as u8, 1); + assert_eq!(LockKind::OpLog as u8, 2); + assert_eq!(LockKind::DocState as u8, 3); + assert_eq!(LockKind::DiffCalculator as u8, 4); + } + + #[test] + fn test_is_locked_functionality() { + let group = LoroLockGroup::new(); + let mutex = group.new_lock(42, LockKind::Txn); + + assert!(!mutex.is_locked()); + + let _guard = mutex.lock(); + assert!(mutex.is_locked()); + } + + // Helper function to test that panic messages contain caller info + #[test] + #[should_panic(expected = "Locking order violation")] + fn test_panic_message_contains_location_info() { + let group = LoroLockGroup::new(); + let mutex1 = group.new_lock(1, LockKind::DocState); + let mutex2 = group.new_lock(2, LockKind::Txn); + + let _guard1 = mutex1.lock(); + + // This line should be reported in the panic message + let _guard2 = mutex2.lock(); + } + + #[test] + #[should_panic(expected = "poisoned LoroMutex")] + fn test_poisoned_lock_panics_after_unwind() { + let group = LoroLockGroup::new(); + let mutex = group.new_lock(42, LockKind::Txn); + + let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _guard = mutex.lock(); + panic!("poison the lock"); + })); + + let _ = mutex.lock(); + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/loro.rs b/3rdparty/loro/crates/loro-internal/src/loro.rs new file mode 100644 index 0000000000..4c8a74f560 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/loro.rs @@ -0,0 +1,2976 @@ +use crate::encoding::json_schema::{encode_change, export_json_in_id_span}; +pub use crate::encoding::ExportMode; +use crate::pre_commit::{FirstCommitFromPeerCallback, FirstCommitFromPeerPayload}; +pub use crate::state::analyzer::{ContainerAnalysisInfo, DocAnalysis}; +use crate::sync::{AtomicBool, AtomicUsize}; +pub(crate) use crate::LoroDocInner; +use crate::{ + arena::SharedArena, + change::{Change, Timestamp}, + configure::{Configure, DefaultRandom, SecureRandomGenerator, StyleConfig}, + container::{ + idx::ContainerIdx, list::list_op::InnerListOp, richtext::config::StyleConfigMap, + IntoContainerId, + }, + cursor::{AbsolutePosition, CannotFindRelativePosition, Cursor, PosQueryResult}, + dag::{Dag, DagUtils}, + diff_calc::DiffCalculator, + encoding::{ + self, decode_snapshot, export_fast_snapshot, export_fast_updates, + export_fast_updates_in_range, export_shallow_snapshot, export_snapshot_at, + export_state_only_snapshot, + json_schema::{encode_change_to_json, json::JsonSchema}, + parse_header_and_body, EncodeMode, ImportBlobMetadata, ImportStatus, ParsedHeaderAndBody, + }, + event::{str_to_path, EventTriggerKind, Index, InternalDocDiff}, + handler::{Handler, MovableListHandler, TextHandler, TreeHandler, ValueOrHandler}, + id::PeerID, + json::JsonChange, + op::InnerContent, + oplog::{loro_dag::FrontiersNotIncluded, OpLog}, + state::DocState, + subscription::{LocalUpdateCallback, Observer, Subscriber}, + undo::DiffBatch, + utils::subscription::{SubscriberSetWithQueue, Subscription}, + version::{shrink_frontiers, Frontiers, ImVersionVector, VersionRange, VersionVectorDiff}, + ChangeMeta, DocDiff, HandlerTrait, InternalString, ListHandler, LoroDoc, LoroError, MapHandler, + VersionVector, +}; +use crate::{change::ChangeRef, lock::LockKind}; +use crate::{lock::LoroMutexGuard, pre_commit::PreCommitCallback}; +use crate::{ + lock::{LoroLockGroup, LoroMutex}, + txn::Transaction, +}; +use either::Either; +use loro_common::{ + ContainerID, ContainerType, HasCounterSpan, HasIdSpan, HasLamportSpan, IdSpan, LoroEncodeError, + LoroResult, LoroValue, ID, +}; +use rle::HasLength; +use rustc_hash::{FxHashMap, FxHashSet}; +use std::{ + borrow::Cow, + cmp::Ordering, + collections::{hash_map::Entry, BinaryHeap}, + ops::ControlFlow, + sync::{ + atomic::Ordering::{Acquire, Release}, + Arc, + }, +}; +use tracing::{debug_span, info_span, instrument, warn}; + +impl Default for LoroDoc { + fn default() -> Self { + Self::new() + } +} + +impl std::fmt::Debug for LoroDocInner { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("LoroDoc") + .field("config", &self.config) + .field("auto_commit", &self.auto_commit) + .field("detached", &self.detached) + .finish() + } +} + +impl LoroDoc { + /// Run the provided closure within a commit barrier. + /// + /// This finalizes any pending auto-commit transaction first (preserving + /// options across an empty txn), executes `f`, then renews the transaction + /// (carrying preserved options) if auto-commit is enabled. This is the + /// common implicit-commit pattern used by internal operations such as + /// import/export/checkouts. + #[inline] + pub fn with_barrier(&self, f: F) -> R + where + F: FnOnce() -> R, + { + let (options, guard) = self.implicit_commit_then_stop(); + let result = f(); + drop(guard); + self.renew_txn_if_auto_commit(options); + result + } + + pub fn new() -> Self { + let visible_op_count = Arc::new(AtomicUsize::new(0)); + let oplog = OpLog::new(visible_op_count.clone()); + let arena = oplog.arena.clone(); + let config: Configure = oplog.configure.clone(); + let lock_group = LoroLockGroup::new(); + let global_txn = Arc::new(lock_group.new_lock(None, LockKind::Txn)); + let inner = Arc::new_cyclic(|w| { + let state = DocState::new_arc(w.clone(), arena.clone(), config.clone(), &lock_group); + LoroDocInner { + oplog: Arc::new(lock_group.new_lock(oplog, LockKind::OpLog)), + state, + config, + visible_op_count, + detached: AtomicBool::new(false), + auto_commit: AtomicBool::new(false), + observer: Arc::new(Observer::new(arena.clone())), + diff_calculator: Arc::new( + lock_group.new_lock(DiffCalculator::new(true), LockKind::DiffCalculator), + ), + txn: global_txn, + arena, + local_update_subs: SubscriberSetWithQueue::new(), + peer_id_change_subs: SubscriberSetWithQueue::new(), + pre_commit_subs: SubscriberSetWithQueue::new(), + first_commit_from_peer_subs: SubscriberSetWithQueue::new(), + } + }); + LoroDoc { inner } + } + + pub fn fork(&self) -> Self { + if self.is_detached() { + return self + .fork_at(&self.state_frontiers()) + .expect("fork_at on detached doc should not fail"); + } + + let snapshot = self.with_barrier(|| encoding::fast_snapshot::encode_snapshot_inner(self)); + let doc = Self::new(); + doc.with_barrier(|| { + encoding::fast_snapshot::decode_snapshot_inner(snapshot, &doc, Default::default()) + }) + .unwrap(); + doc.set_config(&self.config); + if self.auto_commit.load(std::sync::atomic::Ordering::Relaxed) { + doc.start_auto_commit(); + } + doc + } + /// Enables editing of the document in detached mode. + /// + /// By default, the document cannot be edited in detached mode (after calling + /// `detach` or checking out a version other than the latest). This method + /// allows editing in detached mode. + /// + /// # Important Notes: + /// + /// - After enabling this mode, the document will use a different PeerID. Each + /// time you call checkout, a new PeerID will be used. + /// - If you set a custom PeerID while this mode is enabled, ensure that + /// concurrent operations with the same PeerID are not possible. + /// - On detached mode, importing will not change the state of the document. + /// It also doesn't change the version of the [DocState]. The changes will be + /// recorded into [OpLog] only. You need to call `checkout` to make it take effect. + pub fn set_detached_editing(&self, enable: bool) { + self.config.set_detached_editing(enable); + if enable && self.is_detached() { + self.with_barrier(|| { + self.renew_peer_id(); + }); + } + } + + /// Create a doc with auto commit enabled. + #[inline] + pub fn new_auto_commit() -> Self { + let doc = Self::new(); + doc.start_auto_commit(); + doc + } + + #[inline(always)] + pub fn set_peer_id(&self, peer: PeerID) -> LoroResult<()> { + if peer == PeerID::MAX { + return Err(LoroError::InvalidPeerID); + } + let next_id = self.oplog.lock().next_id(peer); + if self.auto_commit.load(Acquire) { + let doc_state = self.state.lock(); + doc_state + .peer + .store(peer, std::sync::atomic::Ordering::Relaxed); + + if doc_state.is_in_txn() { + drop(doc_state); + // Use implicit-style barrier to avoid swallowing next-commit options + self.with_barrier(|| {}); + } + self.peer_id_change_subs.emit(&(), next_id); + return Ok(()); + } + + let doc_state = self.state.lock(); + if doc_state.is_in_txn() { + return Err(LoroError::TransactionError( + "Cannot change peer id during transaction" + .to_string() + .into_boxed_str(), + )); + } + + doc_state + .peer + .store(peer, std::sync::atomic::Ordering::Relaxed); + drop(doc_state); + self.peer_id_change_subs.emit(&(), next_id); + Ok(()) + } + + /// Renews the PeerID for the document. + pub(crate) fn renew_peer_id(&self) { + let mut peer_id = DefaultRandom.next_u64(); + while peer_id == PeerID::MAX { + peer_id = DefaultRandom.next_u64(); + } + self.set_peer_id(peer_id).unwrap(); + } + + /// Implicitly commit the cumulative auto-commit transaction. + /// This method only has effect when `auto_commit` is true. + /// + /// Follow-ups: the caller is responsible for renewing the transaction + /// as needed (e.g., via `renew_txn_if_auto_commit`). Prefer using + /// `with_barrier(...)` for most internal flows to handle this safely. + /// + /// Empty-commit behavior: if the pending transaction is empty, the returned + /// `Some(CommitOptions)` preserves next-commit options such as message and + /// timestamp so they can carry into the renewed transaction. Transient + /// labels like `origin` do not carry across an empty commit. + #[inline] + #[must_use] + pub fn implicit_commit_then_stop( + &self, + ) -> ( + Option, + LoroMutexGuard<'_, Option>, + ) { + // Implicit commit: preserve options on empty commit + let (a, b) = self.commit_internal(CommitOptions::new().immediate_renew(false), true); + (a, b.unwrap()) + } + + /// Commit the cumulative auto commit transaction. + /// It will start the next one immediately + /// + /// It only returns Some(options_of_the_empty_txn) when the txn is empty + #[inline] + pub fn commit_then_renew(&self) -> Option { + // Explicit commit: swallow options on empty commit + self.commit_internal(CommitOptions::new().immediate_renew(true), false) + .0 + } + + /// This method is called before the commit. + /// It can be used to modify the change before it is committed. + /// + /// It return Some(txn) if the txn is None + fn before_commit(&self) -> Option>> { + let mut txn_guard = self.txn.lock(); + let Some(txn) = txn_guard.as_mut() else { + return Some(txn_guard); + }; + + if txn.is_peer_first_appearance { + txn.is_peer_first_appearance = false; + drop(txn_guard); + // First commit from a peer + self.first_commit_from_peer_subs.emit( + &(), + FirstCommitFromPeerPayload { + peer: self.peer_id(), + }, + ); + } + + None + } + + /// Core implementation for committing the cumulative auto-commit transaction. + /// + /// - When `preserve_on_empty` is true (implicit commits like export/checkout), + /// commit options from an empty transaction are carried over to the next transaction + /// (except `origin`, which never carries across an empty commit). + /// - When `preserve_on_empty` is false (explicit commits), commit options from an + /// empty transaction are swallowed and NOT carried over. + #[instrument(skip_all)] + fn commit_internal( + &self, + config: CommitOptions, + preserve_on_empty: bool, + ) -> ( + Option, + Option>>, + ) { + if !self.auto_commit.load(Acquire) { + let txn_guard = self.txn.lock(); + // if not auto_commit, nothing should happen + // because the global txn is not used + return (None, Some(txn_guard)); + } + + loop { + if let Some(txn_guard) = self.before_commit() { + return (None, Some(txn_guard)); + } + + let mut txn_guard = self.txn.lock(); + let txn = txn_guard.take(); + let Some(mut txn) = txn else { + return (None, Some(txn_guard)); + }; + let on_commit = txn.take_on_commit(); + if let Some(origin) = config.origin.clone() { + txn.set_origin(origin); + } + + if let Some(timestamp) = config.timestamp { + txn.set_timestamp(timestamp); + } + + if let Some(msg) = config.commit_msg.as_ref() { + txn.set_msg(Some(msg.clone())); + } + + let id_span = txn.id_span(); + let mut options = txn.commit().unwrap(); + // Empty commit returns Some(options). We may preserve parts of it for implicit commits. + if let Some(opts) = options.as_mut() { + // `origin` is an event-only label and never carries across an empty commit + if config.origin.is_some() { + opts.set_origin(None); + } + // For explicit commits, swallow options from empty commit entirely + if !preserve_on_empty { + options = None; + } + } + if config.immediate_renew && self.can_edit() { + let mut t = self.txn().unwrap(); + if let Some(options) = options.as_ref() { + t.set_options(options.clone()); + } + *txn_guard = Some(t); + } + + if let Some(on_commit) = on_commit { + drop(txn_guard); + on_commit(&self.state, &self.oplog, id_span); + txn_guard = self.txn.lock(); + if !config.immediate_renew && txn_guard.is_some() { + // make sure that txn_guard is None when config.immediate_renew is false + continue; + } + } + + return ( + options, + if !config.immediate_renew { + Some(txn_guard) + } else { + None + }, + ); + } + } + + /// Commit the cumulative auto commit transaction (explicit API). + /// + /// This is used by user-facing explicit commits. If the transaction is empty, + /// any provided commit options are swallowed and will NOT carry over. + #[instrument(skip_all)] + pub fn commit_with( + &self, + config: CommitOptions, + ) -> ( + Option, + Option>>, + ) { + self.commit_internal(config, false) + } + + /// Set the commit message of the next commit + pub fn set_next_commit_message(&self, message: &str) { + let mut binding = self.txn.lock(); + let Some(txn) = binding.as_mut() else { + return; + }; + + if message.is_empty() { + txn.set_msg(None) + } else { + txn.set_msg(Some(message.into())) + } + } + + /// Set the origin of the next commit + pub fn set_next_commit_origin(&self, origin: &str) { + let mut txn = self.txn.lock(); + if let Some(txn) = txn.as_mut() { + txn.set_origin(origin.into()); + } + } + + /// Set the timestamp of the next commit + pub fn set_next_commit_timestamp(&self, timestamp: Timestamp) { + let mut txn = self.txn.lock(); + if let Some(txn) = txn.as_mut() { + txn.set_timestamp(timestamp); + } + } + + /// Set the options of the next commit + pub fn set_next_commit_options(&self, options: CommitOptions) { + let mut txn = self.txn.lock(); + if let Some(txn) = txn.as_mut() { + txn.set_options(options); + } + } + + /// Clear the options of the next commit + pub fn clear_next_commit_options(&self) { + let mut txn = self.txn.lock(); + if let Some(txn) = txn.as_mut() { + txn.set_options(CommitOptions::new()); + } + } + + /// Set whether to record the timestamp of each change. Default is `false`. + /// + /// If enabled, the Unix timestamp will be recorded for each change automatically. + /// + /// You can also set each timestamp manually when you commit a change. + /// The timestamp manually set will override the automatic one. + /// + /// NOTE: Timestamps are forced to be in ascending order. + /// If you commit a new change with a timestamp that is less than the existing one, + /// the largest existing timestamp will be used instead. + #[inline] + pub fn set_record_timestamp(&self, record: bool) { + self.config.set_record_timestamp(record); + } + + /// Set the interval of mergeable changes, in seconds. + /// + /// If two continuous local changes are within the interval, they will be merged into one change. + /// The default value is 1000 seconds. + #[inline] + pub fn set_change_merge_interval(&self, interval: i64) { + self.config.set_merge_interval(interval); + } + + pub fn can_edit(&self) -> bool { + !self.is_detached() || self.config.detached_editing() + } + + pub fn is_detached_editing_enabled(&self) -> bool { + self.config.detached_editing() + } + + #[inline] + pub fn config_text_style(&self, text_style: StyleConfigMap) { + self.config.text_style_config.write().map = text_style.map; + } + + #[inline] + pub fn config_default_text_style(&self, text_style: Option) { + self.config.text_style_config.write().default_style = text_style; + } + pub fn from_snapshot(bytes: &[u8]) -> LoroResult { + let doc = Self::new(); + let ParsedHeaderAndBody { mode, body, .. } = parse_header_and_body(bytes, true)?; + if mode.is_snapshot() { + doc.with_barrier(|| -> Result<(), LoroError> { + decode_snapshot(&doc, mode, body, Default::default())?; + Ok(()) + })?; + Ok(doc) + } else { + Err(LoroError::DecodeError( + "Invalid encode mode".to_string().into(), + )) + } + } + + /// Is the document empty? (no ops) + #[inline(always)] + pub fn can_reset_with_snapshot(&self) -> bool { + let oplog = self.oplog.lock(); + if oplog.batch_importing { + return false; + } + + if self.is_detached() { + return false; + } + + oplog.is_empty() && self.state.lock().can_import_snapshot() + } + + /// Whether [OpLog] and [DocState] are detached. + /// + /// If so, the document is in readonly mode by default and importing will not change the state of the document. + /// It also doesn't change the version of the [DocState]. The changes will be recorded into [OpLog] only. + /// You need to call `checkout` to make it take effect. + #[inline(always)] + pub fn is_detached(&self) -> bool { + self.detached.load(Acquire) + } + + pub(crate) fn set_detached(&self, detached: bool) { + self.detached.store(detached, Release); + } + + #[inline(always)] + pub fn peer_id(&self) -> PeerID { + self.state + .lock() + .peer + .load(std::sync::atomic::Ordering::Relaxed) + } + + #[inline(always)] + pub fn detach(&self) { + self.with_barrier(|| self.set_detached(true)); + } + + #[inline(always)] + pub fn attach(&self) { + self.checkout_to_latest() + } + + /// Get the timestamp of the current state. + /// It's the last edit time of the [DocState]. + pub fn state_timestamp(&self) -> Timestamp { + // Acquire locks in correct order: read frontiers first, then query OpLog. + let f = { self.state.lock().frontiers.clone() }; + self.oplog.lock().get_timestamp_of_version(&f) + } + + #[inline(always)] + pub fn app_state(&self) -> &Arc> { + &self.state + } + + #[inline] + pub fn get_state_deep_value(&self) -> LoroValue { + self.state.lock().get_deep_value() + } + + #[inline(always)] + pub fn oplog(&self) -> &Arc> { + &self.oplog + } + + #[inline(always)] + pub fn import(&self, bytes: &[u8]) -> Result { + let s = debug_span!("import", peer = self.peer_id()); + let _e = s.enter(); + self.import_with(bytes, Default::default()) + } + + #[inline] + pub fn import_with( + &self, + bytes: &[u8], + origin: InternalString, + ) -> Result { + self.with_barrier(|| self._import_with(bytes, origin)) + } + + #[tracing::instrument(skip_all)] + fn _import_with( + &self, + bytes: &[u8], + origin: InternalString, + ) -> Result { + ensure_cov::notify_cov("loro_internal::import"); + let parsed = parse_header_and_body(bytes, true)?; + loro_common::info!("Importing with mode={:?}", &parsed.mode); + let result = match parsed.mode { + EncodeMode::OutdatedRle => { + if self.state.lock().is_in_txn() { + return Err(LoroError::ImportWhenInTxn); + } + + let s = tracing::span!( + tracing::Level::INFO, + "Import updates ", + peer = self.peer_id() + ); + let _e = s.enter(); + self.update_oplog_and_apply_delta_to_state_if_needed( + |oplog| oplog.decode(parsed), + origin, + ) + } + EncodeMode::OutdatedSnapshot => { + if self.can_reset_with_snapshot() { + loro_common::info!("Init by snapshot {}", self.peer_id()); + decode_snapshot(self, parsed.mode, parsed.body, origin) + } else { + self.update_oplog_and_apply_delta_to_state_if_needed( + |oplog| oplog.decode(parsed), + origin, + ) + } + } + EncodeMode::FastSnapshot => { + if self.can_reset_with_snapshot() { + ensure_cov::notify_cov("loro_internal::import::snapshot"); + loro_common::info!("Init by fast snapshot {}", self.peer_id()); + decode_snapshot(self, parsed.mode, parsed.body, origin) + } else { + self.import_changes_and_apply_delta_to_state_if_needed( + |oplog| encoding::decode_oplog_changes(oplog, parsed), + origin, + ) + + // let new_doc = LoroDoc::new(); + // new_doc.import(bytes)?; + // let updates = new_doc.export(ExportMode::updates(&self.oplog_vv())).unwrap(); + // return self.import_with(updates.as_slice(), origin); + } + } + EncodeMode::FastUpdates => self.import_changes_and_apply_delta_to_state_if_needed( + |oplog| encoding::decode_oplog_changes(oplog, parsed), + origin, + ), + EncodeMode::Auto => { + unreachable!() + } + }; + + self.emit_events(); + + result + } + + #[tracing::instrument(skip_all)] + pub(crate) fn update_oplog_and_apply_delta_to_state_if_needed( + &self, + f: impl FnOnce(&mut OpLog) -> Result, + origin: InternalString, + ) -> Result { + let mut oplog = self.oplog.lock(); + oplog.begin_import_rollback(); + if !self.is_detached() { + let old_vv = oplog.vv().clone(); + let old_frontiers = oplog.frontiers().clone(); + let result = f(&mut oplog); + if &old_vv != oplog.vv() { + let mut diff = DiffCalculator::new(false); + let (diff, diff_mode) = diff.calc_diff_internal( + &oplog, + &old_vv, + &old_frontiers, + oplog.vv(), + oplog.dag.get_frontiers(), + None, + ); + let mut state = self.state.lock(); + if let Err(e) = state.apply_diff( + InternalDocDiff { + origin, + diff: (diff).into(), + by: EventTriggerKind::Import, + new_version: Cow::Owned(oplog.frontiers().clone()), + }, + diff_mode, + ) { + oplog.rollback_import(); + return Err(e); + } + } + match result { + Ok(result) => { + oplog.commit_import_rollback(); + Ok(result) + } + Err(e) => { + // Some import errors are reported after a valid prefix has + // been inserted into the oplog. Keep that prefix if it was + // also applied to state; rollback is for failed state apply + // or decode errors that made no visible oplog progress. + if &old_vv == oplog.vv() { + oplog.rollback_import(); + } else { + oplog.commit_import_rollback(); + } + Err(e) + } + } + } else { + match f(&mut oplog) { + Ok(result) => { + oplog.commit_import_rollback(); + Ok(result) + } + Err(e) => { + oplog.rollback_import(); + Err(e) + } + } + } + } + + #[tracing::instrument(skip_all)] + pub(crate) fn import_changes_and_apply_delta_to_state_if_needed( + &self, + decode_changes: impl FnOnce(&mut OpLog) -> Result, LoroError>, + origin: InternalString, + ) -> Result { + let mut oplog = self.oplog.lock(); + let arena_checkpoint = oplog.arena.checkpoint_for_rollback(); + let changes = match decode_changes(&mut oplog) { + Ok(changes) => changes, + Err(e) => { + oplog.arena.rollback(arena_checkpoint); + return Err(e); + } + }; + + let preflight = oplog.preflight_import_changes(&changes); + if preflight.has_deps_before_shallow_root + && (self.is_detached() || !preflight.applies_to_dag) + { + oplog.arena.rollback(arena_checkpoint); + return Err(LoroError::ImportUpdatesThatDependsOnOutdatedVersion); + } + + if self.is_detached() { + let result = encoding::apply_decoded_changes_to_oplog(&mut oplog, changes); + if result.has_deps_before_shallow_root { + return Err(LoroError::ImportUpdatesThatDependsOnOutdatedVersion); + } + + return Ok(result.status); + } + + if !preflight.applies_to_dag { + let pending_root_containers = pending_root_containers_to_materialize(&oplog, &changes); + let result = encoding::apply_decoded_changes_to_oplog(&mut oplog, changes); + if result.has_deps_before_shallow_root { + oplog.arena.rollback(arena_checkpoint); + return Err(LoroError::ImportUpdatesThatDependsOnOutdatedVersion); + } + + if !pending_root_containers.is_empty() { + let mut state = self.state.lock(); + for id in pending_root_containers { + state.ensure_container(&id); + } + } + + return Ok(result.status); + } + + let old_vv = oplog.vv().clone(); + let old_frontiers = oplog.frontiers().clone(); + let rollback_enabled = preflight.needs_state_apply_rollback; + if rollback_enabled { + oplog.begin_import_rollback_with_arena(arena_checkpoint); + } + + let result = encoding::apply_decoded_changes_to_oplog(&mut oplog, changes); + if &old_vv != oplog.vv() { + let mut diff = DiffCalculator::new(false); + let (diff, diff_mode) = diff.calc_diff_internal( + &oplog, + &old_vv, + &old_frontiers, + oplog.vv(), + oplog.dag.get_frontiers(), + None, + ); + let mut state = self.state.lock(); + if let Err(e) = state.apply_diff( + InternalDocDiff { + origin, + diff: (diff).into(), + by: EventTriggerKind::Import, + new_version: Cow::Owned(oplog.frontiers().clone()), + }, + diff_mode, + ) { + if rollback_enabled { + oplog.rollback_import(); + return Err(e); + } + + panic!("state apply returned Err for import without rollback guard: {e}"); + } + } + + if result.has_deps_before_shallow_root { + if rollback_enabled { + oplog.commit_import_rollback(); + } + return Err(LoroError::ImportUpdatesThatDependsOnOutdatedVersion); + } + + if rollback_enabled { + oplog.commit_import_rollback(); + } + Ok(result.status) + } + + fn emit_events(&self) { + // we should not hold the lock when emitting events + let events = { + let mut state = self.state.lock(); + state.take_events() + }; + for event in events { + self.observer.emit(event); + } + } + + pub(crate) fn drop_pending_events(&self) -> Vec { + let mut state = self.state.lock(); + state.take_events() + } + + /// Import the json schema updates. + /// + /// only supports backward compatibility but not forward compatibility. + #[tracing::instrument(skip_all)] + pub fn import_json_updates>(&self, json: T) -> LoroResult { + let json = json.try_into().map_err(|_| LoroError::InvalidJsonSchema)?; + self.with_barrier(|| { + let result = self.import_changes_and_apply_delta_to_state_if_needed( + |oplog| crate::encoding::json_schema::decode_json_changes(json, &oplog.arena), + Default::default(), + ); + self.emit_events(); + result + }) + } + + pub fn export_json_updates( + &self, + start_vv: &VersionVector, + end_vv: &VersionVector, + with_peer_compression: bool, + ) -> JsonSchema { + self.with_barrier(|| { + let oplog = self.oplog.lock(); + let mut start_vv = start_vv; + let _temp: Option; + if !oplog.dag.shallow_since_vv().is_empty() { + // Make sure that start_vv >= shallow_since_vv + let mut include_all = true; + for (peer, counter) in oplog.dag.shallow_since_vv().iter() { + if start_vv.get(peer).unwrap_or(&0) < counter { + include_all = false; + break; + } + } + if !include_all { + let mut vv = start_vv.clone(); + for (&peer, &counter) in oplog.dag.shallow_since_vv().iter() { + vv.extend_to_include_end_id(ID::new(peer, counter)); + } + _temp = Some(vv); + start_vv = _temp.as_ref().unwrap(); + } + } + + crate::encoding::json_schema::export_json( + &oplog, + start_vv, + end_vv, + with_peer_compression, + ) + }) + } + + pub fn export_json_in_id_span(&self, id_span: IdSpan) -> Vec { + let oplog = self.oplog.lock(); + let mut changes = export_json_in_id_span(&oplog, id_span); + if let Some(uncommit) = oplog.get_uncommitted_change_in_span(id_span) { + let change_json = encode_change(ChangeRef::from_change(&uncommit), &self.arena, None); + changes.push(change_json); + } + changes + } + + /// Get the version vector of the current OpLog + #[inline] + pub fn oplog_vv(&self) -> VersionVector { + self.oplog.lock().vv().clone() + } + + /// Get the version vector of the current [DocState] + #[inline] + pub fn state_vv(&self) -> VersionVector { + let oplog = self.oplog.lock(); + let f = &self.state.lock().frontiers; + oplog.dag.frontiers_to_vv(f).unwrap() + } + + pub fn get_by_path(&self, path: &[Index]) -> Option { + let value: LoroValue = self.state.lock().get_value_by_path(path)?; + if let LoroValue::Container(c) = value { + Some(ValueOrHandler::Handler(Handler::new_attached( + c.clone(), + self.clone(), + ))) + } else { + Some(ValueOrHandler::Value(value)) + } + } + + /// Get the handler by the string path. + pub fn get_by_str_path(&self, path: &str) -> Option { + let path = str_to_path(path)?; + self.get_by_path(&path) + } + + pub fn get_uncommitted_ops_as_json(&self) -> Option { + let arena = &self.arena; + let txn = self.txn.lock(); + let txn = txn.as_ref()?; + let ops_ = txn.local_ops(); + let new_id = ID { + peer: *txn.peer(), + counter: ops_.first()?.counter, + }; + let change = ChangeRef { + id: &new_id, + deps: txn.frontiers(), + timestamp: &txn + .timestamp() + .as_ref() + .copied() + .unwrap_or_else(|| self.oplog.lock().get_timestamp_for_next_txn()), + commit_msg: txn.msg(), + ops: ops_, + lamport: txn.lamport(), + }; + let json = encode_change_to_json(change, arena); + Some(json) + } + + #[inline] + pub fn get_handler(&self, id: ContainerID) -> Option { + if self.has_container(&id) { + self.ensure_root_container(&id); + Some(Handler::new_attached(id, self.clone())) + } else { + None + } + } + + #[inline] + fn ensure_root_container(&self, id: &ContainerID) { + // Mergeable roots stay lazily materialized: their existence is derived from the + // parent map's child ref, and their state is only created on first op (or at + // export via the alive-container walk). + if id.is_root() && !id.is_mergeable() { + self.state.lock().ensure_container(id); + } + } + + /// id can be a str, ContainerID, or ContainerIdRaw. + /// if it's str it will use Root container, which will not be None + #[inline] + pub fn try_get_text(&self, id: I) -> Option { + let id = id.into_container_id(&self.arena, ContainerType::Text); + if !self.has_container(&id) { + return None; + } + self.ensure_root_container(&id); + Handler::new_attached(id, self.clone()).into_text().ok() + } + + /// id can be a str, ContainerID, or ContainerIdRaw. + /// if it's str it will use Root container, which will not be None + #[inline] + pub fn get_text(&self, id: I) -> TextHandler { + self.try_get_text(id) + .expect("The container does not exist in the document. Use `try_get_text` or `get_container` to check for existence.") + } + + /// id can be a str, ContainerID, or ContainerIdRaw. + /// if it's str it will use Root container, which will not be None + #[inline] + pub fn try_get_list(&self, id: I) -> Option { + let id = id.into_container_id(&self.arena, ContainerType::List); + if !self.has_container(&id) { + return None; + } + self.ensure_root_container(&id); + Handler::new_attached(id, self.clone()).into_list().ok() + } + + /// id can be a str, ContainerID, or ContainerIdRaw. + /// if it's str it will use Root container, which will not be None + #[inline] + pub fn get_list(&self, id: I) -> ListHandler { + self.try_get_list(id) + .expect("The container does not exist in the document. Use `try_get_list` or `get_container` to check for existence.") + } + + /// id can be a str, ContainerID, or ContainerIdRaw. + /// if it's str it will use Root container, which will not be None + #[inline] + pub fn try_get_movable_list(&self, id: I) -> Option { + let id = id.into_container_id(&self.arena, ContainerType::MovableList); + if !self.has_container(&id) { + return None; + } + self.ensure_root_container(&id); + Handler::new_attached(id, self.clone()) + .into_movable_list() + .ok() + } + + /// id can be a str, ContainerID, or ContainerIdRaw. + /// if it's str it will use Root container, which will not be None + #[inline] + pub fn get_movable_list(&self, id: I) -> MovableListHandler { + self.try_get_movable_list(id) + .expect("The container does not exist in the document. Use `try_get_movable_list` or `get_container` to check for existence.") + } + + /// id can be a str, ContainerID, or ContainerIdRaw. + /// if it's str it will use Root container, which will not be None + #[inline] + pub fn try_get_map(&self, id: I) -> Option { + let id = id.into_container_id(&self.arena, ContainerType::Map); + if !self.has_container(&id) { + return None; + } + self.ensure_root_container(&id); + Handler::new_attached(id, self.clone()).into_map().ok() + } + + /// id can be a str, ContainerID, or ContainerIdRaw. + /// if it's str it will use Root container, which will not be None + #[inline] + pub fn get_map(&self, id: I) -> MapHandler { + self.try_get_map(id) + .expect("The container does not exist in the document. Use `try_get_map` or `get_container` to check for existence.") + } + + /// id can be a str, ContainerID, or ContainerIdRaw. + /// if it's str it will use Root container, which will not be None + #[inline] + pub fn try_get_tree(&self, id: I) -> Option { + let id = id.into_container_id(&self.arena, ContainerType::Tree); + if !self.has_container(&id) { + return None; + } + self.ensure_root_container(&id); + Handler::new_attached(id, self.clone()).into_tree().ok() + } + + /// id can be a str, ContainerID, or ContainerIdRaw. + /// if it's str it will use Root container, which will not be None + #[inline] + pub fn get_tree(&self, id: I) -> TreeHandler { + self.try_get_tree(id) + .expect("The container does not exist in the document. Use `try_get_tree` or `get_container` to check for existence.") + } + + #[cfg(feature = "counter")] + pub fn try_get_counter( + &self, + id: I, + ) -> Option { + let id = id.into_container_id(&self.arena, ContainerType::Counter); + if !self.has_container(&id) { + return None; + } + self.ensure_root_container(&id); + Handler::new_attached(id, self.clone()).into_counter().ok() + } + + #[cfg(feature = "counter")] + pub fn get_counter( + &self, + id: I, + ) -> crate::handler::counter::CounterHandler { + self.try_get_counter(id) + .expect("The container does not exist in the document. Use `try_get_counter` or `get_container` to check for existence.") + } + + #[must_use] + pub fn has_container(&self, id: &ContainerID) -> bool { + if id.is_root() && !id.is_mergeable() { + return true; + } + + let exist = self.state.lock().does_container_exist(id); + exist + } + + /// Undo the operations between the given id_span. It can be used even in a collaborative environment. + /// + /// This is an internal API. You should NOT use it directly. + /// + /// # Internal + /// + /// This method will use the diff calculator to calculate the diff required to time travel + /// from the end of id_span to the beginning of the id_span. Then it will convert the diff to + /// operations and apply them to the OpLog with a dep on the last id of the given id_span. + /// + /// This implementation is kinda slow, but it's simple and maintainable. We can optimize it + /// further when it's needed. The time complexity is O(n + m), n is the ops in the id_span, m is the + /// distance from id_span to the current latest version. + #[instrument(level = "info", skip_all)] + pub fn undo_internal( + &self, + id_span: IdSpan, + container_remap: &mut FxHashMap, + post_transform_base: Option<&DiffBatch>, + before_diff: &mut dyn FnMut(&DiffBatch), + ) -> LoroResult> { + if !self.can_edit() { + return Err(LoroError::EditWhenDetached); + } + + let (options, txn) = self.implicit_commit_then_stop(); + if !self.oplog().lock().vv().includes_id(id_span.id_last()) { + self.renew_txn_if_auto_commit(options); + return Err(LoroError::UndoInvalidIdSpan(id_span.id_last())); + } + + let (was_recording, latest_frontiers) = { + let mut state = self.state.lock(); + let was_recording = state.is_recording(); + state.stop_and_clear_recording(); + (was_recording, state.frontiers.clone()) + }; + + let spans = self.oplog.lock().split_span_based_on_deps(id_span); + let diff = crate::undo::undo( + spans, + match post_transform_base { + Some(d) => Either::Right(d), + None => Either::Left(&latest_frontiers), + }, + |from, to| { + self._checkout_without_emitting(from, false, false).unwrap(); + self.state.lock().start_recording(); + self._checkout_without_emitting(to, false, false).unwrap(); + let mut state = self.state.lock(); + let e = state.take_events(); + state.stop_and_clear_recording(); + DiffBatch::new(e) + }, + before_diff, + ); + + // println!("\nundo_internal: diff: {:?}", diff); + // println!("container remap: {:?}", container_remap); + + self._checkout_without_emitting(&latest_frontiers, false, false)?; + self.set_detached(false); + if was_recording { + self.state.lock().start_recording(); + } + drop(txn); + self.start_auto_commit(); + // Try applying the diff, but ignore the error if it happens. + // MovableList's undo behavior is too tricky to handle in a collaborative env + // so in edge cases this may be an Error + if let Err(e) = self._apply_diff(diff, container_remap, true) { + warn!("Undo Failed {:?}", e); + } + + if let Some(options) = options { + self.set_next_commit_options(options); + } + Ok(CommitWhenDrop { + doc: self, + default_options: CommitOptions::new().origin("undo"), + }) + } + + /// Generate a series of local operations that can revert the current doc to the target + /// version. + /// + /// Internally, it will calculate the diff between the current state and the target state, + /// and apply the diff to the current state. + pub fn revert_to(&self, target: &Frontiers) -> LoroResult<()> { + // TODO: test when the doc is readonly + // TODO: test when the doc is detached but enabled editing + let f = self.state_frontiers(); + let diff = self.diff(&f, target)?; + self._apply_diff(diff, &mut Default::default(), false) + } + + /// Calculate the diff between two versions so that apply diff on a will make the state same as b. + /// + /// NOTE: This method will make the doc enter the **detached mode**. + // FIXME: This method needs testing (no event should be emitted during processing this) + pub fn diff(&self, a: &Frontiers, b: &Frontiers) -> LoroResult { + { + // Check whether a and b are valid before checkout so this returns a normal error + // instead of panicking on shallow docs. + let oplog = self.oplog.lock(); + let validate_frontiers = |frontiers: &Frontiers| -> LoroResult<()> { + for id in frontiers.iter() { + if !oplog.dag.contains(id) { + return Err(LoroError::FrontiersNotFound(id)); + } + } + + if oplog.dag.is_before_shallow_root(frontiers) { + return Err(LoroError::SwitchToVersionBeforeShallowRoot); + } + + Ok(()) + }; + + validate_frontiers(a)?; + validate_frontiers(b)?; + } + + let (options, txn) = self.implicit_commit_then_stop(); + let was_detached = self.is_detached(); + let old_frontiers = self.state_frontiers(); + let was_recording = { + let mut state = self.state.lock(); + let is_recording = state.is_recording(); + state.stop_and_clear_recording(); + is_recording + }; + let result = (|| { + self._checkout_without_emitting(a, true, false)?; + self.state.lock().start_recording(); + self._checkout_without_emitting(b, true, false)?; + let mut state = self.state.lock(); + let e = state.take_events(); + state.stop_and_clear_recording(); + Ok::<_, LoroError>(e) + })(); + + // Always restore state regardless of whether diff calculation succeeded + self._checkout_without_emitting(&old_frontiers, false, false) + .unwrap(); + drop(txn); + if !was_detached { + self.set_detached(false); + self.renew_txn_if_auto_commit(options); + } + if was_recording { + self.state.lock().start_recording(); + } + result.map(DiffBatch::new) + } + + /// Apply a diff to the current state. + #[inline(always)] + pub fn apply_diff(&self, diff: DiffBatch) -> LoroResult<()> { + self._apply_diff(diff, &mut Default::default(), true) + } + + /// Apply a diff to the current state. + /// + /// This method will not recreate containers with the same [ContainerID]s. + /// While this can be convenient in certain cases, it can break several internal invariants: + /// + /// 1. Each container should appear only once in the document. Allowing containers with the same ID + /// would result in multiple instances of the same container in the document. + /// 2. Unreachable containers should be removable from the state when necessary. + /// + /// However, the diff may contain operations that depend on container IDs. + /// Therefore, users need to provide a `container_remap` to record and retrieve the container ID remapping. + pub(crate) fn _apply_diff( + &self, + diff: DiffBatch, + container_remap: &mut FxHashMap, + skip_unreachable: bool, + ) -> LoroResult<()> { + if !self.can_edit() { + return Err(LoroError::EditWhenDetached); + } + + let mut ans: LoroResult<()> = Ok(()); + let mut missing_containers: Vec = Vec::new(); + for (mut id, diff) in diff.into_iter() { + let mut remapped = false; + while let Some(rid) = container_remap.get(&id) { + remapped = true; + id = rid.clone(); + } + + if matches!(&id, ContainerID::Normal { .. }) && self.arena.id_to_idx(&id).is_none() { + // Not in arena does not imply non-existent; consult state/kv and register lazily + let exists = self.state.lock().does_container_exist(&id); + if !exists { + missing_containers.push(id); + continue; + } + // Ensure registration so handlers can be created + self.state.lock().ensure_container(&id); + } + + if skip_unreachable && !remapped && !self.state.lock().get_reachable(&id) { + continue; + } + + let Some(h) = self.get_handler(id.clone()) else { + return Err(LoroError::ContainersNotFound { + containers: Box::new(vec![id]), + }); + }; + if let Err(e) = h.apply_diff(diff, container_remap) { + ans = Err(e); + } + } + + if !missing_containers.is_empty() { + return Err(LoroError::ContainersNotFound { + containers: Box::new(missing_containers), + }); + } + + ans + } + + /// This is for debugging purpose. It will travel the whole oplog + #[inline] + pub fn diagnose_size(&self) { + self.oplog().lock().diagnose_size(); + } + + #[inline] + pub fn oplog_frontiers(&self) -> Frontiers { + self.oplog().lock().frontiers().clone() + } + + #[inline] + pub fn state_frontiers(&self) -> Frontiers { + self.state.lock().frontiers.clone() + } + + /// - Ordering::Less means self is less than target or parallel + /// - Ordering::Equal means versions equal + /// - Ordering::Greater means self's version is greater than target + #[inline] + pub fn cmp_with_frontiers(&self, other: &Frontiers) -> Ordering { + self.oplog().lock().cmp_with_frontiers(other) + } + + /// Compare two [Frontiers] causally. + /// + /// If one of the [Frontiers] are not included, it will return [FrontiersNotIncluded]. + #[inline] + pub fn cmp_frontiers( + &self, + a: &Frontiers, + b: &Frontiers, + ) -> Result, FrontiersNotIncluded> { + self.oplog().lock().cmp_frontiers(a, b) + } + + pub fn subscribe_root(&self, callback: Subscriber) -> Subscription { + let mut state = self.state.lock(); + if !state.is_recording() { + state.start_recording(); + } + + self.observer.subscribe_root(callback) + } + + pub fn subscribe(&self, container_id: &ContainerID, callback: Subscriber) -> Subscription { + let mut state = self.state.lock(); + if !state.is_recording() { + state.start_recording(); + } + + self.observer.subscribe(container_id, callback) + } + + pub fn subscribe_local_update(&self, callback: LocalUpdateCallback) -> Subscription { + let (sub, activate) = self.local_update_subs.inner().insert((), callback); + activate(); + sub + } + + // PERF: opt + #[tracing::instrument(skip_all)] + pub fn import_batch(&self, bytes: &[Vec]) -> LoroResult { + if bytes.is_empty() { + return Ok(ImportStatus::default()); + } + + if bytes.len() == 1 { + return self.import(&bytes[0]); + } + + let mut success = VersionRange::default(); + let mut meta_arr = bytes + .iter() + .map(|b| Ok((LoroDoc::decode_import_blob_meta(b, false)?, b))) + .collect::)>>>()?; + meta_arr.sort_by(|a, b| { + a.0.mode + .cmp(&b.0.mode) + .then(b.0.change_num.cmp(&a.0.change_num)) + }); + + let (options, txn) = self.implicit_commit_then_stop(); + // Why we should keep locking `txn` here + // + // In a multi-threaded environment, `import_batch` used to drop the txn lock + // (via `commit_then_stop` + `drop(txn)`) and call `detach()`/`checkout_to_latest()` + // around the batch import. That created a race where another thread could + // start or renew the auto-commit txn and perform local edits while we were + // importing and temporarily detached. Those interleaved local edits could + // violate invariants between `OpLog` and `DocState` (e.g., state being + // updated when we expect it not to, missed events, or inconsistent + // frontiers), as exposed by the loom test `local_edits_during_batch_import`. + // + // The fix is to hold the txn mutex for the entire critical section: + // - Stop the current txn and keep the mutex guard. + // - Force-detach with `set_detached(true)` (avoids `detach()` side effects), + // then run each `_import_with(...)` while detached so imports only touch + // the `OpLog`. + // - After importing, reattach by checking out to latest and renew the txn + // using `_checkout_to_latest_with_guard`, which keeps the mutex held while + // (re)starting the auto-commit txn. + // + // Holding the lock ensures no concurrent thread can create/renew a txn and + // do local edits in the middle of the batch import, making the whole + // operation atomic with respect to local edits. + let is_detached = self.is_detached(); + self.set_detached(true); + self.oplog.lock().batch_importing = true; + let mut err = None; + for (_meta, data) in meta_arr { + match self._import_with(data, Default::default()) { + Ok(s) => { + for (peer, (start, end)) in s.success.iter() { + match success.0.entry(*peer) { + Entry::Occupied(mut e) => { + e.get_mut().1 = *end.max(&e.get().1); + } + Entry::Vacant(e) => { + e.insert((*start, *end)); + } + } + } + } + Err(e) => { + err = Some(e); + } + } + } + + let mut oplog = self.oplog.lock(); + oplog.batch_importing = false; + let pending = oplog.pending_changes.version_range(); + drop(oplog); + if !is_detached { + self._checkout_to_latest_with_guard(txn); + } else { + drop(txn); + } + + self.renew_txn_if_auto_commit(options); + if let Some(err) = err { + return Err(err); + } + + Ok(ImportStatus { + success, + pending: if pending.is_empty() { + None + } else { + Some(pending) + }, + }) + } + + /// Get shallow value of the document. + #[inline] + pub fn get_value(&self) -> LoroValue { + self.state.lock().get_value() + } + + /// Get deep value of the document. + #[inline] + pub fn get_deep_value(&self) -> LoroValue { + self.state.lock().get_deep_value() + } + + /// Get deep value of the document with container id + #[inline] + pub fn get_deep_value_with_id(&self) -> LoroValue { + self.state.lock().get_deep_value_with_id() + } + + pub fn checkout_to_latest(&self) { + let (options, _guard) = self.implicit_commit_then_stop(); + if !self.is_detached() { + drop(_guard); + self.renew_txn_if_auto_commit(options); + return; + } + + self._checkout_to_latest_without_commit(true) + .expect("checkout to oplog frontiers should succeed"); + self.emit_events(); + drop(_guard); + self.renew_txn_if_auto_commit(options); + } + + fn _checkout_to_latest_with_guard(&self, guard: LoroMutexGuard>) { + if !self.is_detached() { + self._renew_txn_if_auto_commit_with_guard(None, guard); + return; + } + + self._checkout_to_latest_without_commit(true) + .expect("checkout to oplog frontiers should succeed"); + self._renew_txn_if_auto_commit_with_guard(None, guard); + } + + /// NOTE: The caller of this method should ensure the txn is locked and set to None + pub(crate) fn _checkout_to_latest_without_commit( + &self, + to_commit_then_renew: bool, + ) -> LoroResult<()> { + self._checkout_to_latest_without_commit_with_event( + to_commit_then_renew, + "checkout".into(), + EventTriggerKind::Checkout, + ) + } + + pub(crate) fn _checkout_to_latest_without_commit_as_import( + &self, + to_commit_then_renew: bool, + origin: InternalString, + ) -> LoroResult<()> { + self._checkout_to_latest_without_commit_with_event( + to_commit_then_renew, + origin, + EventTriggerKind::Import, + ) + } + + fn _checkout_to_latest_without_commit_with_event( + &self, + to_commit_then_renew: bool, + origin: InternalString, + triggered_by: EventTriggerKind, + ) -> LoroResult<()> { + tracing::info_span!("CheckoutToLatest", peer = self.peer_id()).in_scope(|| { + let f = self.oplog_frontiers(); + let this = &self; + let frontiers = &f; + this._checkout_without_emitting_with_event( + frontiers, + false, + to_commit_then_renew, + origin, + triggered_by, + )?; + // We don't need to shrink frontiers because oplog's frontiers are already shrinked. + this.emit_events(); + if this.config.detached_editing() { + this.renew_peer_id(); + } + + self.set_detached(false); + Ok(()) + }) + } + + /// Checkout [DocState] to a specific version. + /// + /// This will make the current [DocState] detached from the latest version of [OpLog]. + /// Any further import will not be reflected on the [DocState], until user call [LoroDoc::attach()] + pub fn checkout(&self, frontiers: &Frontiers) -> LoroResult<()> { + let was_detached = self.is_detached(); + let (options, guard) = self.implicit_commit_then_stop(); + let result = self._checkout_without_emitting(frontiers, true, true); + if result.is_ok() { + self.emit_events(); + } + drop(guard); + if self.config.detached_editing() { + if result.is_ok() { + self.renew_peer_id(); + } + self.renew_txn_if_auto_commit(options); + } else if result.is_err() { + if !was_detached { + self.renew_txn_if_auto_commit(options); + } + } else if !self.is_detached() { + self.renew_txn_if_auto_commit(options); + } + + result + } + + /// NOTE: The caller of this method should ensure the txn is locked and set to None + #[instrument(level = "info", skip(self))] + pub(crate) fn _checkout_without_emitting( + &self, + frontiers: &Frontiers, + to_shrink_frontiers: bool, + to_commit_then_renew: bool, + ) -> Result<(), LoroError> { + self._checkout_without_emitting_with_event( + frontiers, + to_shrink_frontiers, + to_commit_then_renew, + "checkout".into(), + EventTriggerKind::Checkout, + ) + } + + fn _checkout_without_emitting_with_event( + &self, + frontiers: &Frontiers, + to_shrink_frontiers: bool, + _to_commit_then_renew: bool, + origin: InternalString, + triggered_by: EventTriggerKind, + ) -> Result<(), LoroError> { + if !self.txn.is_locked() { + return Err(LoroError::TransactionError( + "checkout requires the transaction mutex to be held" + .to_string() + .into_boxed_str(), + )); + } + let from_frontiers = self.state_frontiers(); + loro_common::info!( + "checkout from={:?} to={:?} cur_vv={:?}", + from_frontiers, + frontiers, + self.oplog_vv() + ); + + if &from_frontiers == frontiers { + self.set_detached(frontiers != &self.oplog_frontiers()); + return Ok(()); + } + + let oplog = self.oplog.lock(); + if oplog.dag.is_before_shallow_root(frontiers) { + return Err(LoroError::SwitchToVersionBeforeShallowRoot); + } + + let frontiers = if to_shrink_frontiers { + shrink_frontiers(frontiers, &oplog.dag).map_err(LoroError::FrontiersNotFound)? + } else { + frontiers.clone() + }; + + if from_frontiers == frontiers { + return Ok(()); + } + + let mut state = self.state.lock(); + let mut calc = self.diff_calculator.lock(); + for i in frontiers.iter() { + if !oplog.dag.contains(i) { + return Err(LoroError::FrontiersNotFound(i)); + } + } + + let before = oplog.dag.frontiers_to_vv(&state.frontiers).ok_or_else(|| { + LoroError::NotFoundError( + format!( + "Cannot find the current state version {:?}", + state.frontiers + ) + .into_boxed_str(), + ) + })?; + let Some(after) = &oplog.dag.frontiers_to_vv(&frontiers) else { + return Err(LoroError::NotFoundError( + format!("Cannot find the specified version {:?}", frontiers).into_boxed_str(), + )); + }; + + self.set_detached(true); + let (diff, diff_mode) = + calc.calc_diff_internal(&oplog, &before, &state.frontiers, after, &frontiers, None); + state.apply_diff( + InternalDocDiff { + origin, + diff: Cow::Owned(diff), + by: triggered_by, + new_version: Cow::Owned(frontiers.clone()), + }, + diff_mode, + )?; + + Ok(()) + } + + #[inline] + pub fn vv_to_frontiers(&self, vv: &VersionVector) -> Frontiers { + self.oplog.lock().dag.vv_to_frontiers(vv) + } + + #[inline] + pub fn frontiers_to_vv(&self, frontiers: &Frontiers) -> Option { + self.oplog.lock().dag.frontiers_to_vv(frontiers) + } + + /// Import ops from other doc. + /// + /// After `a.merge(b)` and `b.merge(a)`, `a` and `b` will have the same content if they are in attached mode. + pub fn merge(&self, other: &Self) -> LoroResult { + let updates = other.export(ExportMode::updates(&self.oplog_vv())).unwrap(); + self.import(&updates) + } + + pub(crate) fn arena(&self) -> &SharedArena { + &self.arena + } + + #[inline] + pub fn len_ops(&self) -> usize { + if self.oplog.can_lock_in_this_thread() { + return self.oplog.lock().visible_op_count_exact(); + } + + self.visible_op_count.load(Acquire) + } + + #[inline] + pub fn len_changes(&self) -> usize { + let oplog = self.oplog.lock(); + oplog.len_changes() + } + + pub fn config(&self) -> &Configure { + &self.config + } + + /// This method compare the consistency between the current doc state + /// and the state calculated by diff calculator from beginning. + /// + /// Panic when it's not consistent + pub fn check_state_diff_calc_consistency_slow(&self) { + // #[cfg(any(test, debug_assertions, feature = "test_utils"))] + { + static IS_CHECKING: std::sync::atomic::AtomicBool = + std::sync::atomic::AtomicBool::new(false); + if IS_CHECKING.load(std::sync::atomic::Ordering::Acquire) { + return; + } + + IS_CHECKING.store(true, std::sync::atomic::Ordering::Release); + let peer_id = self.peer_id(); + let s = info_span!("CheckStateDiffCalcConsistencySlow", ?peer_id); + let _g = s.enter(); + let options = self.implicit_commit_then_stop().0; + self.oplog.lock().check_dag_correctness(); + if self.is_shallow() { + // For shallow documents, we cannot replay from the beginning as the history is not complete. + // + // Instead, we: + // 1. Export the initial state from the GC snapshot. + // 2. Create a new document and import the initial snapshot. + // 3. Export updates from the shallow start version vector to the current version. + // 4. Import these updates into the new document. + // 5. Compare the states of the new document and the current document. + + // Step 1: Export the initial state from the GC snapshot. + let initial_snapshot = self + .export(ExportMode::state_only(Some( + &self.shallow_since_frontiers(), + ))) + .unwrap(); + + // Step 2: Create a new document and import the initial snapshot. + let doc = LoroDoc::new(); + doc.import(&initial_snapshot).unwrap(); + self.checkout(&self.shallow_since_frontiers()).unwrap(); + assert_eq!(self.get_deep_value(), doc.get_deep_value()); + + // Step 3: Export updates since the shallow start version vector to the current version. + let updates = self.export(ExportMode::all_updates()).unwrap(); + + // Step 4: Import these updates into the new document. + doc.import(&updates).unwrap(); + self.checkout_to_latest(); + + // Step 5: Checkout to the current state's frontiers and compare the states. + // doc.checkout(&self.state_frontiers()).unwrap(); + assert_eq!(doc.get_deep_value(), self.get_deep_value()); + let mut calculated_state = doc.app_state().lock(); + let mut current_state = self.app_state().lock(); + current_state.check_is_the_same(&mut calculated_state); + } else { + let f = self.state_frontiers(); + let vv = self.oplog().lock().dag.frontiers_to_vv(&f).unwrap(); + let bytes = self.export(ExportMode::updates_till(&vv)).unwrap(); + let doc = Self::new(); + doc.import(&bytes).unwrap(); + let mut calculated_state = doc.app_state().lock(); + let mut current_state = self.app_state().lock(); + current_state.check_is_the_same(&mut calculated_state); + } + + self.renew_txn_if_auto_commit(options); + IS_CHECKING.store(false, std::sync::atomic::Ordering::Release); + } + } + + pub fn query_pos(&self, pos: &Cursor) -> Result { + self.query_pos_internal(pos, true) + } + + /// Get position in a seq container + pub(crate) fn query_pos_internal( + &self, + pos: &Cursor, + ret_event_index: bool, + ) -> Result { + if !self.has_container(&pos.container) { + return Err(CannotFindRelativePosition::IdNotFound); + } + + let mut state = self.state.lock(); + if let Some(ans) = state.get_relative_position(pos, ret_event_index) { + Ok(PosQueryResult { + update: None, + current: AbsolutePosition { + pos: ans, + side: pos.side, + }, + }) + } else { + // We need to trace back to the version where the relative position is valid. + // The optimal way to find that version is to have succ info like Automerge. + // + // But we don't have that info now, so an alternative way is to trace back + // to version with frontiers of `[pos.id]`. But this may be very slow even if + // the target is just deleted a few versions ago. + // + // What we need is to trace back to the latest version that deletes the target + // id. + + // commit the txn to make sure we can query the history correctly, preserving options + drop(state); + let result = self.with_barrier(|| { + let oplog = self.oplog().lock(); + // TODO: assert pos.id is not unknown + if let Some(id) = pos.id { + // Ensure the container is registered if it exists lazily + if oplog.arena.id_to_idx(&pos.container).is_none() { + let mut s = self.state.lock(); + if !s.does_container_exist(&pos.container) { + return Err(CannotFindRelativePosition::ContainerDeleted); + } + s.ensure_container(&pos.container); + drop(s); + } + let idx = oplog.arena.id_to_idx(&pos.container).unwrap(); + // We know where the target id is when we trace back to the delete_op_id. + let Some(delete_op_id) = find_last_delete_op(&oplog, id, idx) else { + if oplog.shallow_since_vv().includes_id(id) { + return Err(CannotFindRelativePosition::HistoryCleared); + } + + tracing::error!("Cannot find id {}", id); + return Err(CannotFindRelativePosition::IdNotFound); + }; + // Should use persist mode so that it will force all the diff calculators to use the `checkout` mode + let mut diff_calc = DiffCalculator::new(true); + let before_frontiers: Frontiers = oplog.dag.find_deps_of_id(delete_op_id); + let before = &oplog.dag.frontiers_to_vv(&before_frontiers).unwrap(); + // TODO: PERF: it doesn't need to calc the effects here + diff_calc.calc_diff_internal( + &oplog, + before, + &before_frontiers, + oplog.vv(), + oplog.frontiers(), + Some(&|target| idx == target), + ); + // TODO: remove depth info + let depth = self.arena.get_depth(idx); + let (_, diff_calc) = &mut diff_calc.get_or_create_calc(idx, depth); + match diff_calc { + crate::diff_calc::ContainerDiffCalculator::Richtext(text) => { + let c = text.get_id_latest_pos(id).unwrap(); + let new_pos = c.pos; + let handler = self.get_text(&pos.container); + let current_pos = handler.convert_entity_index_to_event_index(new_pos); + Ok(PosQueryResult { + update: handler.get_cursor(current_pos, c.side), + current: AbsolutePosition { + pos: current_pos, + side: c.side, + }, + }) + } + crate::diff_calc::ContainerDiffCalculator::List(list) => { + let c = list.get_id_latest_pos(id).unwrap(); + let new_pos = c.pos; + let handler = self.get_list(&pos.container); + Ok(PosQueryResult { + update: handler.get_cursor(new_pos, c.side), + current: AbsolutePosition { + pos: new_pos, + side: c.side, + }, + }) + } + crate::diff_calc::ContainerDiffCalculator::MovableList(list) => { + let c = list.get_id_latest_pos(id).unwrap(); + let new_pos = c.pos; + let handler = self.get_movable_list(&pos.container); + let new_pos = handler.op_pos_to_user_pos(new_pos); + Ok(PosQueryResult { + update: handler.get_cursor(new_pos, c.side), + current: AbsolutePosition { + pos: new_pos, + side: c.side, + }, + }) + } + crate::diff_calc::ContainerDiffCalculator::Tree(_) => unreachable!(), + crate::diff_calc::ContainerDiffCalculator::Map(_) => unreachable!(), + #[cfg(feature = "counter")] + crate::diff_calc::ContainerDiffCalculator::Counter(_) => unreachable!(), + crate::diff_calc::ContainerDiffCalculator::Unknown(_) => unreachable!(), + } + } else { + match pos.container.container_type() { + ContainerType::Text => { + let text = self.get_text(&pos.container); + Ok(PosQueryResult { + update: Some(Cursor { + id: None, + container: text.id(), + side: pos.side, + origin_pos: text.len_unicode(), + }), + current: AbsolutePosition { + pos: text.len_event(), + side: pos.side, + }, + }) + } + ContainerType::List => { + let list = self.get_list(&pos.container); + Ok(PosQueryResult { + update: Some(Cursor { + id: None, + container: list.id(), + side: pos.side, + origin_pos: list.len(), + }), + current: AbsolutePosition { + pos: list.len(), + side: pos.side, + }, + }) + } + ContainerType::MovableList => { + let list = self.get_movable_list(&pos.container); + Ok(PosQueryResult { + update: Some(Cursor { + id: None, + container: list.id(), + side: pos.side, + origin_pos: list.len(), + }), + current: AbsolutePosition { + pos: list.len(), + side: pos.side, + }, + }) + } + ContainerType::Map | ContainerType::Tree | ContainerType::Unknown(_) => { + unreachable!() + } + #[cfg(feature = "counter")] + ContainerType::Counter => unreachable!(), + } + } + }); + result + } + } + + /// Free the history cache that is used for making checkout faster. + /// + /// If you use checkout that switching to an old/concurrent version, the history cache will be built. + /// You can free it by calling this method. + pub fn free_history_cache(&self) { + self.oplog.lock().free_history_cache(); + } + + /// Free the cached diff calculator that is used for checkout. + pub fn free_diff_calculator(&self) { + *self.diff_calculator.lock() = DiffCalculator::new(true); + } + + /// If you use checkout that switching to an old/concurrent version, the history cache will be built. + /// You can free it by calling `free_history_cache`. + pub fn has_history_cache(&self) -> bool { + self.oplog.lock().has_history_cache() + } + + /// Encoded all ops and history cache to bytes and store them in the kv store. + /// + /// The parsed ops will be dropped + #[inline] + pub fn compact_change_store(&self) { + self.with_barrier(|| { + self.oplog.lock().compact_change_store(); + }); + } + + /// Analyze the container info of the doc + /// + /// This is used for development and debugging + #[inline] + pub fn analyze(&self) -> DocAnalysis { + DocAnalysis::analyze(self) + } + + /// Get the path from the root to the container + pub fn get_path_to_container(&self, id: &ContainerID) -> Option> { + let mut state = self.state.lock(); + if state.arena.id_to_idx(id).is_none() { + if id.is_mergeable() { + // Mergeable children can be logically active via the parent map marker + // before they have their own encoded state. Register only the arena edge; do not + // create container state or change `has_container` semantics. + state.arena.register_container(id); + } else if !state.does_container_exist(id) { + return None; + } else { + state.ensure_container(id); + } + } + let idx = state.arena.id_to_idx(id).unwrap(); + state.get_path(idx) + } + + #[instrument(skip(self))] + pub fn export(&self, mode: ExportMode) -> Result, LoroEncodeError> { + self.with_barrier(|| { + let ans = match mode { + ExportMode::Snapshot => export_fast_snapshot(self), + ExportMode::Updates { from } => export_fast_updates(self, &from), + ExportMode::UpdatesInRange { spans } => { + export_fast_updates_in_range(&self.oplog.lock(), spans.as_ref()) + } + ExportMode::ShallowSnapshot(f) => export_shallow_snapshot(self, &f)?, + ExportMode::StateOnly(f) => match f { + Some(f) => export_state_only_snapshot(self, &f)?, + None => export_state_only_snapshot(self, &self.oplog_frontiers())?, + }, + ExportMode::SnapshotAt { version } => export_snapshot_at(self, &version)?, + }; + Ok(ans) + }) + } + + /// The doc only contains the history since the shallow history start version vector. + /// + /// This is empty if the doc is not shallow. + /// + /// The ops included by the shallow history start version vector are not in the doc. + pub fn shallow_since_vv(&self) -> ImVersionVector { + self.oplog().lock().shallow_since_vv().clone() + } + + pub fn shallow_since_frontiers(&self) -> Frontiers { + self.oplog().lock().shallow_since_frontiers().clone() + } + + /// Check if the doc contains the full history. + pub fn is_shallow(&self) -> bool { + !self.oplog().lock().shallow_since_vv().is_empty() + } + + /// Get the number of operations in the pending transaction. + /// + /// The pending transaction is the one that is not committed yet. It will be committed + /// after calling `doc.commit()`, `doc.export(mode)` or `doc.checkout(version)`. + pub fn get_pending_txn_len(&self) -> usize { + if let Some(txn) = self.txn.lock().as_ref() { + txn.len() + } else { + 0 + } + } + + #[inline] + pub fn find_id_spans_between(&self, from: &Frontiers, to: &Frontiers) -> VersionVectorDiff { + self.oplog().lock().dag.find_path(from, to) + } + + /// Subscribe to the first commit from a peer. Operations performed on the `LoroDoc` within this callback + /// will be merged into the current commit. + /// + /// This is useful for managing the relationship between `PeerID` and user information. + /// For example, you could store user names in a `LoroMap` using `PeerID` as the key and the `UserID` as the value. + pub fn subscribe_first_commit_from_peer( + &self, + callback: FirstCommitFromPeerCallback, + ) -> Subscription { + let (s, enable) = self + .first_commit_from_peer_subs + .inner() + .insert((), callback); + enable(); + s + } + + /// Subscribe to the pre-commit event. + /// + /// The callback will be called when the changes are committed but not yet applied to the OpLog. + /// You can modify the commit message and timestamp in the callback by [`ChangeModifier`]. + pub fn subscribe_pre_commit(&self, callback: PreCommitCallback) -> Subscription { + let (s, enable) = self.pre_commit_subs.inner().insert((), callback); + enable(); + s + } +} + +fn pending_root_containers_to_materialize(oplog: &OpLog, changes: &[Change]) -> Vec { + let mut roots = FxHashSet::default(); + for change in changes { + if change.ctr_end() <= oplog.vv().get(&change.id.peer).copied().unwrap_or(0) { + continue; + } + + if oplog.dag.is_before_shallow_root(&change.deps) + || oplog + .dag + .get_change_lamport_from_deps(&change.deps) + .is_some() + { + continue; + } + + for op in change.ops.iter() { + let id = oplog + .arena + .get_container_id(op.container) + .expect("decoded op container should be registered"); + // Mergeable containers share the `ContainerID::Root` namespace but are logical + // *children*: their existence is governed by their parent map's marker, and their + // parent can be any (possibly not-yet-imported) map. Eagerly materializing one + // while its causal dependencies are still pending has no valid parent edge to + // resolve its depth against, which used to panic in `ContainerWrapper::new`. + // They get materialized correctly through the normal diff path once the creating + // change applies, so skip them here (mirrors the `!is_mergeable()` guard in + // `ensure_root_container`). See the `mergeable_container::pending` regression test. + if id.is_root() && !id.is_mergeable() { + roots.insert(id); + } + } + } + + roots.into_iter().collect() +} + +#[derive(Debug, thiserror::Error)] +pub enum ChangeTravelError { + #[error("Target id not found {0:?}")] + TargetIdNotFound(ID), + #[error("The shallow history of the doc doesn't include the target version")] + TargetVersionNotIncluded, +} + +impl LoroDoc { + pub fn travel_change_ancestors( + &self, + ids: &[ID], + f: &mut dyn FnMut(ChangeMeta) -> ControlFlow<()>, + ) -> Result<(), ChangeTravelError> { + let (options, guard) = self.implicit_commit_then_stop(); + drop(guard); + struct PendingNode(ChangeMeta); + impl PartialEq for PendingNode { + fn eq(&self, other: &Self) -> bool { + self.0.lamport_last() == other.0.lamport_last() && self.0.id.peer == other.0.id.peer + } + } + + impl Eq for PendingNode {} + impl PartialOrd for PendingNode { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } + } + + impl Ord for PendingNode { + fn cmp(&self, other: &Self) -> Ordering { + self.0 + .lamport_last() + .cmp(&other.0.lamport_last()) + .then_with(|| self.0.id.peer.cmp(&other.0.id.peer)) + } + } + + for id in ids { + let op_log = &self.oplog().lock(); + if !op_log.vv().includes_id(*id) { + return Err(ChangeTravelError::TargetIdNotFound(*id)); + } + if op_log.dag.shallow_since_vv().includes_id(*id) { + return Err(ChangeTravelError::TargetVersionNotIncluded); + } + } + + let mut visited = FxHashSet::default(); + let mut pending: BinaryHeap = BinaryHeap::new(); + for id in ids { + pending.push(PendingNode(ChangeMeta::from_change( + &self.oplog().lock().get_change_at(*id).unwrap(), + ))); + } + while let Some(PendingNode(node)) = pending.pop() { + let deps = node.deps.clone(); + if f(node).is_break() { + break; + } + + for dep in deps.iter() { + let Some(dep_node) = self.oplog().lock().get_change_at(dep) else { + continue; + }; + if visited.contains(&dep_node.id) { + continue; + } + + visited.insert(dep_node.id); + pending.push(PendingNode(ChangeMeta::from_change(&dep_node))); + } + } + + let ans = Ok(()); + self.renew_txn_if_auto_commit(options); + ans + } + + pub fn get_changed_containers_in(&self, id: ID, len: usize) -> FxHashSet { + self.with_barrier(|| { + let mut set = FxHashSet::default(); + let len = i64::try_from(len).unwrap_or(i64::MAX); + let start = i64::from(id.counter); + let end = start.saturating_add(len); + if end <= 0 { + return set; + } + + let start = start.max(0).min(i64::from(i32::MAX)); + let end = end.max(0).min(i64::from(i32::MAX)); + if start >= end { + return set; + } + + { + let oplog = self.oplog().lock(); + let span = IdSpan::new(id.peer, start as i32, end as i32); + for op in oplog.iter_ops(span) { + let id = oplog.arena.get_container_id(op.container()).unwrap(); + set.insert(id); + } + } + set + }) + } + + pub fn delete_root_container(&self, cid: ContainerID) { + if !cid.is_root() { + return; + } + + // Do not treat "not in arena" as non-existence; consult state/kv + if !self.has_container(&cid) { + return; + } + + let Some(h) = self.get_handler(cid.clone()) else { + return; + }; + + self.config + .deleted_root_containers + .lock() + .insert(cid.clone()); + if let Err(e) = h.clear() { + self.config.deleted_root_containers.lock().remove(&cid); + eprintln!("Failed to clear handler: {:?}", e); + } + } + + pub fn set_hide_empty_root_containers(&self, hide: bool) { + self.config + .hide_empty_root_containers + .store(hide, std::sync::atomic::Ordering::Relaxed); + } +} + +fn find_last_delete_op(oplog: &OpLog, id: ID, idx: ContainerIdx) -> Option { + // Any delete op that covers `id` must have observed it, so its peer's counter + // at delete time was > id.counter. start_vv (the vv at `id`) is therefore a + // valid lower bound: changes at or before start_vv[peer] predate `id` and can + // be skipped. We scan peer-by-peer rather than using the DAG-ordered + // iter_changes_causally_rev, which is O(total changes). + // + // We choose the matching delete op with the greatest (op_lamport, peer, counter) + // ordering. op_lamport is the Lamport of the specific op within the change + // (change.lamport + op offset), not just the change's starting Lamport, so + // concurrent deletes with equal change Lamports are broken deterministically. + let start_vv = oplog + .dag + .frontiers_to_vv(&id.into()) + .unwrap_or_else(|| oplog.shallow_since_vv().to_vv()); + + // (op_lamport, peer) gives a deterministic total order for concurrent deletes. + // A single peer cannot produce two ops with the same lamport, so peer suffices + // as a tie-breaker. + let mut best: Option<((loro_common::Lamport, loro_common::PeerID), ID)> = None; + + for change in oplog.iter_changes_peer_by_peer(&start_vv, oplog.vv()) { + let peer = change.peer(); + for op in change.ops.iter() { + if op.container != idx { + continue; + } + if let InnerContent::List(InnerListOp::Delete(d)) = &op.content { + if d.id_start.to_span(d.atom_len()).contains(id) { + debug_assert!(op.counter >= change.id().counter); + let op_lamport = + change.lamport + (op.counter - change.id().counter) as loro_common::Lamport; + let key = (op_lamport, peer); + if best.is_none_or(|(bk, _)| key > bk) { + best = Some((key, ID::new(peer, op.counter))); + } + } + } + } + } + + best.map(|(_, op_id)| op_id) +} + +#[derive(Debug)] +pub struct CommitWhenDrop<'a> { + doc: &'a LoroDoc, + default_options: CommitOptions, +} + +impl Drop for CommitWhenDrop<'_> { + fn drop(&mut self) { + { + let mut guard = self.doc.txn.lock(); + if let Some(txn) = guard.as_mut() { + txn.set_default_options(std::mem::take(&mut self.default_options)); + }; + } + + self.doc.commit_then_renew(); + } +} + +/// Options for configuring a commit operation. +#[derive(Debug, Clone)] +pub struct CommitOptions { + /// Origin identifier for the commit event, used to track the source of changes. + /// It doesn't persist. + pub origin: Option, + + /// Whether to immediately start a new transaction after committing. + /// Defaults to true. + pub immediate_renew: bool, + + /// Custom timestamp for the commit in seconds since Unix epoch. + /// If None, the current time will be used. + pub timestamp: Option, + + /// Optional commit message to attach to the changes. It will be persisted. + pub commit_msg: Option>, +} + +impl CommitOptions { + /// Creates a new CommitOptions with default values. + pub fn new() -> Self { + Self { + origin: None, + immediate_renew: true, + timestamp: None, + commit_msg: None, + } + } + + /// Sets the origin identifier for this commit. + pub fn origin(mut self, origin: &str) -> Self { + self.origin = Some(origin.into()); + self + } + + /// Sets whether to immediately start a new transaction after committing. + pub fn immediate_renew(mut self, immediate_renew: bool) -> Self { + self.immediate_renew = immediate_renew; + self + } + + /// Set the timestamp of the commit. + /// + /// The timestamp is the number of **seconds** that have elapsed since 00:00:00 UTC on January 1, 1970. + pub fn timestamp(mut self, timestamp: Timestamp) -> Self { + self.timestamp = Some(timestamp); + self + } + + /// Sets a commit message to be attached to the changes. + pub fn commit_msg(mut self, commit_msg: &str) -> Self { + self.commit_msg = Some(commit_msg.into()); + self + } + + /// Sets the origin identifier for this commit. + pub fn set_origin(&mut self, origin: Option<&str>) { + self.origin = origin.map(|x| x.into()) + } + + /// Sets the timestamp for this commit. + pub fn set_timestamp(&mut self, timestamp: Option) { + self.timestamp = timestamp; + } +} + +impl Default for CommitOptions { + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +mod test { + use std::{ + panic::AssertUnwindSafe, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, + }; + + use crate::{ + cursor::PosType, + encoding::json_schema::json::{JsonOpContent, JsonSchema, ListOp}, + encoding::{fast_snapshot::EMPTY_MARK, EncodeMode}, + loro::ExportMode, + version::{Frontiers, VersionVector}, + LoroDoc, ToJson, TreeParentId, + }; + use bytes::{BufMut, Bytes}; + use loro_common::ID; + use loro_kv_store::{mem_store::MemKvConfig, MemKvStore}; + + const XXH_SEED: u32 = u32::from_le_bytes(*b"LORO"); + + fn encode_import_blob(mode: EncodeMode, body: &[u8]) -> Vec { + let mut ans = Vec::new(); + ans.extend_from_slice(b"loro"); + ans.extend_from_slice(&[0; 16]); + ans.extend_from_slice(&mode.to_bytes()); + ans.extend_from_slice(body); + let checksum = xxhash_rust::xxh32::xxh32(&ans[20..], XXH_SEED); + ans[16..20].copy_from_slice(&checksum.to_le_bytes()); + ans + } + + fn encode_fast_snapshot_import(oplog_bytes: &[u8]) -> Vec { + let mut body = Vec::new(); + body.put_u32_le(oplog_bytes.len() as u32); + body.extend_from_slice(oplog_bytes); + body.put_u32_le(EMPTY_MARK.len() as u32); + body.extend_from_slice(EMPTY_MARK); + body.put_u32_le(0); + encode_import_blob(EncodeMode::FastSnapshot, &body) + } + + fn sstable_with_huge_meta_block_count() -> Vec { + let mut bytes = Vec::new(); + bytes.extend_from_slice(b"LORO"); + bytes.push(0); + bytes.put_u32_le(10_000_000); + bytes.put_u32_le(xxhash_rust::xxh32::xxh32(&[], XXH_SEED)); + bytes.put_u32_le(5); + bytes + } + + fn snapshot_oplog_with_malformed_block() -> Vec { + let peer = 1; + let id = ID::new(peer, 0); + let vv = VersionVector::from_iter([(peer, 1)]); + let frontiers = Frontiers::from_id(id); + let mut store = MemKvStore::new(MemKvConfig::default()); + store.set(b"vv", vv.encode().into()); + store.set(b"fr", frontiers.encode().into()); + store.set(&id.to_bytes(), Bytes::from_static(&[0])); + store.export_all().to_vec() + } + + fn make_json_import_stress_doc(peer: u64) -> LoroDoc { + let doc = LoroDoc::new_auto_commit(); + doc.set_peer_id(peer).unwrap(); + + let text = doc.get_text("text"); + let mut text_pos = 0; + for i in 0..32 { + let chunk = format!("segment-{i}-abcdefghijklmnopqrstuvwxyz;"); + text.insert_unicode(text_pos, &chunk).unwrap(); + text_pos += chunk.chars().count(); + } + + let list = doc.get_list("list"); + for i in 0..32 { + list.insert(i, format!("item-{i}")).unwrap(); + } + + let map = doc.get_map("map"); + for i in 0..32 { + let key = format!("key-{i}"); + map.insert(&key, format!("value-{i}")).unwrap(); + } + + let tree = doc.get_tree("tree"); + let mut parent = TreeParentId::Root; + for i in 0..16 { + let node = tree.create(parent).unwrap(); + let meta = tree.get_meta(node).unwrap(); + meta.insert("name", format!("node-{i}")).unwrap(); + meta.insert("payload", format!("payload-{i}-{}", "x".repeat(16))) + .unwrap(); + parent = TreeParentId::Node(node); + } + + doc + } + + fn make_json_list_update_with_four_ops(peer: u64) -> (LoroDoc, JsonSchema) { + let doc = LoroDoc::new(); + doc.set_peer_id(peer).unwrap(); + let map = doc.get_map("map"); + let list = doc.get_list("list"); + let text = doc.get_text("text"); + + let mut txn = doc.txn().unwrap(); + map.insert_with_txn(&mut txn, "prefix", "map-value".into()) + .unwrap(); + list.insert_with_txn(&mut txn, 0, "seed".into()).unwrap(); + text.insert_with_txn(&mut txn, 0, "text-value", PosType::Unicode) + .unwrap(); + list.insert_with_txn(&mut txn, 1, "tail".into()).unwrap(); + txn.commit().unwrap(); + + let json = doc.export_json_updates(&Default::default(), &doc.oplog_vv(), false); + assert_eq!(json.changes.len(), 1); + assert_eq!(json.changes[0].ops.len(), 4); + (doc, json) + } + + fn move_last_list_insert_far_out_of_bounds(json: &mut JsonSchema) { + let last_change = json.changes.last_mut().unwrap(); + let last_op = last_change.ops.last_mut().unwrap(); + match &mut last_op.content { + JsonOpContent::List(ListOp::Insert { pos, .. }) => { + *pos = 1_000; + } + other => panic!("expected list insert op, got {other:?}"), + } + } + + #[test] + fn test_sync() { + fn is_send_sync(_v: T) {} + let loro = super::LoroDoc::new(); + is_send_sync(loro) + } + + #[test] + fn import_rejects_huge_sstable_meta_block_count_without_panic() { + let bytes = encode_fast_snapshot_import(&sstable_with_huge_meta_block_count()); + + let result = std::panic::catch_unwind(AssertUnwindSafe(|| LoroDoc::new().import(&bytes))); + assert!(result.is_ok(), "malformed import should not panic"); + assert!(result.unwrap().is_err()); + } + + #[test] + fn import_rejects_malformed_change_block_without_panic() { + let bytes = encode_fast_snapshot_import(&snapshot_oplog_with_malformed_block()); + + let result = std::panic::catch_unwind(AssertUnwindSafe(|| LoroDoc::new().import(&bytes))); + assert!(result.is_ok(), "malformed import should not panic"); + assert!(result.unwrap().is_err()); + } + + #[test] + fn failed_import_rolls_back_oplog_and_arena() { + let src = LoroDoc::new(); + src.set_peer_id(1).unwrap(); + let text = src.get_text("text"); + let mut txn = src.txn().unwrap(); + text.insert_with_txn(&mut txn, 0, "hello", PosType::Unicode) + .unwrap(); + txn.commit().unwrap(); + let update = src.export(ExportMode::all_updates()).unwrap(); + + let dst = LoroDoc::new(); + let vv_before_import = dst.oplog_vv(); + let state_before_import = dst.get_deep_value(); + let err = dst + .import_with(&update, "__loro_fail_import_state_apply".into()) + .unwrap_err(); + assert!(err.to_string().contains("state apply failpoint")); + assert_eq!(dst.oplog_vv(), vv_before_import); + assert_eq!(dst.get_deep_value(), state_before_import); + assert!(dst.oplog().lock().is_empty()); + + dst.import(&update).unwrap(); + assert_eq!(dst.get_deep_value(), src.get_deep_value()); + } + + #[test] + fn failed_incremental_import_restores_previous_change_store_block() { + let src = LoroDoc::new(); + src.set_peer_id(1).unwrap(); + let text = src.get_text("text"); + let mut txn = src.txn().unwrap(); + text.insert_with_txn(&mut txn, 0, "a", PosType::Unicode) + .unwrap(); + txn.commit().unwrap(); + let first_update = src.export(ExportMode::all_updates()).unwrap(); + let first_vv = src.oplog_vv(); + + let mut txn = src.txn().unwrap(); + text.insert_with_txn(&mut txn, 1, "b", PosType::Unicode) + .unwrap(); + txn.commit().unwrap(); + let second_update = src.export(ExportMode::updates(&first_vv)).unwrap(); + + let dst = LoroDoc::new(); + dst.import(&first_update).unwrap(); + let vv_before_import = dst.oplog_vv(); + let state_before_import = dst.get_deep_value(); + dst.import_with(&second_update, "__loro_fail_import_state_apply".into()) + .unwrap_err(); + assert_eq!(dst.oplog_vv(), vv_before_import); + assert_eq!(dst.get_deep_value(), state_before_import); + + dst.import(&second_update).unwrap(); + assert_eq!(dst.get_deep_value(), src.get_deep_value()); + } + + #[test] + fn failed_import_json_updates_rolls_back_complex_empty_doc() { + let src = make_json_import_stress_doc(11); + let json = src.export_json_updates(&Default::default(), &src.oplog_vv(), false); + + let dst = LoroDoc::new(); + let vv_before_import = dst.oplog_vv(); + let frontiers_before_import = dst.oplog_frontiers(); + let state_before_import = dst.get_deep_value(); + for _ in 0..3 { + crate::state::fail_next_import_state_apply_for_test(); + let err = dst.import_json_updates(json.clone()).unwrap_err(); + assert!(err.to_string().contains("state apply failpoint")); + assert_eq!(dst.oplog_vv(), vv_before_import); + assert_eq!(dst.oplog_frontiers(), frontiers_before_import); + assert_eq!(dst.get_deep_value(), state_before_import); + assert!(dst.oplog().lock().is_empty()); + } + + dst.import_json_updates(json).unwrap(); + assert_eq!(dst.oplog_vv(), src.oplog_vv()); + assert_eq!(dst.oplog_frontiers(), src.oplog_frontiers()); + assert_eq!(dst.get_deep_value(), src.get_deep_value()); + } + + #[test] + fn failed_incremental_import_json_updates_restores_previous_change_store_block() { + let src = LoroDoc::new_auto_commit(); + src.set_peer_id(12).unwrap(); + let text = src.get_text("text"); + text.insert_unicode(0, "a").unwrap(); + let list = src.get_list("list"); + list.push("seed").unwrap(); + let map = src.get_map("map"); + map.insert("seed", "value").unwrap(); + let tree = src.get_tree("tree"); + let root = tree.create(TreeParentId::Root).unwrap(); + tree.get_meta(root).unwrap().insert("name", "root").unwrap(); + + let first_vv = src.oplog_vv(); + let first_json = src.export_json_updates(&Default::default(), &first_vv, false); + + let mut text_pos = text.len_unicode(); + for i in 0..64 { + let chunk = format!("chunk-{i};"); + text.insert_unicode(text_pos, &chunk).unwrap(); + text_pos += chunk.chars().count(); + } + for i in 0..32 { + list.push(format!("after-{i}")).unwrap(); + let key = format!("after-{i}"); + map.insert(&key, format!("value-{i}")).unwrap(); + } + let child = tree.create(TreeParentId::Node(root)).unwrap(); + tree.get_meta(child) + .unwrap() + .insert("name", "child") + .unwrap(); + + let second_json = src.export_json_updates(&first_vv, &src.oplog_vv(), false); + + let dst = LoroDoc::new(); + dst.import_json_updates(first_json).unwrap(); + let vv_before_import = dst.oplog_vv(); + let frontiers_before_import = dst.oplog_frontiers(); + let state_before_import = dst.get_deep_value(); + + for _ in 0..2 { + crate::state::fail_next_import_state_apply_for_test(); + let err = dst.import_json_updates(second_json.clone()).unwrap_err(); + assert!(err.to_string().contains("state apply failpoint")); + assert_eq!(dst.oplog_vv(), vv_before_import); + assert_eq!(dst.oplog_frontiers(), frontiers_before_import); + assert_eq!(dst.get_deep_value(), state_before_import); + } + + dst.import_json_updates(second_json).unwrap(); + assert_eq!(dst.oplog_vv(), src.oplog_vv()); + assert_eq!(dst.oplog_frontiers(), src.oplog_frontiers()); + assert_eq!(dst.get_deep_value(), src.get_deep_value()); + } + + #[test] + fn malformed_later_import_json_update_rolls_back_after_valid_prefix_enters_oplog() { + let peer = 13; + let (src, good_json) = make_json_list_update_with_four_ops(peer); + let mut bad_json = good_json.clone(); + move_last_list_insert_far_out_of_bounds(&mut bad_json); + + let good_dst = LoroDoc::new(); + good_dst.import_json_updates(good_json.clone()).unwrap(); + assert_eq!(good_dst.get_deep_value(), src.get_deep_value()); + + let last_op_counter = good_json.changes[0].ops.last().unwrap().counter; + let prefix_vv = VersionVector::from_iter([(peer, last_op_counter)]); + let prefix_json = src.export_json_updates(&Default::default(), &prefix_vv, false); + assert_eq!( + prefix_json.changes[0].ops.len(), + good_json.changes[0].ops.len() - 1 + ); + let good_suffix_json = src.export_json_updates(&prefix_vv, &src.oplog_vv(), false); + assert_eq!(good_suffix_json.changes[0].ops.len(), 1); + let mut bad_suffix_json = good_suffix_json.clone(); + move_last_list_insert_far_out_of_bounds(&mut bad_suffix_json); + + let prefix_dst = LoroDoc::new(); + prefix_dst.import_json_updates(prefix_json.clone()).unwrap(); + let vv_before_bad_suffix = prefix_dst.oplog_vv(); + let frontiers_before_bad_suffix = prefix_dst.oplog_frontiers(); + let state_before_bad_suffix = prefix_dst.get_deep_value(); + + let bad_suffix_json = serde_json::to_string(&bad_suffix_json).unwrap(); + let err = prefix_dst + .import_json_updates(&bad_suffix_json) + .unwrap_err(); + assert!( + err.to_string().contains("list diff"), + "expected state list bounds validation, got {err:?}" + ); + assert_eq!(prefix_dst.oplog_vv(), vv_before_bad_suffix); + assert_eq!(prefix_dst.oplog_frontiers(), frontiers_before_bad_suffix); + assert_eq!(prefix_dst.get_deep_value(), state_before_bad_suffix); + + prefix_dst.import_json_updates(good_suffix_json).unwrap(); + assert_eq!(prefix_dst.get_deep_value(), src.get_deep_value()); + assert_eq!(prefix_dst.oplog_vv(), src.oplog_vv()); + + let dst = LoroDoc::new(); + let vv_before_import = dst.oplog_vv(); + let frontiers_before_import = dst.oplog_frontiers(); + let state_before_import = dst.get_deep_value(); + let bad_json = serde_json::to_string(&bad_json).unwrap(); + let err = dst.import_json_updates(&bad_json).unwrap_err(); + assert!( + err.to_string().contains("list diff"), + "expected state list bounds validation, got {err:?}" + ); + assert_eq!(dst.oplog_vv(), vv_before_import); + assert_eq!(dst.oplog_frontiers(), frontiers_before_import); + assert_eq!(dst.get_deep_value(), state_before_import); + assert!(dst.oplog().lock().is_empty()); + } + + #[test] + fn failed_import_restores_pending_changes_that_were_applied_during_import() { + let src = LoroDoc::new(); + src.set_peer_id(14).unwrap(); + let text = src.get_text("text"); + + let mut txn = src.txn().unwrap(); + text.insert_with_txn(&mut txn, 0, "a", PosType::Unicode) + .unwrap(); + txn.commit().unwrap(); + let first_update = src.export(ExportMode::all_updates()).unwrap(); + let first_vv = src.oplog_vv(); + + let mut txn = src.txn().unwrap(); + text.insert_with_txn(&mut txn, 1, "b", PosType::Unicode) + .unwrap(); + txn.commit().unwrap(); + let second_update = src.export(ExportMode::updates(&first_vv)).unwrap(); + + let dst = LoroDoc::new(); + let status = dst.import(&second_update).unwrap(); + assert!(status.success.is_empty()); + assert!(status.pending.is_some()); + let vv_before_dependency = dst.oplog_vv(); + let frontiers_before_dependency = dst.oplog_frontiers(); + let state_before_dependency = dst.get_deep_value(); + + crate::state::fail_next_import_state_apply_for_test(); + let err = dst.import(&first_update).unwrap_err(); + assert!(err.to_string().contains("state apply failpoint")); + assert_eq!(dst.oplog_vv(), vv_before_dependency); + assert_eq!(dst.oplog_frontiers(), frontiers_before_dependency); + assert_eq!(dst.get_deep_value(), state_before_dependency); + + dst.import(&first_update).unwrap(); + assert_eq!(dst.oplog_vv(), src.oplog_vv()); + assert_eq!(dst.oplog_frontiers(), src.oplog_frontiers()); + assert_eq!(dst.get_deep_value(), src.get_deep_value()); + } + + #[test] + fn failed_import_json_updates_does_not_emit_or_leave_events() { + let (src, good_json) = make_json_list_update_with_four_ops(15); + let mut bad_json = good_json.clone(); + move_last_list_insert_far_out_of_bounds(&mut bad_json); + + let dst = LoroDoc::new(); + let event_count = Arc::new(AtomicUsize::new(0)); + let event_count_cloned = event_count.clone(); + let _sub = dst.subscribe_root(Arc::new(move |_| { + event_count_cloned.fetch_add(1, Ordering::SeqCst); + })); + + let bad_json = serde_json::to_string(&bad_json).unwrap(); + let err = dst.import_json_updates(&bad_json).unwrap_err(); + assert!( + err.to_string().contains("list diff"), + "expected state list bounds validation, got {err:?}" + ); + assert_eq!(event_count.load(Ordering::SeqCst), 0); + assert!(dst.drop_pending_events().is_empty()); + assert!(dst.oplog().lock().is_empty()); + + dst.import_json_updates(good_json).unwrap(); + assert_eq!(event_count.load(Ordering::SeqCst), 1); + assert_eq!(dst.get_deep_value(), src.get_deep_value()); + } + + #[test] + fn test_checkout() { + let loro = LoroDoc::new(); + loro.set_peer_id(1).unwrap(); + let text = loro.get_text("text"); + let map = loro.get_map("map"); + let list = loro.get_list("list"); + let mut txn = loro.txn().unwrap(); + for i in 0..10 { + map.insert_with_txn(&mut txn, "key", i.into()).unwrap(); + text.insert_with_txn(&mut txn, 0, &i.to_string(), PosType::Unicode) + .unwrap(); + list.insert_with_txn(&mut txn, 0, i.into()).unwrap(); + } + txn.commit().unwrap(); + let b = LoroDoc::new(); + b.import(&loro.export(ExportMode::Snapshot).unwrap()) + .unwrap(); + loro.checkout(&Frontiers::default()).unwrap(); + { + let json = &loro.get_deep_value(); + assert_eq!( + json.to_json_value(), + serde_json::json!({"text":"","list":[],"map":{}}) + ); + } + + b.checkout(&ID::new(1, 2).into()).unwrap(); + { + let json = &b.get_deep_value(); + assert_eq!( + json.to_json_value(), + serde_json::json!({"text":"0","list":[0],"map":{"key":0}}) + ); + } + + loro.checkout(&ID::new(1, 3).into()).unwrap(); + { + let json = &loro.get_deep_value(); + assert_eq!( + json.to_json_value(), + serde_json::json!({"text":"0","list":[0],"map":{"key":1}}) + ); + } + + b.checkout(&ID::new(1, 29).into()).unwrap(); + { + let json = &b.get_deep_value(); + assert_eq!( + json.to_json_value(), + serde_json::json!({"text":"9876543210","list":[9,8,7,6,5,4,3,2,1,0],"map":{"key":9}}) + ); + } + } + + #[test] + fn import_batch_err_181() { + let a = LoroDoc::new_auto_commit(); + let update_a = a.export(ExportMode::Snapshot); + let b = LoroDoc::new_auto_commit(); + b.import_batch(&[update_a.unwrap()]).unwrap(); + b.get_text("text") + .insert(0, "hello", PosType::Unicode) + .unwrap(); + b.commit_then_renew(); + let oplog = b.oplog().lock(); + drop(oplog); + b.export(ExportMode::all_updates()).unwrap(); + } + + #[test] + fn poisoned_mutex_keeps_follow_up_operations_failed() { + let doc = LoroDoc::new(); + let oplog = doc.oplog.clone(); + let _ = std::panic::catch_unwind(AssertUnwindSafe(|| { + let _guard = oplog.lock(); + panic!("poison oplog"); + })); + + let err = std::panic::catch_unwind(AssertUnwindSafe(|| doc.oplog_vv())) + .expect_err("poisoned lock should continue to fail fast"); + let msg = if let Some(msg) = err.downcast_ref::<&str>() { + (*msg).to_string() + } else if let Some(msg) = err.downcast_ref::() { + msg.clone() + } else { + String::new() + }; + assert!(msg.contains("poisoned LoroMutex"), "{msg}"); + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/macros.rs b/3rdparty/loro/crates/loro-internal/src/macros.rs new file mode 100644 index 0000000000..1fbd90af1e --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/macros.rs @@ -0,0 +1,98 @@ +/// ```no_run +/// use rustc_hash::FxHashMap; +/// use loro_internal::fx_map; +/// +/// let mut expected = FxHashMap::default(); +/// expected.insert("test".to_string(), "test".to_string()); +/// expected.insert("test2".to_string(), "test2".to_string()); +/// let actual = fx_map!("test".into() => "test".into(), "test2".into() => "test2".into()); +/// assert_eq!(expected, actual); +/// ``` +#[macro_export] +macro_rules! fx_map { + ($($key:expr => $value:expr),*) => { + { + let mut m = rustc_hash::FxHashMap::default(); + $( + m.insert($key, $value); + )* + m + } + }; +} + +/// ```no_run +/// use loro_internal::vv; +/// +/// let v = vv!(1 => 2, 2 => 3); +/// assert_eq!(v.get(&1), Some(&2)); +/// assert_eq!(v.get(&2), Some(&3)); +/// ``` +#[macro_export] +macro_rules! vv { + ($($key:expr => $value:expr),*) => { + { + let mut m = $crate::version::VersionVector::default(); + $( + m.insert($key, $value); + )* + m + } + }; +} + +#[macro_export] +macro_rules! array_mut_ref { + ($arr:expr, [$a0:expr, $a1:expr]) => {{ + #[inline] + fn borrow_mut_ref(arr: &mut [T], a0: usize, a1: usize) -> (&mut T, &mut T) { + assert!(a0 != a1); + // SAFETY: this is safe because we know a0 != a1 + unsafe { + ( + &mut *(&mut arr[a0] as *mut _), + &mut *(&mut arr[a1] as *mut _), + ) + } + } + + borrow_mut_ref($arr, $a0, $a1) + }}; + ($arr:expr, [$a0:expr, $a1:expr, $a2:expr]) => {{ + #[inline] + fn borrow_mut_ref( + arr: &mut [T], + a0: usize, + a1: usize, + a2: usize, + ) -> (&mut T, &mut T, &mut T) { + assert!(a0 != a1 && a1 != a2 && a0 != a2); + // SAFETY: this is safe because we know there are not multiple mutable references to the same element + unsafe { + ( + &mut *(&mut arr[a0] as *mut _), + &mut *(&mut arr[a1] as *mut _), + &mut *(&mut arr[a2] as *mut _), + ) + } + } + + borrow_mut_ref($arr, $a0, $a1, $a2) + }}; +} + +#[cfg(test)] +mod test { + + #[test] + fn test_macro() { + let mut arr = vec![100, 101, 102, 103]; + let (a, b, _c) = array_mut_ref!(&mut arr, [1, 2, 3]); + assert_eq!(*a, 101); + assert_eq!(*b, 102); + *a = 50; + *b = 51; + assert!(arr[1] == 50); + assert!(arr[2] == 51); + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/op.rs b/3rdparty/loro/crates/loro-internal/src/op.rs new file mode 100644 index 0000000000..155f64cb8a --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/op.rs @@ -0,0 +1,621 @@ +use crate::{ + change::{Change, Lamport, Timestamp}, + container::{idx::ContainerIdx, ContainerID}, + estimated_size::EstimatedSize, + id::{Counter, PeerID, ID}, + oplog::BlockChangeRef, + span::{HasCounter, HasId, HasLamport}, +}; +use crate::{delta::DeltaValue, LoroValue}; +use either::Either; +use enum_as_inner::EnumAsInner; +use loro_common::{CompactIdLp, ContainerType, CounterSpan, IdFull, IdLp, IdSpan}; +use rle::{HasIndex, HasLength, Mergable, Sliceable}; +use serde::{Deserialize, Serialize}; +use std::{borrow::Cow, ops::Range}; + +mod content; +pub use content::*; + +/// Operation is a unit of change. +/// +/// A Op may have multiple atomic operations, since Op can be merged. +#[derive(Debug, Clone)] +pub struct Op { + pub(crate) counter: Counter, + pub(crate) container: ContainerIdx, + pub(crate) content: InnerContent, +} + +impl EstimatedSize for Op { + fn estimate_storage_size(&self) -> usize { + // TODO: use benchmark to get the optimal estimated size for each container type + self.content + .estimate_storage_size(self.container.get_type()) + } +} + +#[derive(Debug, Clone)] +pub(crate) struct OpWithId { + pub peer: PeerID, + pub op: Op, +} + +impl OpWithId { + #[allow(unused)] + pub fn id_span(&self) -> IdSpan { + IdSpan::new( + self.peer, + self.op.counter, + self.op.counter + self.op.atom_len() as Counter, + ) + } +} + +#[derive(Debug, Clone, PartialEq)] +#[cfg_attr(feature = "wasm", derive(Serialize, Deserialize))] +pub struct RemoteOp<'a> { + pub(crate) counter: Counter, + pub(crate) container: ContainerID, + pub(crate) content: RawOpContent<'a>, +} + +/// This is used to propagate messages between inner module. +/// It's a temporary struct, and will be converted to Op when it's persisted. +#[derive(Debug, Clone)] +pub struct RawOp<'a> { + pub id: ID, + pub lamport: Lamport, + pub container: ContainerIdx, + pub content: RawOpContent<'a>, +} + +impl RawOp<'_> { + pub(crate) fn id_full(&self) -> loro_common::IdFull { + IdFull::new(self.id.peer, self.id.counter, self.lamport) + } + + pub(crate) fn idlp(&self) -> loro_common::IdLp { + IdLp::new(self.id.peer, self.lamport) + } +} + +/// RichOp includes lamport and timestamp info, which is used for conflict resolution. +#[derive(Debug, Clone)] +pub struct RichOp<'a> { + op: Cow<'a, Op>, + pub peer: PeerID, + lamport: Lamport, + pub timestamp: Timestamp, + pub start: usize, + pub end: usize, +} + +impl Op { + #[inline] + #[allow(unused)] + pub(crate) fn new(id: ID, content: InnerContent, container: ContainerIdx) -> Self { + Op { + counter: id.counter, + content, + container, + } + } + + /// If the estimated storage size of the content is greater than the given size, + /// return the length of the content that makes the estimated storage size equal to the given size. + /// Otherwise, return None. + pub(crate) fn check_whether_slice_content_to_fit_in_size(&self, size: usize) -> Option { + if self.estimate_storage_size() <= size { + return None; + } + + match &self.content { + InnerContent::List(l) => match l { + crate::container::list::list_op::InnerListOp::Insert { .. } => { + if matches!(self.container.get_type(), ContainerType::Text) { + Some(size.min(self.atom_len())) + } else { + Some((size / 4).min(self.atom_len())) + } + } + crate::container::list::list_op::InnerListOp::InsertText { .. } => { + Some(size.min(self.atom_len())) + } + _ => unreachable!(), + }, + _ => unreachable!(), + } + } +} + +impl RemoteOp<'_> { + #[allow(unused)] + pub(crate) fn into_static(self) -> RemoteOp<'static> { + RemoteOp { + counter: self.counter, + container: self.container, + content: self.content.to_static(), + } + } +} + +impl Mergable for Op { + fn is_mergable(&self, other: &Self, cfg: &()) -> bool { + self.counter + self.content_len() as Counter == other.counter + && self.container == other.container + && self.content.is_mergable(&other.content, cfg) + } + + fn merge(&mut self, other: &Self, cfg: &()) { + self.content.merge(&other.content, cfg) + } +} + +impl HasLength for Op { + fn content_len(&self) -> usize { + self.content.content_len() + } +} + +impl Sliceable for Op { + fn slice(&self, from: usize, to: usize) -> Self { + assert!(to > from, "{to} should be greater than {from}"); + assert!(to <= self.atom_len()); + let content: InnerContent = self.content.slice(from, to); + Op { + counter: (self.counter + from as Counter), + content, + container: self.container, + } + } +} + +impl Mergable for RemoteOp<'_> { + fn is_mergable(&self, _other: &Self, _cfg: &()) -> bool { + // don't merge remote op, because it's already merged. + false + } + + fn merge(&mut self, _other: &Self, _: &()) { + unreachable!() + } +} + +impl HasLength for RemoteOp<'_> { + fn content_len(&self) -> usize { + self.content.atom_len() + } +} + +impl HasIndex for Op { + type Int = Counter; + + fn get_start_index(&self) -> Self::Int { + self.counter + } +} + +impl HasIndex for RemoteOp<'_> { + type Int = Counter; + + fn get_start_index(&self) -> Self::Int { + self.counter + } +} + +impl HasCounter for Op { + fn ctr_start(&self) -> Counter { + self.counter + } +} + +impl HasCounter for RemoteOp<'_> { + fn ctr_start(&self) -> Counter { + self.counter + } +} + +impl HasId for RichOp<'_> { + fn id_start(&self) -> ID { + ID { + peer: self.peer, + counter: self.op.counter + self.start as Counter, + } + } +} + +impl HasLength for RichOp<'_> { + fn content_len(&self) -> usize { + self.end - self.start + } +} + +impl HasLamport for RichOp<'_> { + fn lamport(&self) -> Lamport { + self.lamport + self.start as Lamport + } +} + +impl<'a> RichOp<'a> { + pub fn new_by_change(change: &Change, op: &'a Op) -> Self { + let diff = op.counter - change.id.counter; + RichOp { + op: Cow::Borrowed(op), + peer: change.id.peer, + lamport: change.lamport + diff as Lamport, + timestamp: change.timestamp, + start: 0, + end: op.atom_len(), + } + } + + /// we want the overlap part of the op and change[start..end] + /// + /// op is contained in the change, but it's not necessary overlap with change[start..end] + pub fn new_by_slice_on_change(change: &Change, start: i32, end: i32, op: &'a Op) -> Self { + debug_assert!(end > start); + let op_index_in_change = op.counter - change.id.counter; + let op_slice_start = (start - op_index_in_change).clamp(0, op.atom_len() as i32); + let op_slice_end = (end - op_index_in_change).clamp(0, op.atom_len() as i32); + RichOp { + op: Cow::Borrowed(op), + peer: change.id.peer, + lamport: change.lamport + op_index_in_change as Lamport, + timestamp: change.timestamp, + start: op_slice_start as usize, + end: op_slice_end as usize, + } + } + + pub fn new_by_cnt_range(change: &Change, span: CounterSpan, op: &'a Op) -> Option { + let op_index_in_change = op.counter - change.id.counter; + let op_slice_start = (span.start - op.counter).clamp(0, op.atom_len() as i32); + let op_slice_end = (span.end - op.counter).clamp(0, op.atom_len() as i32); + if op_slice_start == op_slice_end { + return None; + } + Some(RichOp { + op: Cow::Borrowed(op), + peer: change.id.peer, + lamport: change.lamport + op_index_in_change as Lamport, + timestamp: change.timestamp, + start: op_slice_start as usize, + end: op_slice_end as usize, + }) + } + + pub(crate) fn new_iter_by_cnt_range( + change: BlockChangeRef, + span: CounterSpan, + ) -> RichOpBlockIter { + RichOpBlockIter { + change, + span, + op_index: 0, + } + } + + pub fn op(&self) -> Cow<'_, Op> { + if self.start == 0 && self.end == self.op.content_len() { + self.op.clone() + } else { + Cow::Owned(self.op.slice(self.start, self.end)) + } + } + + pub fn raw_op(&self) -> &Op { + &self.op + } + + pub fn client_id(&self) -> u64 { + self.peer + } + + pub fn container(&self) -> ContainerIdx { + self.op.container + } + + pub fn timestamp(&self) -> i64 { + self.timestamp + } + + pub fn start(&self) -> usize { + self.start + } + + pub fn end(&self) -> usize { + self.end + } + + pub fn counter(&self) -> Counter { + self.op.counter + self.start as Counter + } + + #[allow(unused)] + pub(crate) fn id(&self) -> ID { + ID { + peer: self.peer, + counter: self.op.counter + self.start as Counter, + } + } + + pub(crate) fn id_full(&self) -> IdFull { + IdFull::new( + self.peer, + self.op.counter + self.start as Counter, + self.lamport + self.start as Lamport, + ) + } + + pub fn idlp(&self) -> IdLp { + IdLp { + lamport: self.lamport + self.start as Lamport, + peer: self.peer, + } + } +} + +pub(crate) struct RichOpBlockIter { + change: BlockChangeRef, + span: CounterSpan, + op_index: usize, +} + +impl Iterator for RichOpBlockIter { + type Item = RichOp<'static>; + + fn next(&mut self) -> Option { + let op = self.change.ops.get(self.op_index)?.clone(); + let op_offset_in_change = op.counter - self.change.id.counter; + let op_slice_start = (self.span.start - op.counter).clamp(0, op.atom_len() as i32); + let op_slice_end = (self.span.end - op.counter).clamp(0, op.atom_len() as i32); + self.op_index += 1; + if op_slice_start == op_slice_end { + return self.next(); + } + + Some(RichOp { + op: Cow::Owned(op), + peer: self.change.id.peer, + lamport: self.change.lamport + op_offset_in_change as Lamport, + timestamp: self.change.timestamp, + start: op_slice_start as usize, + end: op_slice_end as usize, + }) + } +} + +// Note: It will be encoded into binary format, so the order of its fields should not be changed. +#[derive(PartialEq, Debug, EnumAsInner, Clone, Serialize, Deserialize)] +pub enum ListSlice<'a> { + RawData(Cow<'a, [LoroValue]>), + RawStr { + str: Cow<'a, str>, + unicode_len: usize, + }, +} + +impl<'a> ListSlice<'a> { + pub fn from_borrowed_str(str: &'a str) -> Self { + Self::RawStr { + str: Cow::Borrowed(str), + unicode_len: str.chars().count(), + } + } +} + +#[repr(transparent)] +#[derive(PartialEq, Eq, Debug, Clone, Serialize)] +pub struct SliceRange(pub Range); + +const UNKNOWN_START: u32 = u32::MAX / 2; +impl SliceRange { + #[inline(always)] + pub fn is_unknown(&self) -> bool { + self.0.start == UNKNOWN_START + } + + #[inline(always)] + pub fn new_unknown(size: u32) -> Self { + Self(UNKNOWN_START..UNKNOWN_START + size) + } + + #[inline(always)] + pub fn new(range: Range) -> Self { + Self(range) + } + + #[inline(always)] + pub fn to_range(&self) -> Range { + self.0.start as usize..self.0.end as usize + } +} + +impl From> for SliceRange { + fn from(a: Range) -> Self { + SliceRange(a) + } +} + +impl HasLength for SliceRange { + fn content_len(&self) -> usize { + self.0.len() + } +} + +impl Sliceable for SliceRange { + fn slice(&self, from: usize, to: usize) -> Self { + if self.is_unknown() { + Self::new_unknown((to - from) as u32) + } else { + SliceRange(self.0.start + from as u32..self.0.start + to as u32) + } + } +} + +impl Mergable for SliceRange { + fn merge(&mut self, other: &Self, _: &()) { + if self.is_unknown() { + self.0.end += other.0.end - other.0.start; + } else { + self.0.end = other.0.end; + } + } + + fn is_mergable(&self, other: &Self, _conf: &()) -> bool + where + Self: Sized, + { + (self.is_unknown() && other.is_unknown()) || self.0.end == other.0.start + } +} + +impl ListSlice<'_> { + #[inline(always)] + pub fn unknown_range(len: usize) -> SliceRange { + let start = UNKNOWN_START; + let end = len as u32 + UNKNOWN_START; + SliceRange(start..end) + } + + #[inline(always)] + pub fn is_unknown(range: &SliceRange) -> bool { + range.is_unknown() + } + + pub fn to_static(&self) -> ListSlice<'static> { + match self { + ListSlice::RawData(x) => ListSlice::RawData(Cow::Owned(x.to_vec())), + ListSlice::RawStr { str, unicode_len } => ListSlice::RawStr { + str: Cow::Owned(str.to_string()), + unicode_len: *unicode_len, + }, + } + } +} + +impl HasLength for ListSlice<'_> { + fn content_len(&self) -> usize { + match self { + ListSlice::RawStr { unicode_len, .. } => *unicode_len, + ListSlice::RawData(x) => x.len(), + } + } +} + +impl Sliceable for ListSlice<'_> { + fn slice(&self, from: usize, to: usize) -> Self { + match self { + ListSlice::RawStr { + str, + unicode_len: _, + } => { + let ans = str.chars().skip(from).take(to - from).collect::(); + ListSlice::RawStr { + str: Cow::Owned(ans), + unicode_len: to - from, + } + } + ListSlice::RawData(x) => match x { + Cow::Borrowed(x) => ListSlice::RawData(Cow::Borrowed(&x[from..to])), + Cow::Owned(x) => ListSlice::RawData(Cow::Owned(x[from..to].into())), + }, + } + } +} + +impl Mergable for ListSlice<'_> { + fn is_mergable(&self, _other: &Self, _: &()) -> bool { + false + } +} + +#[derive(Debug, Clone)] +pub struct SliceWithId { + pub values: Either, + /// This field is no-none when diff calculating movable list that need to handle_unknown + pub elem_id: Option, + pub id: IdFull, +} + +impl DeltaValue for SliceWithId { + fn value_extend(&mut self, other: Self) -> Result<(), Self> { + if self.id.peer != other.id.peer { + return Err(other); + } + + if self.id.counter + self.length() as Counter != other.id.counter { + return Err(other); + } + + if self.id.lamport + self.length() as Lamport != other.id.lamport { + return Err(other); + } + + match (&mut self.values, &other.values) { + (Either::Left(left_range), Either::Left(right_range)) + if left_range.0.end == right_range.0.start => + { + left_range.0.end = right_range.0.end; + Ok(()) + } + _ => { + // If one is SliceRange and the other is LoroValue, we can't merge + Err(other) + } + } + } + + fn take(&mut self, target_len: usize) -> Self { + match &mut self.values { + Either::Left(range) => { + let ans = range.slice(0, target_len); + let this = range.slice(target_len, range.atom_len()); + *range = this; + let old_id = self.id; + self.id = self.id.inc(target_len as i32); + Self { + id: old_id, + values: Either::Left(ans), + elem_id: None, + } + } + Either::Right(_) => unimplemented!(), + } + } + + fn length(&self) -> usize { + match &self.values { + Either::Left(r) => r.atom_len(), + Either::Right(_) => 1, + } + } +} + +#[cfg(test)] +mod test { + use crate::LoroValue; + + use super::ListSlice; + + #[test] + fn fix_fields_order() { + let list_slice = vec![ + ListSlice::RawData(vec![LoroValue::Bool(true)].into()), + ListSlice::RawStr { + str: "".into(), + unicode_len: 0, + }, + ]; + let list_slice_buf = vec![2, 0, 1, 1, 1, 1, 0, 0]; + assert_eq!( + &postcard::to_allocvec(&list_slice).unwrap(), + &list_slice_buf + ); + assert_eq!( + postcard::from_bytes::>(&list_slice_buf).unwrap(), + list_slice + ); + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/op/content.rs b/3rdparty/loro/crates/loro-internal/src/op/content.rs new file mode 100644 index 0000000000..919b653432 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/op/content.rs @@ -0,0 +1,476 @@ +use std::sync::Arc; + +use enum_as_inner::EnumAsInner; +use loro_common::{ContainerID, ContainerType, LoroValue}; +use rle::{HasLength, Mergable, Sliceable}; +#[cfg(feature = "wasm")] +use serde::{Deserialize, Serialize}; + +use crate::{ + arena::SharedArena, + container::{ + list::list_op::{InnerListOp, ListOp}, + map::MapSet, + tree::tree_op::TreeOp, + }, + encoding::OwnedValue, +}; + +#[derive(EnumAsInner, Debug, Clone)] +pub enum InnerContent { + List(InnerListOp), + Map(MapSet), + Tree(Arc), + // The future content should not use any encoded arena context. + Future(FutureInnerContent), +} + +impl InnerContent { + pub fn visit_created_children(&self, arena: &SharedArena, f: &mut dyn FnMut(&ContainerID)) { + match self { + InnerContent::List(l) => match l { + InnerListOp::Insert { slice, .. } => { + for v in arena.iter_value_slice(slice.to_range()) { + if let LoroValue::Container(c) = v { + f(&c); + } + } + } + InnerListOp::Set { value, .. } => { + if let LoroValue::Container(c) = value { + f(c); + } + } + + InnerListOp::Move { .. } => {} + InnerListOp::InsertText { .. } => {} + InnerListOp::Delete(_) => {} + InnerListOp::StyleStart { .. } => {} + InnerListOp::StyleEnd => {} + }, + crate::op::InnerContent::Map(m) => { + if let Some(LoroValue::Container(c)) = &m.value { + f(c); + } + } + crate::op::InnerContent::Tree(t) => { + if let TreeOp::Create { target, .. } = t.as_ref() { + let id = target.associated_meta_container(); + f(&id); + } + } + crate::op::InnerContent::Future(f) => match &f { + #[cfg(feature = "counter")] + crate::op::FutureInnerContent::Counter(_) => {} + crate::op::FutureInnerContent::Unknown { .. } => {} + }, + } + } +} + +impl InnerContent { + pub fn estimate_storage_size(&self, kind: ContainerType) -> usize { + match self { + InnerContent::List(l) => l.estimate_storage_size(kind), + InnerContent::Map(_) => 3, + InnerContent::Tree(_) => 8, + InnerContent::Future(f) => f.estimate_storage_size(), + } + } +} + +#[derive(EnumAsInner, Debug, Clone)] +pub enum FutureInnerContent { + #[cfg(feature = "counter")] + Counter(f64), + Unknown { + prop: i32, + value: Box, + }, +} +impl FutureInnerContent { + fn estimate_storage_size(&self) -> usize { + match self { + #[cfg(feature = "counter")] + FutureInnerContent::Counter(_) => 4, + FutureInnerContent::Unknown { .. } => 6, + } + } +} + +// Note: It will be encoded into binary format, so the order of its fields should not be changed. +#[derive(EnumAsInner, Debug, PartialEq)] +#[cfg_attr(feature = "wasm", derive(Serialize, Deserialize,))] +pub enum RawOpContent<'a> { + Map(MapSet), + List(ListOp<'a>), + Tree(Arc), + #[cfg(feature = "counter")] + Counter(f64), + Unknown { + prop: i32, + value: OwnedValue, + }, +} + +impl Clone for RawOpContent<'_> { + fn clone(&self) -> Self { + match self { + Self::Map(arg0) => Self::Map(arg0.clone()), + Self::List(arg0) => Self::List(arg0.clone()), + Self::Tree(arg0) => Self::Tree(arg0.clone()), + #[cfg(feature = "counter")] + Self::Counter(x) => Self::Counter(*x), + Self::Unknown { prop, value } => Self::Unknown { + prop: *prop, + value: value.clone(), + }, + } + } +} + +impl RawOpContent<'_> { + pub fn to_static(&self) -> RawOpContent<'static> { + match self { + Self::Map(arg0) => RawOpContent::Map(arg0.clone()), + Self::List(arg0) => match arg0 { + ListOp::Insert { slice, pos } => RawOpContent::List(ListOp::Insert { + slice: slice.to_static(), + pos: *pos, + }), + ListOp::Delete(x) => RawOpContent::List(ListOp::Delete(*x)), + ListOp::StyleStart { + start, + end, + key, + value, + info, + } => RawOpContent::List(ListOp::StyleStart { + start: *start, + end: *end, + key: key.clone(), + value: value.clone(), + info: *info, + }), + ListOp::StyleEnd => RawOpContent::List(ListOp::StyleEnd), + ListOp::Move { + from, + to, + elem_id: from_id, + } => RawOpContent::List(ListOp::Move { + from: *from, + to: *to, + elem_id: *from_id, + }), + ListOp::Set { elem_id, value } => RawOpContent::List(ListOp::Set { + elem_id: *elem_id, + value: value.clone(), + }), + }, + Self::Tree(arg0) => RawOpContent::Tree(arg0.clone()), + #[cfg(feature = "counter")] + Self::Counter(x) => RawOpContent::Counter(*x), + Self::Unknown { prop, value } => RawOpContent::Unknown { + prop: *prop, + value: value.clone(), + }, + } + } +} + +impl HasLength for RawOpContent<'_> { + fn content_len(&self) -> usize { + match self { + RawOpContent::Map(x) => x.content_len(), + RawOpContent::List(x) => x.content_len(), + RawOpContent::Tree(x) => x.content_len(), + #[cfg(feature = "counter")] + RawOpContent::Counter(_) => 1, + RawOpContent::Unknown { .. } => 1, + } + } +} + +impl Mergable for RawOpContent<'_> { + fn is_mergable(&self, other: &Self, _conf: &()) -> bool + where + Self: Sized, + { + match (self, other) { + (RawOpContent::List(x), RawOpContent::List(y)) => x.is_mergable(y, &()), + (RawOpContent::Tree(x), RawOpContent::Tree(y)) => x.is_mergable(y, &()), + _ => false, + } + } + + fn merge(&mut self, _other: &Self, _conf: &()) + where + Self: Sized, + { + match self { + RawOpContent::List(x) => match _other { + RawOpContent::List(y) => x.merge(y, &()), + _ => unreachable!(), + }, + _ => unreachable!(), + } + } +} + +impl HasLength for InnerContent { + fn content_len(&self) -> usize { + match self { + InnerContent::List(list) => list.atom_len(), + InnerContent::Map(_) => 1, + InnerContent::Tree(_) => 1, + InnerContent::Future(_) => 1, + } + } +} + +impl Sliceable for InnerContent { + fn slice(&self, from: usize, to: usize) -> Self { + match self { + a @ InnerContent::Map(_) => { + assert!(from == 0 && to == 1); + a.clone() + } + a @ InnerContent::Tree(_) => { + assert!(from == 0 && to == 1); + a.clone() + } + InnerContent::List(x) => InnerContent::List(x.slice(from, to)), + InnerContent::Future(f) => { + assert!(from == 0 && to == 1); + InnerContent::Future(f.clone()) + } + } + } +} + +impl Mergable for InnerContent { + fn is_mergable(&self, other: &Self, _conf: &()) -> bool + where + Self: Sized, + { + match (self, other) { + (InnerContent::List(x), InnerContent::List(y)) => x.is_mergable(y, &()), + _ => false, + } + } + + fn merge(&mut self, _other: &Self, _conf: &()) + where + Self: Sized, + { + match self { + InnerContent::List(x) => match _other { + InnerContent::List(y) => x.merge(y, &()), + _ => unreachable!(), + }, + _ => unreachable!(), + } + } +} + +#[cfg(test)] +mod tests { + use std::{borrow::Cow, sync::Arc}; + + use fractional_index::FractionalIndex; + use loro_common::{ContainerID, ContainerType, TreeID}; + use rle::{HasLength, Mergable, Sliceable}; + + use super::*; + use crate::{ + container::{ + list::list_op::{DeleteSpanWithId, ListOp}, + map::MapSet, + tree::tree_op::TreeOp, + }, + op::{ListSlice, SliceRange}, + InternalString, LoroValue, ID, + }; + + #[test] + fn raw_op_content_reports_lengths_and_static_content_without_borrowing() { + let raw_insert = RawOpContent::List(ListOp::Insert { + slice: ListSlice::RawData(Cow::Owned(vec![1_i64.into(), 2_i64.into()])), + pos: 3, + }); + assert_eq!(raw_insert.content_len(), 2); + match raw_insert.to_static() { + RawOpContent::List(ListOp::Insert { slice, pos }) => { + assert_eq!(pos, 3); + assert_eq!( + slice, + ListSlice::RawData(Cow::Owned(vec![1_i64.into(), 2_i64.into()])) + ); + } + other => panic!("expected list insert, got {other:?}"), + } + + let raw_map = RawOpContent::Map(MapSet { + key: InternalString::from("title"), + value: Some("draft".into()), + }); + assert_eq!(raw_map.content_len(), 1); + assert_eq!(raw_map.to_static(), raw_map); + + let raw_tree = RawOpContent::Tree(Arc::new(TreeOp::Delete { + target: TreeID::new(2, 9), + })); + assert_eq!(raw_tree.content_len(), 1); + assert_eq!(raw_tree.to_static(), raw_tree); + + let raw_unknown = RawOpContent::Unknown { + prop: 7, + value: OwnedValue::I64(11), + }; + assert_eq!(raw_unknown.content_len(), 1); + assert_eq!(raw_unknown.to_static(), raw_unknown); + + #[cfg(feature = "counter")] + { + let raw_counter = RawOpContent::Counter(1.25); + assert_eq!(raw_counter.content_len(), 1); + assert_eq!(raw_counter.to_static(), raw_counter); + } + } + + #[test] + fn raw_op_content_merges_only_mergeable_list_operations() { + let mut left = + RawOpContent::List(ListOp::Delete(DeleteSpanWithId::new(ID::new(1, 0), 0, 1))); + let right = RawOpContent::List(ListOp::Delete(DeleteSpanWithId::new(ID::new(1, 1), 0, 1))); + assert!(left.is_mergable(&right, &())); + left.merge(&right, &()); + assert_eq!(left.content_len(), 2); + + let map = RawOpContent::Map(MapSet { + key: InternalString::from("k"), + value: Some(1_i64.into()), + }); + assert!(!left.is_mergable(&map, &())); + } + + #[test] + fn inner_content_lengths_storage_estimates_slices_and_merges_follow_inner_ops() { + let mut left = InnerContent::List(InnerListOp::Insert { + slice: SliceRange(0..2), + pos: 0, + }); + let right = InnerContent::List(InnerListOp::Insert { + slice: SliceRange(2..4), + pos: 2, + }); + assert_eq!(left.content_len(), 2); + assert_eq!(left.estimate_storage_size(ContainerType::List), 8); + assert!(left.is_mergable(&right, &())); + left.merge(&right, &()); + assert_eq!(left.content_len(), 4); + + let sliced = left.slice(1, 3); + match sliced { + InnerContent::List(InnerListOp::Insert { slice, pos }) => { + assert_eq!(slice, SliceRange(1..3)); + assert_eq!(pos, 1); + } + other => panic!("expected sliced list insert, got {other:?}"), + } + + let map = InnerContent::Map(MapSet { + key: InternalString::from("flag"), + value: Some(true.into()), + }); + assert_eq!(map.content_len(), 1); + assert_eq!(map.estimate_storage_size(ContainerType::Map), 3); + assert!(matches!(map.slice(0, 1), InnerContent::Map(_))); + + let tree = InnerContent::Tree(Arc::new(TreeOp::Delete { + target: TreeID::new(4, 5), + })); + assert_eq!(tree.content_len(), 1); + assert_eq!(tree.estimate_storage_size(ContainerType::Tree), 8); + assert!(matches!(tree.slice(0, 1), InnerContent::Tree(_))); + + let future = InnerContent::Future(FutureInnerContent::Unknown { + prop: 99, + value: Box::new(OwnedValue::Null), + }); + assert_eq!(future.content_len(), 1); + assert_eq!(future.estimate_storage_size(ContainerType::Unknown(3)), 6); + assert!(matches!(future.slice(0, 1), InnerContent::Future(_))); + + #[cfg(feature = "counter")] + { + let counter = InnerContent::Future(FutureInnerContent::Counter(2.0)); + assert_eq!(counter.estimate_storage_size(ContainerType::Counter), 4); + } + } + + #[test] + fn visit_created_children_reports_containers_created_by_list_map_and_tree_content() { + let arena = SharedArena::new(); + let list_child = ContainerID::new_normal(ID::new(1, 1), ContainerType::List); + let text_child = ContainerID::new_normal(ID::new(1, 2), ContainerType::Text); + let values = arena.alloc_values( + vec![ + LoroValue::Container(list_child.clone()), + "plain".into(), + LoroValue::Container(text_child.clone()), + ] + .into_iter(), + ); + let list = InnerContent::List(InnerListOp::Insert { + slice: SliceRange(values.start as u32..values.end as u32), + pos: 0, + }); + let mut children = Vec::new(); + list.visit_created_children(&arena, &mut |id| children.push(id.clone())); + assert_eq!(children, vec![list_child.clone(), text_child.clone()]); + + let map_child = ContainerID::new_normal(ID::new(2, 1), ContainerType::Map); + let map = InnerContent::Map(MapSet { + key: InternalString::from("child"), + value: Some(LoroValue::Container(map_child.clone())), + }); + let mut children = Vec::new(); + map.visit_created_children(&arena, &mut |id| children.push(id.clone())); + assert_eq!(children, vec![map_child]); + + let tree_target = TreeID::new(3, 7); + let tree = InnerContent::Tree(Arc::new(TreeOp::Create { + target: tree_target, + parent: None, + position: FractionalIndex::default(), + })); + let mut children = Vec::new(); + tree.visit_created_children(&arena, &mut |id| children.push(id.clone())); + assert_eq!(children, vec![tree_target.associated_meta_container()]); + + for tree in [ + TreeOp::Move { + target: tree_target, + parent: None, + position: FractionalIndex::default(), + }, + TreeOp::Delete { + target: tree_target, + }, + ] { + let tree = InnerContent::Tree(Arc::new(tree)); + let mut children = Vec::new(); + tree.visit_created_children(&arena, &mut |id| children.push(id.clone())); + assert!(children.is_empty()); + } + + let future = InnerContent::Future(FutureInnerContent::Unknown { + prop: 1, + value: Box::new(OwnedValue::False), + }); + let mut children = Vec::new(); + future.visit_created_children(&arena, &mut |id| children.push(id.clone())); + assert!(children.is_empty()); + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/oplog.rs b/3rdparty/loro/crates/loro-internal/src/oplog.rs new file mode 100644 index 0000000000..27146cdebc --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/oplog.rs @@ -0,0 +1,1018 @@ +mod change_store; +pub(crate) mod loro_dag; +mod pending_changes; + +use crate::sync::{AtomicUsize, Mutex}; +use bytes::Bytes; +use std::borrow::Cow; +use std::cell::RefCell; +use std::cmp::Ordering; +use std::rc::Rc; +use std::sync::Arc; +use tracing::trace_span; + +use self::change_store::iter::MergedChangeIter; +use self::pending_changes::{PendingChanges, PendingChangesRollback}; +use super::arena::{SharedArena, SharedArenaRollback}; +use crate::change::{get_sys_timestamp, Change, Lamport, Timestamp}; +use crate::configure::Configure; +use crate::container::list::list_op; +use crate::dag::{Dag, DagUtils}; +use crate::diff_calc::DiffMode; +use crate::encoding::decode_oplog; +use crate::encoding::{ImportStatus, ParsedHeaderAndBody}; +use crate::history_cache::ContainerHistoryCache; +use crate::id::{Counter, PeerID, ID}; +use crate::op::{FutureInnerContent, ListSlice, RawOpContent, RemoteOp, RichOp}; +use crate::span::{HasCounterSpan, HasLamportSpan}; +use crate::version::{Frontiers, ImVersionVector, VersionVector}; +use crate::LoroError; +use change_store::{BlockOpRef, ChangeStoreRollback}; +use loro_common::{ContainerType, HasIdSpan, IdLp, IdSpan}; +use rle::{HasLength, RleVec, Sliceable}; +use smallvec::SmallVec; + +pub use self::loro_dag::{AppDag, AppDagNode, FrontiersNotIncluded}; +pub use change_store::{BlockChangeRef, ChangeStore}; + +/// [OpLog] store all the ops i.e. the history. +/// It allows multiple [AppState] to attach to it. +/// So you can derive different versions of the state from the [OpLog]. +/// It allows us to build a version control system. +/// +/// The causal graph should always be a DAG and complete. So we can always find the LCA. +/// If deps are missing, we can't import the change. It will be put into the `pending_changes`. +pub struct OpLog { + pub(crate) dag: AppDag, + pub(crate) arena: SharedArena, + visible_op_count: Arc, + change_store: ChangeStore, + history_cache: Mutex, + /// Pending changes that haven't been applied to the dag. + /// A change can be imported only when all its deps are already imported. + /// Key is the ID of the missing dep + pub(crate) pending_changes: PendingChanges, + /// Whether we are importing a batch of changes. + /// If so the Dag's frontiers won't be updated until the batch is finished. + pub(crate) batch_importing: bool, + pub(crate) configure: Configure, + /// The uncommitted change, it's a placeholder for the change + /// that is being edited in pre-commit callback. + pub(crate) uncommitted_change: Option, + pub(crate) import_rollback: Option, +} + +pub(crate) struct ImportRollback { + old_vv: VersionVector, + arena: SharedArenaRollback, + change_store: ChangeStoreRollback, + pending: PendingChangesRollback, +} + +#[derive(Debug, Default, Clone, Copy)] +pub(crate) struct ImportChangesPreflight { + pub applies_to_dag: bool, + pub has_deps_before_shallow_root: bool, + pub needs_state_apply_rollback: bool, +} + +impl std::fmt::Debug for OpLog { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("OpLog") + .field("dag", &self.dag) + .field("pending_changes", &self.pending_changes) + .finish() + } +} + +impl OpLog { + #[inline] + pub(crate) fn new(visible_op_count: Arc) -> Self { + let arena = SharedArena::new(); + let cfg = Configure::default(); + let change_store = ChangeStore::new_mem(&arena, cfg.merge_interval_in_s.clone()); + Self { + visible_op_count, + history_cache: Mutex::new(ContainerHistoryCache::new(change_store.clone(), None)), + dag: AppDag::new(change_store.clone()), + change_store, + arena, + pending_changes: Default::default(), + batch_importing: false, + configure: cfg, + uncommitted_change: None, + import_rollback: None, + } + } + + #[inline] + fn calc_visible_op_count(&self) -> usize { + let total = self.dag.vv().values().sum::() as usize; + let shallow = self + .dag + .shallow_since_vv() + .iter() + .map(|(_, ops)| *ops) + .sum::() as usize; + total - shallow + } + + #[inline] + pub(crate) fn visible_op_count_exact(&self) -> usize { + self.calc_visible_op_count() + } + + #[inline] + pub(crate) fn refresh_visible_op_count(&self) -> usize { + let count = self.calc_visible_op_count(); + self.visible_op_count + .store(count, std::sync::atomic::Ordering::Release); + count + } + + /// Incrementally bump the cached visible op count for newly applied *local* + /// ops. Local ops are always visible (never behind the shallow root), so the + /// visible count grows by exactly `delta`. This avoids a per-op full + /// recompute via [`Self::calc_visible_op_count`], which iterates the version + /// vectors and heap-allocates an `im::HashMap` iterator on every call. + #[inline] + pub(crate) fn inc_visible_op_count(&self, delta: usize) { + self.visible_op_count + .fetch_add(delta, std::sync::atomic::Ordering::Release); + } + + #[cfg(test)] + pub(crate) fn cached_visible_op_count(&self) -> usize { + self.visible_op_count + .load(std::sync::atomic::Ordering::Acquire) + } + + #[inline] + pub fn dag(&self) -> &AppDag { + &self.dag + } + + pub fn change_store(&self) -> &ChangeStore { + &self.change_store + } + + /// Get the change with the given peer and lamport. + /// + /// If not found, return the change with the greatest lamport that is smaller than the given lamport. + pub fn get_change_with_lamport_lte( + &self, + peer: PeerID, + lamport: Lamport, + ) -> Option { + let ans = self + .change_store + .get_change_by_lamport_lte(IdLp::new(peer, lamport))?; + debug_assert!(ans.lamport <= lamport); + Some(ans) + } + + pub fn get_timestamp_of_version(&self, f: &Frontiers) -> Timestamp { + let mut timestamp = Timestamp::default(); + for id in f.iter() { + if let Some(change) = self.lookup_change(id) { + timestamp = timestamp.max(change.timestamp); + } + } + + timestamp + } + + #[inline] + pub fn is_empty(&self) -> bool { + self.dag.is_empty() && self.arena.can_import_snapshot() + } + + /// This is the **only** place to update the `OpLog.changes` + pub(crate) fn insert_new_change(&mut self, change: Change, from_local: bool) { + let s = trace_span!( + "insert_new_change", + id = ?change.id, + lamport = change.lamport, + deps = ?change.deps + ); + let _enter = s.enter(); + let rollback_old_vv = self + .import_rollback + .as_ref() + .and_then(|x| (!x.old_vv.is_empty()).then_some(&x.old_vv)); + self.dag + .handle_new_change(&change, from_local, rollback_old_vv); + self.history_cache + .lock() + .insert_by_new_change(&change, true, true); + self.register_container_and_parent_link(&change); + if let Some(rollback) = self.import_rollback.as_mut() { + self.change_store.insert_change_with_rollback( + change, + true, + from_local, + &mut rollback.change_store, + ); + } else { + self.change_store.insert_change(change, true, from_local); + } + self.refresh_visible_op_count(); + } + + pub(crate) fn begin_import_rollback(&mut self) { + let arena = self.arena.checkpoint_for_rollback(); + self.begin_import_rollback_with_arena(arena); + } + + pub(crate) fn begin_import_rollback_with_arena(&mut self, arena: SharedArenaRollback) { + debug_assert!(self.import_rollback.is_none()); + let old_vv = self.vv().clone(); + self.dag.begin_import_rollback(); + self.import_rollback = Some(ImportRollback { + old_vv: old_vv.clone(), + arena, + change_store: ChangeStoreRollback::new(old_vv), + pending: Default::default(), + }); + } + + pub(crate) fn commit_import_rollback(&mut self) { + self.dag.commit_import_rollback(); + self.import_rollback = None; + } + + pub(crate) fn preflight_import_changes(&self, changes: &[Change]) -> ImportChangesPreflight { + let mut ans = ImportChangesPreflight::default(); + let pending_needs_state_apply_rollback = + self.pending_changes.has_state_apply_rollback_ops(); + for change in changes { + if change.ctr_end() <= self.vv().get(&change.id.peer).copied().unwrap_or(0) { + continue; + } + + if self.dag.import_deps_before_shallow_root(&change.deps) { + ans.has_deps_before_shallow_root = true; + continue; + } + + if self + .dag + .get_change_lamport_from_deps(&change.deps) + .is_none() + { + continue; + } + + ans.applies_to_dag = true; + if change.ops.iter().any(|op| { + matches!( + op.container.get_type(), + ContainerType::List | ContainerType::Tree + ) + }) { + ans.needs_state_apply_rollback = true; + } + } + + // Any newly applied change can unlock pending changes whose ops are not + // visible in `changes`, so include pending in the rollback decision. + // Keep this narrow: text/map-only pending changes cannot return a + // state-apply error, and forcing rollback there adds lock traffic to + // small sync/import workloads. + if ans.applies_to_dag && pending_needs_state_apply_rollback { + ans.needs_state_apply_rollback = true; + } + + #[cfg(test)] + if ans.applies_to_dag { + ans.needs_state_apply_rollback = true; + } + + ans + } + + pub(crate) fn rollback_import(&mut self) { + let Some(rollback) = self.import_rollback.take() else { + return; + }; + + self.change_store.rollback_import(rollback.change_store); + self.dag.rollback_import(); + rollback.pending.rollback(&mut self.pending_changes); + self.history_cache.lock().free_all(); + self.arena.rollback(rollback.arena); + self.refresh_visible_op_count(); + } + + pub(crate) fn reset_to_empty_for_failed_snapshot_import( + &mut self, + arena_checkpoint: SharedArenaRollback, + ) { + let arena = self.arena.clone(); + let configure = self.configure.clone(); + arena.rollback(arena_checkpoint); + let change_store = ChangeStore::new_mem(&arena, configure.merge_interval_in_s.clone()); + self.history_cache = Mutex::new(ContainerHistoryCache::new(change_store.clone(), None)); + self.dag = AppDag::new(change_store.clone()); + self.change_store = change_store; + self.pending_changes = Default::default(); + self.batch_importing = false; + self.configure = configure; + self.uncommitted_change = None; + self.import_rollback = None; + self.visible_op_count + .store(0, std::sync::atomic::Ordering::Release); + } + + #[inline(always)] + pub(crate) fn with_history_cache(&self, f: F) -> R + where + F: FnOnce(&mut ContainerHistoryCache) -> R, + { + let mut history_cache = self.history_cache.lock(); + f(&mut history_cache) + } + + pub fn has_history_cache(&self) -> bool { + self.history_cache.lock().has_cache() + } + + pub fn free_history_cache(&self) { + let mut history_cache = self.history_cache.lock(); + history_cache.free(); + } + + #[cfg(test)] + #[allow(dead_code)] + pub(crate) fn pending_changes_len(&self) -> usize { + self.pending_changes.len() + } + + /// Import a change. + /// + /// Pending changes that haven't been applied to the dag. + /// A change can be imported only when all its deps are already imported. + /// Key is the ID of the missing dep + /// + /// # Err + /// + /// - Return Err(LoroError::UsedOpID) when the change's id is occupied + /// - Return Err(LoroError::DecodeError) when the change's deps are missing + pub(crate) fn import_local_change(&mut self, change: Change) -> Result<(), LoroError> { + self.insert_new_change(change, true); + Ok(()) + } + + /// Trim the known part of change + pub(crate) fn trim_the_known_part_of_change(&self, change: Change) -> Option { + let Some(&end) = self.dag.vv().get(&change.id.peer) else { + return Some(change); + }; + + if change.id.counter >= end { + return Some(change); + } + + if change.ctr_end() <= end { + return None; + } + + let offset = (end - change.id.counter) as usize; + Some(change.slice(offset, change.atom_len())) + } + + #[allow(unused)] + fn check_id_is_not_duplicated(&self, id: ID) -> Result<(), LoroError> { + let cur_end = self.dag.vv().get(&id.peer).cloned().unwrap_or(0); + if cur_end > id.counter { + return Err(LoroError::UsedOpID { id }); + } + + Ok(()) + } + + /// Ensure the new change is greater than the last peer's id and the counter is continuous. + /// + /// It can be false when users use detached editing mode and use a custom peer id. + // This method might be slow and can be optimized if needed in the future. + pub(crate) fn check_change_greater_than_last_peer_id( + &self, + peer: PeerID, + counter: Counter, + deps: &Frontiers, + ) -> Result<(), LoroError> { + if counter == 0 { + return Ok(()); + } + + if !self.configure.detached_editing() { + return Ok(()); + } + + let mut max_last_counter = -1; + for dep in deps.iter() { + let dep_vv = self + .dag + .get_vv(dep) + .ok_or(LoroError::FrontiersNotFound(dep))?; + max_last_counter = max_last_counter.max(dep_vv.get(&peer).cloned().unwrap_or(0) - 1); + } + + if counter != max_last_counter + 1 { + return Err(LoroError::ConcurrentOpsWithSamePeerID { + peer, + last_counter: max_last_counter, + current: counter, + }); + } + + Ok(()) + } + + pub(crate) fn next_id(&self, peer: PeerID) -> ID { + let cnt = self.dag.vv().get(&peer).copied().unwrap_or(0); + ID::new(peer, cnt) + } + + pub(crate) fn vv(&self) -> &VersionVector { + self.dag.vv() + } + + pub(crate) fn frontiers(&self) -> &Frontiers { + self.dag.frontiers() + } + + /// - Ordering::Less means self is less than target or parallel + /// - Ordering::Equal means versions equal + /// - Ordering::Greater means self's version is greater than target + pub fn cmp_with_frontiers(&self, other: &Frontiers) -> Ordering { + self.dag.cmp_with_frontiers(other) + } + + /// Compare two [Frontiers] causally. + /// + /// If one of the [Frontiers] are not included, it will return [FrontiersNotIncluded]. + #[inline] + pub fn cmp_frontiers( + &self, + a: &Frontiers, + b: &Frontiers, + ) -> Result, FrontiersNotIncluded> { + self.dag.cmp_frontiers(a, b) + } + + pub(crate) fn get_min_lamport_at(&self, id: ID) -> Lamport { + self.get_change_at(id).map(|c| c.lamport).unwrap_or(0) + } + + pub(crate) fn get_lamport_at(&self, id: ID) -> Option { + self.get_change_at(id) + .map(|c| c.lamport + (id.counter - c.id.counter) as Lamport) + } + + pub(crate) fn iter_ops(&self, id_span: IdSpan) -> impl Iterator> + '_ { + let change_iter = self.change_store.iter_changes(id_span); + change_iter.flat_map(move |c| RichOp::new_iter_by_cnt_range(c, id_span.counter)) + } + + pub(crate) fn get_max_lamport_at(&self, id: ID) -> Lamport { + self.get_change_at(id) + .map(|c| { + let change_counter = c.id.counter as u32; + c.lamport + c.ops().last().map(|op| op.counter).unwrap_or(0) as u32 - change_counter + }) + .unwrap_or(Lamport::MAX) + } + + pub fn get_change_at(&self, id: ID) -> Option { + self.change_store.get_change(id) + } + + pub(crate) fn set_uncommitted_change(&mut self, change: Change) { + self.uncommitted_change = Some(change); + } + + pub(crate) fn get_uncommitted_change_in_span( + &self, + id_span: IdSpan, + ) -> Option> { + self.uncommitted_change.as_ref().and_then(|c| { + if c.id_span() == id_span { + Some(Cow::Borrowed(c)) + } else if let Some((start, end)) = id_span.get_slice_range_on(&c.id_span()) { + Some(Cow::Owned(c.slice(start, end))) + } else { + None + } + }) + } + + pub fn get_deps_of(&self, id: ID) -> Option { + self.get_change_at(id).map(|c| { + if c.id.counter == id.counter { + c.deps.clone() + } else { + Frontiers::from_id(id.inc(-1)) + } + }) + } + + pub fn get_remote_change_at(&self, id: ID) -> Option>> { + let change = self.get_change_at(id)?; + Some(convert_change_to_remote(&self.arena, &change)) + } + + pub(crate) fn import_unknown_lamport_pending_changes( + &mut self, + remote_changes: Vec, + ) -> Result<(), LoroError> { + self.extend_pending_changes_with_unknown_lamport(remote_changes) + } + + /// lookup change by id. + /// + /// if id does not included in this oplog, return None + pub(crate) fn lookup_change(&self, id: ID) -> Option { + self.change_store.get_change(id) + } + + #[inline(always)] + pub(crate) fn export_change_store_from(&self, vv: &VersionVector, f: &Frontiers) -> Bytes { + self.change_store + .export_from(vv, f, self.vv(), self.frontiers()) + } + + #[inline(always)] + pub(crate) fn export_change_store_in_range( + &self, + vv: &VersionVector, + f: &Frontiers, + to_vv: &VersionVector, + to_frontiers: &Frontiers, + ) -> Bytes { + self.change_store.export_from(vv, f, to_vv, to_frontiers) + } + + #[inline(always)] + pub(crate) fn export_blocks_from(&self, vv: &VersionVector, w: &mut W) { + self.change_store + .export_blocks_from(vv, self.shallow_since_vv(), self.vv(), w) + } + + #[inline(always)] + pub(crate) fn export_blocks_in_range(&self, spans: &[IdSpan], w: &mut W) { + self.change_store.export_blocks_in_range(spans, w) + } + + pub(crate) fn fork_changes_up_to(&self, frontiers: &Frontiers) -> Option { + let vv = self.dag.frontiers_to_vv(frontiers)?; + Some( + self.change_store + .fork_changes_up_to(self.dag.shallow_since_vv(), frontiers, &vv), + ) + } + + #[inline(always)] + pub(crate) fn decode(&mut self, data: ParsedHeaderAndBody) -> Result { + decode_oplog(self, data) + } + + /// iterates over all changes between LCA(common ancestors) to the merged version of (`from` and `to`) causally + /// + /// Tht iterator will include a version vector when the change is applied + /// + /// returns: (common_ancestor_vv, iterator) + /// + /// Note: the change returned by the iterator may include redundant ops at the beginning, you should trim it by yourself. + /// You can trim it by the provided counter value. It should start with the counter. + /// + /// If frontiers are provided, it will be faster (because we don't need to calculate it from version vector + #[allow(clippy::type_complexity)] + pub(crate) fn iter_from_lca_causally( + &self, + from: &VersionVector, + from_frontiers: &Frontiers, + to: &VersionVector, + to_frontiers: &Frontiers, + ) -> ( + VersionVector, + DiffMode, + impl Iterator< + Item = ( + BlockChangeRef, + (Counter, Counter), + Rc>, + ), + > + '_, + ) { + let mut merged_vv = from.clone(); + merged_vv.merge(to); + loro_common::debug!("to_frontiers={:?} vv={:?}", &to_frontiers, to); + let (mut common_ancestors, mut diff_mode) = + self.dag.find_common_ancestor(from_frontiers, to_frontiers); + if diff_mode == DiffMode::Checkout && to > from { + diff_mode = DiffMode::Import; + } + + let mut common_ancestors_vv = self.dag.frontiers_to_vv(&common_ancestors).unwrap(); + let shallow_since_vv = self.dag.shallow_since_vv().to_vv(); + if !common_ancestors_vv.includes_vv(&shallow_since_vv) { + // The replay base cannot point before shallow history because those + // ops are no longer available to the causal iterator. + common_ancestors = self.dag.shallow_since_frontiers().clone(); + common_ancestors_vv = self + .dag + .frontiers_to_vv(&common_ancestors) + .unwrap_or(shallow_since_vv); + } + // go from lca to merged_vv + let diff = common_ancestors_vv.diff(&merged_vv).forward; + let mut iter = self.dag.iter_causal(common_ancestors, diff); + let mut node = iter.next(); + let mut cur_cnt = 0; + let vv = Rc::new(RefCell::new(VersionVector::default())); + ( + common_ancestors_vv.clone(), + diff_mode, + std::iter::from_fn(move || { + if let Some(inner) = &node { + let mut inner_vv = vv.borrow_mut(); + // FIXME: PERF: it looks slow for large vv, like 10000+ entries + inner_vv.clear(); + self.dag.ensure_vv_for(&inner.data); + inner_vv.extend_to_include_vv(inner.data.vv.get().unwrap().iter()); + let peer = inner.data.peer; + let cnt = inner + .data + .cnt + .max(cur_cnt) + .max(common_ancestors_vv.get(&peer).copied().unwrap_or(0)); + let dag_node_end = (inner.data.cnt + inner.data.len as Counter) + .min(merged_vv.get(&peer).copied().unwrap_or(0)); + let change = self.change_store.get_change(ID::new(peer, cnt)).unwrap(); + + if change.ctr_end() < dag_node_end { + cur_cnt = change.ctr_end(); + } else { + node = iter.next(); + cur_cnt = 0; + } + + inner_vv.extend_to_include_end_id(change.id); + + Some((change, (cnt, dag_node_end), vv.clone())) + } else { + None + } + }), + ) + } + + pub fn len_changes(&self) -> usize { + self.change_store.change_num() + } + + pub fn diagnose_size(&self) -> SizeInfo { + let mut total_changes = 0; + let mut total_ops = 0; + let mut total_atom_ops = 0; + let total_dag_node = self.dag.total_parsed_dag_node(); + self.change_store.visit_all_changes(&mut |change| { + total_changes += 1; + total_ops += change.ops.len(); + total_atom_ops += change.atom_len(); + }); + + println!("total changes: {}", total_changes); + println!("total ops: {}", total_ops); + println!("total atom ops: {}", total_atom_ops); + println!("total dag node: {}", total_dag_node); + SizeInfo { + total_changes, + total_ops, + total_atom_ops, + total_dag_node, + } + } + + pub(crate) fn iter_changes_peer_by_peer<'a>( + &'a self, + from: &VersionVector, + to: &VersionVector, + ) -> impl Iterator + 'a { + let spans: Vec<_> = from.diff_iter(to).1.collect(); + spans + .into_iter() + .flat_map(move |span| self.change_store.iter_changes(span)) + } + + #[allow(dead_code)] + pub(crate) fn iter_changes_causally_rev<'a>( + &'a self, + from: &VersionVector, + to: &VersionVector, + ) -> impl Iterator + 'a { + MergedChangeIter::new_change_iter_rev(self, from, to) + } + + pub fn get_timestamp_for_next_txn(&self) -> Timestamp { + if self.configure.record_timestamp() { + get_timestamp_now_txn() + } else { + 0 + } + } + + #[inline(never)] + pub(crate) fn idlp_to_id(&self, id: loro_common::IdLp) -> Option { + let change = self.change_store.get_change_by_lamport_lte(id)?; + + if change.lamport > id.lamport || change.lamport_end() <= id.lamport { + return None; + } + + Some(ID::new( + change.id.peer, + (id.lamport - change.lamport) as Counter + change.id.counter, + )) + } + + #[allow(unused)] + pub(crate) fn id_to_idlp(&self, id_start: ID) -> IdLp { + let change = self.get_change_at(id_start).unwrap(); + let lamport = change.lamport + (id_start.counter - change.id.counter) as Lamport; + let peer = id_start.peer; + loro_common::IdLp { peer, lamport } + } + + /// NOTE: This may return a op that includes the given id, not necessarily start with the given id + pub(crate) fn get_op_that_includes(&self, id: ID) -> Option { + let change = self.get_change_at(id)?; + change.get_op_with_counter(id.counter) + } + + pub(crate) fn split_span_based_on_deps(&self, id_span: IdSpan) -> Vec<(IdSpan, Frontiers)> { + let peer = id_span.peer; + let mut counter = id_span.counter.min(); + let span_end = id_span.counter.norm_end(); + let mut ans = Vec::new(); + + while counter < span_end { + let id = ID::new(peer, counter); + let node = self.dag.get(id).unwrap(); + + let f = if node.cnt == counter { + node.deps.clone() + } else if counter > 0 { + id.inc(-1).into() + } else { + unreachable!() + }; + + let cur_end = node.cnt + node.len as Counter; + let len = cur_end.min(span_end) - counter; + ans.push((id.to_span(len as usize), f)); + counter += len; + } + + ans + } + + #[inline] + pub fn compact_change_store(&mut self) { + self.change_store + .flush_and_compact(self.dag.vv(), self.dag.frontiers()); + } + + #[inline] + pub fn change_store_kv_size(&self) -> usize { + self.change_store.kv_size() + } + + pub fn encode_change_store(&self) -> bytes::Bytes { + self.change_store + .encode_all(self.dag.vv(), self.dag.frontiers()) + } + + pub fn check_dag_correctness(&self) { + self.dag.check_dag_correctness(); + } + + pub fn shallow_since_vv(&self) -> &ImVersionVector { + self.dag.shallow_since_vv() + } + + pub fn shallow_since_frontiers(&self) -> &Frontiers { + self.dag.shallow_since_frontiers() + } + + pub fn is_shallow(&self) -> bool { + !self.dag.shallow_since_vv().is_empty() + } + + pub fn get_greatest_timestamp(&self, frontiers: &Frontiers) -> Timestamp { + let mut max_timestamp = Timestamp::default(); + for id in frontiers.iter() { + let change = self.get_change_at(id).unwrap(); + if change.timestamp > max_timestamp { + max_timestamp = change.timestamp; + } + } + + max_timestamp + } +} + +#[derive(Debug)] +pub struct SizeInfo { + pub total_changes: usize, + pub total_ops: usize, + pub total_atom_ops: usize, + pub total_dag_node: usize, +} + +pub(crate) fn convert_change_to_remote( + arena: &SharedArena, + change: &Change, +) -> Change> { + let mut ops = RleVec::new(); + for op in change.ops.iter() { + for op in local_op_to_remote(arena, op) { + ops.push(op); + } + } + + Change { + ops, + id: change.id, + deps: change.deps.clone(), + lamport: change.lamport, + timestamp: change.timestamp, + commit_msg: change.commit_msg.clone(), + } +} + +pub(crate) fn local_op_to_remote( + arena: &SharedArena, + op: &crate::op::Op, +) -> SmallVec<[RemoteOp<'static>; 1]> { + let container = arena.get_container_id(op.container).unwrap(); + let mut contents: SmallVec<[_; 1]> = SmallVec::new(); + match &op.content { + crate::op::InnerContent::List(list) => match list { + list_op::InnerListOp::Insert { slice, pos } => match container.container_type() { + loro_common::ContainerType::Text => { + let str = arena + .slice_str_by_unicode_range(slice.0.start as usize..slice.0.end as usize); + contents.push(RawOpContent::List(list_op::ListOp::Insert { + slice: ListSlice::RawStr { + unicode_len: str.chars().count(), + str: Cow::Owned(str), + }, + pos: *pos, + })); + } + loro_common::ContainerType::List | loro_common::ContainerType::MovableList => { + contents.push(RawOpContent::List(list_op::ListOp::Insert { + slice: ListSlice::RawData(Cow::Owned( + arena.get_values(slice.0.start as usize..slice.0.end as usize), + )), + pos: *pos, + })) + } + _ => unreachable!(), + }, + list_op::InnerListOp::InsertText { + slice, + unicode_len: len, + unicode_start: _, + pos, + } => match container.container_type() { + loro_common::ContainerType::Text => { + contents.push(RawOpContent::List(list_op::ListOp::Insert { + slice: ListSlice::RawStr { + unicode_len: *len as usize, + str: Cow::Owned(std::str::from_utf8(slice).unwrap().to_owned()), + }, + pos: *pos as usize, + })); + } + _ => unreachable!(), + }, + list_op::InnerListOp::Delete(del) => { + contents.push(RawOpContent::List(list_op::ListOp::Delete(*del))) + } + list_op::InnerListOp::StyleStart { + start, + end, + key, + value, + info, + } => contents.push(RawOpContent::List(list_op::ListOp::StyleStart { + start: *start, + end: *end, + key: key.clone(), + value: value.clone(), + info: *info, + })), + list_op::InnerListOp::StyleEnd => { + contents.push(RawOpContent::List(list_op::ListOp::StyleEnd)) + } + list_op::InnerListOp::Move { + from, + elem_id: from_id, + to, + } => contents.push(RawOpContent::List(list_op::ListOp::Move { + from: *from, + elem_id: *from_id, + to: *to, + })), + list_op::InnerListOp::Set { elem_id, value } => { + contents.push(RawOpContent::List(list_op::ListOp::Set { + elem_id: *elem_id, + value: value.clone(), + })) + } + }, + crate::op::InnerContent::Map(map) => { + let value = map.value.clone(); + contents.push(RawOpContent::Map(crate::container::map::MapSet { + key: map.key.clone(), + value, + })) + } + crate::op::InnerContent::Tree(tree) => contents.push(RawOpContent::Tree(tree.clone())), + crate::op::InnerContent::Future(f) => match f { + #[cfg(feature = "counter")] + crate::op::FutureInnerContent::Counter(c) => contents.push(RawOpContent::Counter(*c)), + FutureInnerContent::Unknown { prop, value } => { + contents.push(crate::op::RawOpContent::Unknown { + prop: *prop, + value: (**value).clone(), + }) + } + }, + }; + + let mut ans = SmallVec::with_capacity(contents.len()); + for content in contents { + ans.push(RemoteOp { + container: container.clone(), + content, + counter: op.counter, + }) + } + ans +} + +pub(crate) fn get_timestamp_now_txn() -> Timestamp { + (get_sys_timestamp() as Timestamp + 500) / 1000 +} + +#[cfg(test)] +mod visible_op_count_tests { + use crate::{cursor::PosType, loro::ExportMode, LoroDoc}; + + /// The cached `visible_op_count` (bumped incrementally for local ops, and + /// the only value read in release builds where `can_lock_in_this_thread` + /// returns false) must always equal a from-scratch recompute. + #[test] + fn cached_visible_op_count_matches_exact() { + let doc = LoroDoc::new(); + let text = doc.get_text("text"); + let mut txn = doc.txn().unwrap(); + for i in 0..50 { + text.insert_with_txn(&mut txn, i, "a", PosType::Unicode) + .unwrap(); + } + txn.commit().unwrap(); + { + let oplog = doc.oplog().lock(); + assert_eq!( + oplog.cached_visible_op_count(), + oplog.visible_op_count_exact(), + "after local edits" + ); + } + + // Import keeps the cached count exact via full refresh; subsequent local + // edits then increment from that exact base. + let doc2 = LoroDoc::new(); + doc2.import(&doc.export(ExportMode::all_updates()).unwrap()) + .unwrap(); + let text2 = doc2.get_text("text"); + let mut txn2 = doc2.txn().unwrap(); + text2 + .insert_with_txn(&mut txn2, 0, "bbb", PosType::Unicode) + .unwrap(); + txn2.commit().unwrap(); + { + let oplog = doc2.oplog().lock(); + assert_eq!( + oplog.cached_visible_op_count(), + oplog.visible_op_count_exact(), + "after import + local edits" + ); + } + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/oplog/change_store.rs b/3rdparty/loro/crates/loro-internal/src/oplog/change_store.rs new file mode 100644 index 0000000000..2bf6d6bd47 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/oplog/change_store.rs @@ -0,0 +1,1980 @@ +use self::block_encode::{decode_block, decode_header, encode_block, ChangesBlockHeader}; +use super::{loro_dag::AppDagNodeInner, AppDagNode}; +use crate::sync::Mutex; +use crate::{ + arena::SharedArena, + change::Change, + estimated_size::EstimatedSize, + kv_store::KvStore, + op::Op, + parent::register_container_and_parent_link, + version::{Frontiers, ImVersionVector}, + VersionVector, +}; +use block_encode::decode_block_range; +use bytes::Bytes; +use itertools::Itertools; +use loro_common::{ + Counter, HasCounterSpan, HasId, HasIdSpan, HasLamportSpan, IdLp, IdSpan, Lamport, LoroError, + LoroResult, PeerID, ID, +}; +use loro_kv_store::{mem_store::MemKvConfig, MemKvStore}; +use once_cell::sync::OnceCell; +use rle::{HasLength, Mergable, RlePush, RleVec, Sliceable}; +use std::sync::atomic::AtomicI64; +use std::{ + cmp::Ordering, + collections::{BTreeMap, VecDeque}, + ops::{Bound, Deref}, + sync::Arc, +}; +use tracing::{info_span, warn}; +mod block_encode; +mod block_meta_encode; +pub(super) mod iter; + +#[cfg(not(test))] +const MAX_BLOCK_SIZE: usize = 1024 * 4; +#[cfg(test)] +const MAX_BLOCK_SIZE: usize = 128; + +/// # Invariance +/// +/// - We don't allow holes in a block or between two blocks with the same peer id. +/// The [Change] should be continuous for each peer. +/// - However, the first block of a peer can have counter > 0 so that we can trim the history. +/// +/// # Encoding Schema +/// +/// It's based on the underlying KV store. +/// +/// The entries of the KV store is made up of the following fields +/// +/// |Key |Value | +/// |:-- |:---- | +/// |b"vv" |VersionVector | +/// |b"fr" |Frontiers | +/// |b"sv" |Shallow VV | +/// |b"sf" |Shallow Frontiers | +/// |12 bytes PeerID + Counter |Encoded Block | +#[derive(Debug, Clone)] +pub struct ChangeStore { + inner: Arc>, + arena: SharedArena, + /// A change may be in external_kv or in the mem_parsed_kv. + /// mem_parsed_kv is more up-to-date. + /// + /// We cannot directly write into the external_kv except from the initial load + external_kv: Arc>, + /// The version vector of the external kv store. + external_vv: Arc>, + merge_interval: Arc, +} + +#[derive(Debug, Clone)] +struct ChangeStoreInner { + /// The start version vector of the first block for each peer. + /// It allows us to trim the history + start_vv: ImVersionVector, + /// The last version of the shallow history. + start_frontiers: Frontiers, + /// It's more like a parsed cache for binary_kv. + mem_parsed_kv: BTreeMap>, +} + +#[derive(Debug)] +pub(crate) struct ChangeStoreRollback { + old_vv: VersionVector, + blocks_before_mutation: BTreeMap>, +} + +impl ChangeStoreRollback { + pub(crate) fn new(old_vv: VersionVector) -> Self { + Self { + old_vv, + blocks_before_mutation: BTreeMap::new(), + } + } + + fn record_block_before_mutation(&mut self, id: ID, block: Arc) { + let old_end = self.old_vv.get(&id.peer).copied().unwrap_or(0); + if id.counter >= old_end { + return; + } + + self.blocks_before_mutation.entry(id).or_insert(block); + } +} + +#[derive(Debug, Clone)] +pub(crate) struct ChangesBlock { + peer: PeerID, + counter_range: (Counter, Counter), + lamport_range: (Lamport, Lamport), + /// Estimated size of the block in bytes + estimated_size: usize, + flushed: bool, + content: ChangesBlockContent, +} + +#[derive(Clone)] +pub(crate) enum ChangesBlockContent { + Changes(Arc>), + Bytes(ChangesBlockBytes), + Both(Arc>, ChangesBlockBytes), +} + +/// It's cheap to clone this struct because it's cheap to clone the bytes +#[derive(Clone)] +pub(crate) struct ChangesBlockBytes { + bytes: Bytes, + header: OnceCell>, +} + +pub const START_VV_KEY: &[u8] = b"sv"; +pub const START_FRONTIERS_KEY: &[u8] = b"sf"; +pub const VV_KEY: &[u8] = b"vv"; +pub const FRONTIERS_KEY: &[u8] = b"fr"; + +impl ChangeStore { + pub fn new_mem(a: &SharedArena, merge_interval: Arc) -> Self { + Self { + inner: Arc::new(Mutex::new(ChangeStoreInner { + start_vv: ImVersionVector::new(), + start_frontiers: Frontiers::default(), + mem_parsed_kv: BTreeMap::new(), + })), + arena: a.clone(), + external_vv: Arc::new(Mutex::new(VersionVector::new())), + external_kv: Arc::new(Mutex::new(MemKvStore::new(MemKvConfig::default()))), + // external_kv: Arc::new(Mutex::new(BTreeMap::default())), + merge_interval, + } + } + + #[cfg(test)] + fn new_for_test() -> Self { + Self::new_mem(&SharedArena::new(), Arc::new(AtomicI64::new(0))) + } + + pub(super) fn encode_all(&self, vv: &VersionVector, frontiers: &Frontiers) -> Bytes { + self.flush_and_compact(vv, frontiers); + let mut kv = self.external_kv.lock(); + kv.export_all() + } + + #[tracing::instrument(skip(self), level = "debug")] + pub(super) fn export_from( + &self, + start_vv: &VersionVector, + start_frontiers: &Frontiers, + latest_vv: &VersionVector, + latest_frontiers: &Frontiers, + ) -> Bytes { + let new_store = ChangeStore::new_mem(&self.arena, self.merge_interval.clone()); + for span in latest_vv.sub_iter(start_vv) { + // PERF: this can be optimized by reusing the current encoded blocks + // In the current method, it needs to parse and re-encode the blocks + for c in self.iter_changes(span) { + let start = ((start_vv.get(&c.id.peer).copied().unwrap_or(0) - c.id.counter).max(0) + as usize) + .min(c.atom_len()); + let end = ((latest_vv.get(&c.id.peer).copied().unwrap_or(0) - c.id.counter).max(0) + as usize) + .min(c.atom_len()); + + if start == end { + continue; + } + + let ch = c.slice(start, end); + new_store.insert_change(ch, false, false); + } + } + + loro_common::debug!( + "start_vv={:?} start_frontiers={:?}", + &start_vv, + start_frontiers + ); + new_store.encode_from(start_vv, start_frontiers, latest_vv, latest_frontiers) + } + + pub(super) fn export_blocks_in_range(&self, spans: &[IdSpan], w: &mut W) { + let new_store = ChangeStore::new_mem(&self.arena, self.merge_interval.clone()); + for span in spans { + let mut span = *span; + span.normalize_(); + if span.counter.end <= 0 { + continue; + } + + span.counter.start = span.counter.start.max(0); + span.counter.end = span.counter.end.max(0); + if span.counter.start >= span.counter.end { + continue; + } + + // PERF: this can be optimized by reusing the current encoded blocks + // In the current method, it needs to parse and re-encode the blocks + for c in self.iter_changes(span) { + let start = ((span.counter.start - c.id.counter).max(0) as usize).min(c.atom_len()); + let end = ((span.counter.end - c.id.counter).max(0) as usize).min(c.atom_len()); + if start == end { + continue; + } + + let ch = c.slice(start, end); + new_store.insert_change(ch, false, false); + } + } + + encode_blocks_in_store(new_store, &self.arena, w); + } + + fn encode_from( + &self, + start_vv: &VersionVector, + start_frontiers: &Frontiers, + latest_vv: &VersionVector, + latest_frontiers: &Frontiers, + ) -> Bytes { + { + let mut store = self.external_kv.lock(); + store.set(START_VV_KEY, start_vv.encode().into()); + store.set(START_FRONTIERS_KEY, start_frontiers.encode().into()); + let mut inner = self.inner.lock(); + inner.start_frontiers = start_frontiers.clone(); + inner.start_vv = ImVersionVector::from_vv(start_vv); + } + self.flush_and_compact(latest_vv, latest_frontiers); + self.external_kv.lock().export_all() + } + + pub(crate) fn decode_snapshot_for_updates( + bytes: Bytes, + arena: &SharedArena, + self_vv: &VersionVector, + ) -> Result, LoroError> { + let change_store = ChangeStore::new_mem(arena, Arc::new(AtomicI64::new(0))); + let _ = change_store.import_all(bytes)?; + let mut changes = Vec::new(); + change_store.visit_all_changes(&mut |c| { + let cnt_threshold = self_vv.get(&c.id.peer).copied().unwrap_or(0); + if c.id.counter >= cnt_threshold { + changes.push(c.clone()); + return; + } + + let change_end = c.ctr_end(); + if change_end > cnt_threshold { + changes.push(c.slice((cnt_threshold - c.id.counter) as usize, c.atom_len())); + } + }); + + Ok(changes) + } + + pub(crate) fn decode_block_bytes( + bytes: Bytes, + arena: &SharedArena, + self_vv: &VersionVector, + ) -> LoroResult> { + let mut ans = ChangesBlockBytes::new(bytes).parse(arena)?; + if ans.is_empty() { + return Ok(ans); + } + + let start = self_vv.get(&ans[0].peer()).copied().unwrap_or(0); + ans.retain_mut(|c| { + if c.id.counter >= start { + true + } else if c.ctr_end() > start { + *c = c.slice((start - c.id.counter) as usize, c.atom_len()); + true + } else { + false + } + }); + + Ok(ans) + } + + pub(crate) fn rollback_import(&self, rollback: ChangeStoreRollback) { + let mut inner = self.inner.lock(); + inner.mem_parsed_kv.retain(|id, _| { + let old_end = rollback.old_vv.get(&id.peer).copied().unwrap_or(0); + id.counter < old_end + }); + + for (id, block) in rollback.blocks_before_mutation { + inner.mem_parsed_kv.insert(id, block); + } + } + + pub fn get_dag_nodes_that_contains(&self, id: ID) -> Option> { + let block = self.get_block_that_contains(id)?; + Some(block.content.iter_dag_nodes()) + } + + pub fn get_last_dag_nodes_for_peer(&self, peer: PeerID) -> Option> { + let block = self.get_the_last_block_of_peer(peer)?; + Some(block.content.iter_dag_nodes()) + } + + pub fn visit_all_changes(&self, f: &mut dyn FnMut(&Change)) { + self.ensure_block_loaded_in_range(Bound::Unbounded, Bound::Unbounded); + let mut inner = self.inner.lock(); + for (id, block) in inner.mem_parsed_kv.iter_mut() { + if let Err(err) = block.ensure_changes(&self.arena) { + warn!(block_id = ?id, ?err, "failed to parse change block"); + continue; + } + for c in block.content.try_changes().unwrap() { + f(c); + } + } + } + + pub(crate) fn iter_blocks(&self, id_span: IdSpan) -> Vec<(Arc, usize, usize)> { + if id_span.counter.start == id_span.counter.end { + return vec![]; + } + + assert!(id_span.counter.start < id_span.counter.end); + self.ensure_block_loaded_in_range( + Bound::Included(id_span.id_start()), + Bound::Excluded(id_span.id_end()), + ); + let mut inner = self.inner.lock(); + let next_back = inner.mem_parsed_kv.range(..=id_span.id_start()).next_back(); + match next_back { + None => { + return vec![]; + } + Some(next_back) => { + if next_back.0.peer != id_span.peer { + return vec![]; + } + } + } + let start_counter = next_back.map(|(id, _)| id.counter).unwrap_or(0); + let ans = inner + .mem_parsed_kv + .range_mut( + ID::new(id_span.peer, start_counter)..ID::new(id_span.peer, id_span.counter.end), + ) + .filter_map(|(_id, block)| { + if block.counter_range.1 < id_span.counter.start { + return None; + } + + if let Err(err) = block.ensure_changes(&self.arena) { + warn!(block_id = ?_id, ?err, "failed to parse change block"); + return None; + } + let changes = block.content.try_changes().unwrap(); + let start; + let end; + if id_span.counter.start <= block.counter_range.0 + && id_span.counter.end >= block.counter_range.1 + { + start = 0; + end = changes.len(); + } else { + start = block + .get_change_index_by_counter(id_span.counter.start) + .unwrap_or_else(|x| x); + + match block.get_change_index_by_counter(id_span.counter.end - 1) { + Ok(e) => { + end = e + 1; + } + Err(0) => return None, + Err(e) => { + end = e; + } + } + } + if start == end { + return None; + } + + Some((block.clone(), start, end)) + }) + // TODO: PERF avoid alloc + .collect_vec(); + + ans + } + + pub fn iter_changes(&self, id_span: IdSpan) -> impl Iterator + '_ { + let v = self.iter_blocks(id_span); + #[cfg(debug_assertions)] + { + if !v.is_empty() { + assert_eq!(v[0].0.peer, id_span.peer); + assert_eq!(v.last().unwrap().0.peer, id_span.peer); + { + // Test start + let (block, start, _end) = v.first().unwrap(); + let changes = block.content.try_changes().unwrap(); + assert!(changes[*start].id.counter <= id_span.counter.start); + } + { + // Test end + let (block, _start, end) = v.last().unwrap(); + let changes = block.content.try_changes().unwrap(); + assert!(changes[*end - 1].ctr_end() >= id_span.counter.end); + assert!(changes[*end - 1].id.counter < id_span.counter.end); + } + } + } + + v.into_iter().flat_map(move |(block, start, end)| { + (start..end).map(move |i| BlockChangeRef { + change_index: i, + block: block.clone(), + }) + }) + } + + #[allow(dead_code)] + pub(crate) fn get_blocks_in_range(&self, id_span: IdSpan) -> VecDeque> { + let mut inner = self.inner.lock(); + let start_counter = inner + .mem_parsed_kv + .range(..=id_span.id_start()) + .next_back() + .map(|(id, _)| id.counter) + .unwrap_or(0); + let vec = inner + .mem_parsed_kv + .range_mut( + ID::new(id_span.peer, start_counter)..ID::new(id_span.peer, id_span.counter.end), + ) + .filter_map(|(_id, block)| { + if block.counter_range.1 < id_span.counter.start { + return None; + } + + if let Err(err) = block.ensure_changes(&self.arena) { + warn!(block_id = ?_id, ?err, "failed to parse change block"); + return None; + } + Some(block.clone()) + }) + // TODO: PERF avoid alloc + .collect(); + vec + } + + pub(crate) fn get_block_that_contains(&self, id: ID) -> Option> { + self.ensure_block_loaded_in_range(Bound::Included(id), Bound::Included(id)); + let inner = self.inner.lock(); + let block = inner + .mem_parsed_kv + .range(..=id) + .next_back() + .filter(|(_, block)| { + block.peer == id.peer + && block.counter_range.0 <= id.counter + && id.counter < block.counter_range.1 + }) + .map(|(_, block)| block.clone()); + + block + } + + pub(crate) fn get_the_last_block_of_peer(&self, peer: PeerID) -> Option> { + let end_id = ID::new(peer, Counter::MAX); + self.ensure_id_lte(end_id); + let inner = self.inner.lock(); + let block = inner + .mem_parsed_kv + .range(..=end_id) + .next_back() + .filter(|(_, block)| block.peer == peer) + .map(|(_, block)| block.clone()); + + block + } + + pub fn change_num(&self) -> usize { + self.ensure_block_loaded_in_range(Bound::Unbounded, Bound::Unbounded); + let mut inner = self.inner.lock(); + inner + .mem_parsed_kv + .iter_mut() + .map(|(_, block)| block.change_num()) + .sum() + } + + pub fn fork( + &self, + arena: SharedArena, + merge_interval: Arc, + vv: &VersionVector, + frontiers: &Frontiers, + ) -> Self { + self.flush_and_compact(vv, frontiers); + let inner = self.inner.lock(); + Self { + inner: Arc::new(Mutex::new(ChangeStoreInner { + start_vv: inner.start_vv.clone(), + start_frontiers: inner.start_frontiers.clone(), + mem_parsed_kv: BTreeMap::new(), + })), + arena, + external_vv: Arc::new(Mutex::new(self.external_vv.lock().clone())), + external_kv: self.external_kv.lock().clone_store(), + merge_interval, + } + } + + pub fn kv_size(&self) -> usize { + self.external_kv + .lock() + .scan(Bound::Unbounded, Bound::Unbounded) + .map(|(k, v)| k.len() + v.len()) + .sum() + } + + pub(crate) fn export_blocks_from( + &self, + start_vv: &VersionVector, + shallow_since_vv: &ImVersionVector, + latest_vv: &VersionVector, + w: &mut W, + ) { + let new_store = ChangeStore::new_mem(&self.arena, self.merge_interval.clone()); + for mut span in latest_vv.sub_iter(start_vv) { + let counter_lower_bound = shallow_since_vv.get(&span.peer).copied().unwrap_or(0); + span.counter.start = span.counter.start.max(counter_lower_bound); + span.counter.end = span.counter.end.max(counter_lower_bound); + if span.counter.start >= span.counter.end { + continue; + } + + // PERF: this can be optimized by reusing the current encoded blocks + // In the current method, it needs to parse and re-encode the blocks + for c in self.iter_changes(span) { + let start = ((start_vv.get(&c.id.peer).copied().unwrap_or(0) - c.id.counter).max(0) + as usize) + .min(c.atom_len()); + let end = ((latest_vv.get(&c.id.peer).copied().unwrap_or(0) - c.id.counter).max(0) + as usize) + .min(c.atom_len()); + + assert_ne!(start, end); + let ch = c.slice(start, end); + new_store.insert_change(ch, false, false); + } + } + + let arena = &self.arena; + encode_blocks_in_store(new_store, arena, w); + } + + pub(crate) fn fork_changes_up_to( + &self, + start_vv: &ImVersionVector, + frontiers: &Frontiers, + vv: &VersionVector, + ) -> Bytes { + let new_store = ChangeStore::new_mem(&self.arena, self.merge_interval.clone()); + for mut span in vv.sub_iter_im(start_vv) { + let counter_lower_bound = start_vv.get(&span.peer).copied().unwrap_or(0); + span.counter.start = span.counter.start.max(counter_lower_bound); + span.counter.end = span.counter.end.max(counter_lower_bound); + if span.counter.start >= span.counter.end { + continue; + } + + // PERF: this can be optimized by reusing the current encoded blocks + // In the current method, it needs to parse and re-encode the blocks + for c in self.iter_changes(span) { + let start = ((start_vv.get(&c.id.peer).copied().unwrap_or(0) - c.id.counter).max(0) + as usize) + .min(c.atom_len()); + let end = ((vv.get(&c.id.peer).copied().unwrap_or(0) - c.id.counter).max(0) + as usize) + .min(c.atom_len()); + + assert_ne!(start, end); + let ch = c.slice(start, end); + new_store.insert_change(ch, false, false); + } + } + + new_store.encode_all(vv, frontiers) + } +} + +fn encode_blocks_in_store( + new_store: ChangeStore, + arena: &SharedArena, + w: &mut W, +) { + let mut inner = new_store.inner.lock(); + for (_id, block) in inner.mem_parsed_kv.iter_mut() { + let bytes = block.to_bytes(arena); + leb128::write::unsigned(w, bytes.bytes.len() as u64).unwrap(); + w.write_all(&bytes.bytes).unwrap(); + } +} + +mod mut_external_kv { + //! Only this module contains the code that mutate the external kv store + //! All other modules should only read from the external kv store + use super::*; + + impl ChangeStore { + #[tracing::instrument(skip_all, level = "debug", name = "change_store import_all")] + pub(crate) fn import_all(&self, bytes: Bytes) -> Result { + let mut kv_store = self.external_kv.lock(); + assert!( + // 2 because there are vv and frontiers + kv_store.len() <= 2, + "kv store should be empty when using decode_all" + ); + // The snapshot/update body is already integrity-checked by the + // document-level checksum in `parse_header_and_body(.., true)` before + // we reach here, so skip the redundant per-block checksum. + kv_store + .import_all_unchecked(bytes) + .map_err(|e| LoroError::DecodeError(e.into_boxed_str()))?; + drop(kv_store); + let vv_bytes = self.external_kv.lock().get(VV_KEY).unwrap_or_default(); + let vv = VersionVector::decode(&vv_bytes) + .map_err(|_| LoroError::DecodeDataCorruptionError)?; + let start_vv_bytes = self + .external_kv + .lock() + .get(START_VV_KEY) + .unwrap_or_default(); + let start_vv = if start_vv_bytes.is_empty() { + Default::default() + } else { + VersionVector::decode(&start_vv_bytes) + .map_err(|_| LoroError::DecodeDataCorruptionError)? + }; + + #[cfg(test)] + { + // This is for tests + for (peer, cnt) in vv.iter() { + self.get_change(ID::new(*peer, *cnt - 1)) + .ok_or(LoroError::DecodeDataCorruptionError)?; + } + } + + *self.external_vv.lock() = vv.clone(); + let frontiers_bytes = self + .external_kv + .lock() + .get(FRONTIERS_KEY) + .unwrap_or_default(); + let frontiers = Frontiers::decode(&frontiers_bytes) + .map_err(|_| LoroError::DecodeDataCorruptionError)?; + let start_frontiers = self + .external_kv + .lock() + .get(START_FRONTIERS_KEY) + .unwrap_or_default(); + let start_frontiers = if start_frontiers.is_empty() { + Default::default() + } else { + Frontiers::decode(&start_frontiers) + .map_err(|_| LoroError::DecodeDataCorruptionError)? + }; + + let mut max_lamport = None; + let mut max_timestamp = 0; + for id in frontiers.iter() { + let c = self + .get_change(id) + .ok_or(LoroError::DecodeDataCorruptionError)?; + debug_assert_ne!(c.atom_len(), 0); + let l = c.lamport_last(); + if let Some(x) = max_lamport { + if l > x { + max_lamport = Some(l); + } + } else { + max_lamport = Some(l); + } + + let t = c.timestamp; + if t > max_timestamp { + max_timestamp = t; + } + } + + Ok(BatchDecodeInfo { + vv, + frontiers, + start_version: if start_vv.is_empty() { + None + } else { + let mut inner = self.inner.lock(); + inner.start_frontiers = start_frontiers.clone(); + inner.start_vv = ImVersionVector::from_vv(&start_vv); + Some((start_vv, start_frontiers)) + }, + }) + } + + /// Flush the cached change to kv_store + pub(crate) fn flush_and_compact(&self, vv: &VersionVector, frontiers: &Frontiers) { + let mut inner = self.inner.lock(); + let mut store = self.external_kv.lock(); + let mut external_vv = self.external_vv.lock(); + for (id, block) in inner.mem_parsed_kv.iter_mut() { + if !block.flushed { + let id_bytes = id.to_bytes(); + let counter_start = external_vv.get(&id.peer).copied().unwrap_or(0); + assert!( + counter_start < block.counter_range.1, + "Peer={} Block Counter Range={:?}, counter_start={}", + id.peer, + &block.counter_range, + counter_start + ); + if counter_start > block.counter_range.0 { + assert!(store.get(&id_bytes).is_some()); + } + external_vv.insert(id.peer, block.counter_range.1); + let bytes = block.to_bytes(&self.arena); + store.set(&id_bytes, bytes.bytes); + Arc::make_mut(block).flushed = true; + } + } + + if inner.start_vv.is_empty() { + assert_eq!(&*external_vv, vv); + } else { + #[cfg(debug_assertions)] + { + // TODO: makes some assertions here? + } + } + let vv_bytes = vv.encode(); + let frontiers_bytes = frontiers.encode(); + store.set(VV_KEY, vv_bytes.into()); + store.set(FRONTIERS_KEY, frontiers_bytes.into()); + } + } +} + +mod mut_inner_kv { + //! Only this module contains the code that mutate the internal kv store + //! All other modules should only read from the internal kv store + + use super::*; + impl ChangeStore { + /// This method is the **only place** that push a new change into the change store + /// + /// The new change either merges with the previous block or is put into a new block. + /// This method only updates the internal kv store. + pub fn insert_change(&self, change: Change, split_when_exceeds: bool, is_local: bool) { + self.insert_change_inner(change, split_when_exceeds, is_local, None); + } + + pub(crate) fn insert_change_with_rollback( + &self, + change: Change, + split_when_exceeds: bool, + is_local: bool, + rollback: &mut ChangeStoreRollback, + ) { + self.insert_change_inner(change, split_when_exceeds, is_local, Some(rollback)); + } + + fn insert_change_inner( + &self, + mut change: Change, + split_when_exceeds: bool, + is_local: bool, + mut rollback: Option<&mut ChangeStoreRollback>, + ) { + #[cfg(debug_assertions)] + { + let vv = self.external_vv.lock(); + assert!(vv.get(&change.id.peer).copied().unwrap_or(0) <= change.id.counter); + } + + let s = info_span!("change_store insert_change", id = ?change.id); + let _e = s.enter(); + let estimated_size = change.estimate_storage_size(); + if estimated_size > MAX_BLOCK_SIZE && split_when_exceeds { + self.split_change_then_insert(change, rollback.as_deref_mut()); + return; + } + + let id = change.id; + let mut inner = self.inner.lock(); + + // try to merge with previous block + if let Some((_id, block)) = inner.mem_parsed_kv.range_mut(..id).next_back() { + if block.peer == change.id.peer { + if block.counter_range.1 != change.id.counter { + panic!("counter should be continuous") + } + + if let Some(rollback) = &mut rollback { + rollback.record_block_before_mutation(*_id, block.clone()); + } + + match block.push_change( + change, + estimated_size, + if is_local { + // local change should try to merge with previous change when + // the timestamp interval <= the `merge_interval` + self.merge_interval + .load(std::sync::atomic::Ordering::Acquire) + } else { + 0 + }, + &self.arena, + ) { + Ok(_) => { + drop(inner); + debug_assert!(self.get_change(id).is_some()); + return; + } + Err(c) => change = c, + } + } + } + + inner + .mem_parsed_kv + .insert(id, Arc::new(ChangesBlock::new(change, &self.arena))); + drop(inner); + debug_assert!(self.get_change(id).is_some()); + } + + pub fn get_change(&self, id: ID) -> Option { + let block = self.get_parsed_block(id)?; + Some(BlockChangeRef { + change_index: block.get_change_index_by_counter(id.counter).unwrap(), + block: block.clone(), + }) + } + + /// Get the change with the given peer and lamport. + /// + /// If not found, return the change with the greatest lamport that is smaller than the given lamport. + pub fn get_change_by_lamport_lte(&self, idlp: IdLp) -> Option { + // This method is complicated because we impl binary search on top of the range api + // It can be simplified + let mut inner = self.inner.lock(); + let mut iter = inner + .mem_parsed_kv + .range_mut(ID::new(idlp.peer, 0)..ID::new(idlp.peer, i32::MAX)); + + // This won't change, we only adjust upper_bound + let mut lower_bound = 0; + let mut upper_bound = i32::MAX; + let mut is_binary_searching = false; + // The binary search below can stop making progress (e.g. when + // `(lower_bound + upper_bound) / 2` becomes a fixed point while the + // target block is missing from `mem_parsed_kv`, or when block + // metadata is inconsistent). Cap its steps and fall back to the + // external kv scan, which is always correct. + let mut binary_search_steps = 0; + loop { + if is_binary_searching { + binary_search_steps += 1; + if binary_search_steps > 128 { + warn!( + "get_change_by_lamport_lte binary search did not converge; \ + falling back to kv scan" + ); + break; + } + } + match iter.next_back() { + Some((&id, block)) => { + if block.lamport_range.0 <= idlp.lamport + && (!is_binary_searching || idlp.lamport < block.lamport_range.1) + { + if !is_binary_searching + && upper_bound != i32::MAX + && upper_bound != block.counter_range.1 + { + warn!( + "There is a hole between the last block and the current block" + ); + // There is hole between the last block and the current block + // We need to load it from the kv store + break; + } + + // Found the block + if let Err(err) = block.ensure_changes(&self.arena) { + warn!(block_id = ?id, ?err, "failed to parse change block"); + return None; + } + let index = block.get_change_index_by_lamport_lte(idlp.lamport)?; + return Some(BlockChangeRef { + change_index: index, + block: block.clone(), + }); + } + + if is_binary_searching { + let mid_bound = (lower_bound + upper_bound) / 2; + if block.lamport_range.1 <= idlp.lamport { + // Target is larger than the current block (pointed by mid_bound) + lower_bound = mid_bound; + } else { + debug_assert!( + idlp.lamport < block.lamport_range.0, + "{} {:?}", + idlp, + &block.lamport_range + ); + // Target is smaller than the current block (pointed by mid_bound) + upper_bound = mid_bound; + } + + let mid_bound = (lower_bound + upper_bound) / 2; + iter = inner + .mem_parsed_kv + .range_mut(ID::new(idlp.peer, 0)..ID::new(idlp.peer, mid_bound)); + } else { + // Test whether we need to switch to binary search by measuring the gap + if block.lamport_range.0 - idlp.lamport > MAX_BLOCK_SIZE as Lamport * 8 + { + // Use binary search to find the block + upper_bound = id.counter; + let mid_bound = (lower_bound + upper_bound) / 2; + iter = inner.mem_parsed_kv.range_mut( + ID::new(idlp.peer, 0)..ID::new(idlp.peer, mid_bound), + ); + is_binary_searching = true; + } + + upper_bound = id.counter; + } + } + None => { + if !is_binary_searching { + break; + } + + let mid_bound = (lower_bound + upper_bound) / 2; + lower_bound = mid_bound; + if upper_bound - lower_bound <= MAX_BLOCK_SIZE as i32 { + // If they are too close, we can just scan the range + iter = inner.mem_parsed_kv.range_mut( + ID::new(idlp.peer, lower_bound)..ID::new(idlp.peer, upper_bound), + ); + is_binary_searching = false; + } else { + let mid_bound = (lower_bound + upper_bound) / 2; + iter = inner + .mem_parsed_kv + .range_mut(ID::new(idlp.peer, 0)..ID::new(idlp.peer, mid_bound)); + } + } + } + } + + let counter_end = upper_bound; + + // The answer may live only in `mem_parsed_kv` (e.g. local changes + // that have not been flushed to the external kv store yet) or only + // in `external_kv` (blocks that were never parsed into memory). + // Check both and use the block with the greatest start counter: + // within a peer, lamport grows with counter, so that block holds + // the greatest matching lamport. + let mem_block_id: Option = inner + .mem_parsed_kv + .range(ID::new(idlp.peer, 0)..ID::new(idlp.peer, counter_end)) + .rev() + .find(|(_, block)| block.lamport_range.0 <= idlp.lamport) + .map(|(id, _)| *id); + + let external_block = 'block_scan: { + let kv_store = &self.external_kv.lock(); + let scan_end = ID::new(idlp.peer, counter_end).to_bytes(); + let iter = kv_store + .scan( + Bound::Included(&ID::new(idlp.peer, 0).to_bytes()), + Bound::Excluded(&scan_end), + ) + .rev(); + + for (id, bytes) in iter { + let mut block = ChangesBlockBytes::new(bytes.clone()); + let (lamport_start, _lamport_end) = match block.lamport_range() { + Ok(range) => range, + Err(err) => { + let block_id = ID::from_bytes(&id); + warn!( + ?block_id, + ?err, + "failed to decode external change block range" + ); + continue; + } + }; + if lamport_start <= idlp.lamport { + break 'block_scan Some((ID::from_bytes(&id), bytes)); + } + } + + None + }; + + let use_external = match (mem_block_id, &external_block) { + (Some(mem_id), Some((external_id, _))) => external_id.counter > mem_id.counter, + (Some(_), None) => false, + (None, Some(_)) => true, + (None, None) => return None, + }; + + if !use_external { + let block_id = mem_block_id.unwrap(); + let block = inner.mem_parsed_kv.get_mut(&block_id).unwrap(); + if let Err(err) = block.ensure_changes(&self.arena) { + warn!(?block_id, ?err, "failed to parse change block"); + return None; + } + let block = block.clone(); + let index = block.get_change_index_by_lamport_lte(idlp.lamport)?; + return Some(BlockChangeRef { + change_index: index, + block, + }); + } + + let (block_id, bytes) = external_block.unwrap(); + let mut block = match ChangesBlock::from_bytes(bytes) { + Ok(block) => Arc::new(block), + Err(err) => { + warn!(?block_id, ?err, "failed to decode external change block"); + return None; + } + }; + if let Err(err) = block.ensure_changes(&self.arena) { + warn!(?block_id, ?err, "failed to parse external change block"); + return None; + } + inner.mem_parsed_kv.insert(block_id, block.clone()); + let index = block.get_change_index_by_lamport_lte(idlp.lamport)?; + Some(BlockChangeRef { + change_index: index, + block, + }) + } + + fn split_change_then_insert( + &self, + change: Change, + mut rollback: Option<&mut ChangeStoreRollback>, + ) { + let original_len = change.atom_len(); + let mut new_change = Change { + ops: RleVec::new(), + deps: change.deps, + id: change.id, + lamport: change.lamport, + timestamp: change.timestamp, + commit_msg: change.commit_msg.clone(), + }; + + let mut total_len = 0; + let mut estimated_size = new_change.estimate_storage_size(); + 'outer: for mut op in change.ops.into_iter() { + if op.estimate_storage_size() >= MAX_BLOCK_SIZE - estimated_size { + new_change = self._insert_splitted_change( + new_change, + &mut total_len, + &mut estimated_size, + rollback.as_deref_mut(), + ); + } + + while let Some(end) = + op.check_whether_slice_content_to_fit_in_size(MAX_BLOCK_SIZE - estimated_size) + { + // The new op can take the rest of the room + let new = op.slice(0, end); + new_change.ops.push(new); + new_change = self._insert_splitted_change( + new_change, + &mut total_len, + &mut estimated_size, + rollback.as_deref_mut(), + ); + + if end < op.atom_len() { + op = op.slice(end, op.atom_len()); + } else { + continue 'outer; + } + } + + estimated_size += op.estimate_storage_size(); + if estimated_size > MAX_BLOCK_SIZE && !new_change.ops.is_empty() { + new_change = self._insert_splitted_change( + new_change, + &mut total_len, + &mut estimated_size, + rollback.as_deref_mut(), + ); + new_change.ops.push(op); + } else { + new_change.ops.push(op); + } + } + + if !new_change.ops.is_empty() { + total_len += new_change.atom_len(); + self.insert_change_inner(new_change, false, false, rollback); + } + + assert_eq!(total_len, original_len); + } + + fn _insert_splitted_change( + &self, + new_change: Change, + total_len: &mut usize, + estimated_size: &mut usize, + rollback: Option<&mut ChangeStoreRollback>, + ) -> Change { + if new_change.atom_len() == 0 { + return new_change; + } + + let ctr_end = new_change.id.counter + new_change.atom_len() as Counter; + let next_lamport = new_change.lamport + new_change.atom_len() as Lamport; + *total_len += new_change.atom_len(); + let ans = Change { + ops: RleVec::new(), + deps: ID::new(new_change.id.peer, ctr_end - 1).into(), + id: ID::new(new_change.id.peer, ctr_end), + lamport: next_lamport, + timestamp: new_change.timestamp, + commit_msg: new_change.commit_msg.clone(), + }; + + self.insert_change_inner(new_change, false, false, rollback); + *estimated_size = ans.estimate_storage_size(); + ans + } + + fn get_parsed_block(&self, id: ID) -> Option> { + let mut inner = self.inner.lock(); + if let Some((_id, block)) = inner.mem_parsed_kv.range_mut(..=id).next_back() { + if block.peer == id.peer && block.counter_range.1 > id.counter { + if let Err(err) = block.ensure_changes(&self.arena) { + warn!(block_id = ?_id, ?err, "failed to parse cached change block"); + return None; + } + return Some(block.clone()); + } + } + + let store = self.external_kv.lock(); + let mut iter = store + .scan(Bound::Unbounded, Bound::Included(&id.to_bytes())) + .filter(|(id, _)| id.len() == 12); + + // println!( + // "\nkeys {:?}", + // store + // .scan(Bound::Unbounded, Bound::Included(&id.to_bytes())) + // .filter(|(id, _)| id.len() == 12) + // .map(|(k, _v)| ID::from_bytes(&k)) + // .count() + // ); + // println!("id {:?}", id); + + let (b_id, b_bytes) = iter.next_back()?; + let block_id: ID = ID::from_bytes(&b_id[..]); + let block = match ChangesBlock::from_bytes(b_bytes) { + Ok(block) => block, + Err(err) => { + warn!(?block_id, ?err, "failed to decode external change block"); + return None; + } + }; + if block_id.peer == id.peer + && block_id.counter <= id.counter + && block.counter_range.1 > id.counter + { + let mut arc_block = Arc::new(block); + if let Err(err) = arc_block.ensure_changes(&self.arena) { + warn!(?block_id, ?err, "failed to parse external change block"); + return None; + } + inner.mem_parsed_kv.insert(block_id, arc_block.clone()); + return Some(arc_block); + } + + None + } + + /// Load all the blocks that have overlapped with the given ID range into `inner_mem_parsed_kv` + /// + /// This is fast because we don't actually parse the content. + // TODO: PERF: This method feels slow. + pub(super) fn ensure_block_loaded_in_range(&self, start: Bound, end: Bound) { + let mut whether_need_scan_backward = match start { + Bound::Included(id) => Some(id), + Bound::Excluded(id) => Some(id.inc(1)), + Bound::Unbounded => None, + }; + + { + let start = start.map(|id| id.to_bytes()); + let end = end.map(|id| id.to_bytes()); + let kv = self.external_kv.lock(); + let mut inner = self.inner.lock(); + for (id, bytes) in kv + .scan( + start.as_ref().map(|x| x.as_slice()), + end.as_ref().map(|x| x.as_slice()), + ) + .filter(|(id, _)| id.len() == 12) + { + let id = ID::from_bytes(&id); + if let Some(expected_start_id) = whether_need_scan_backward { + if id == expected_start_id { + whether_need_scan_backward = None; + } + } + + if inner.mem_parsed_kv.contains_key(&id) { + continue; + } + + let block = match ChangesBlock::from_bytes(bytes.clone()) { + Ok(block) => block, + Err(err) => { + warn!(?id, ?err, "failed to decode external change block"); + continue; + } + }; + inner.mem_parsed_kv.insert(id, Arc::new(block)); + } + } + + if let Some(start_id) = whether_need_scan_backward { + self.ensure_id_lte(start_id); + } + } + + pub(super) fn ensure_id_lte(&self, id: ID) { + let kv = self.external_kv.lock(); + let mut inner = self.inner.lock(); + let Some((next_back_id, next_back_bytes)) = kv + .scan(Bound::Unbounded, Bound::Included(&id.to_bytes())) + .rfind(|(id, _)| id.len() == 12) + else { + return; + }; + + let next_back_id = ID::from_bytes(&next_back_id); + if next_back_id.peer == id.peer { + if inner.mem_parsed_kv.contains_key(&next_back_id) { + return; + } + + let block = match ChangesBlock::from_bytes(next_back_bytes) { + Ok(block) => block, + Err(err) => { + warn!( + ?next_back_id, + ?err, + "failed to decode external change block" + ); + return; + } + }; + inner.mem_parsed_kv.insert(next_back_id, Arc::new(block)); + } + } + } +} + +#[must_use] +#[derive(Clone, Debug)] +pub(crate) struct BatchDecodeInfo { + pub vv: VersionVector, + pub frontiers: Frontiers, + pub start_version: Option<(VersionVector, Frontiers)>, +} + +#[derive(Clone, Debug)] +pub struct BlockChangeRef { + block: Arc, + change_index: usize, +} + +impl Deref for BlockChangeRef { + type Target = Change; + fn deref(&self) -> &Change { + &self.block.content.try_changes().unwrap()[self.change_index] + } +} + +impl BlockChangeRef { + pub(crate) fn get_op_with_counter(&self, counter: Counter) -> Option { + if counter >= self.ctr_end() { + return None; + } + + let index = self.ops.search_atom_index(counter); + Some(BlockOpRef { + block: self.block.clone(), + change_index: self.change_index, + op_index: index, + }) + } +} + +#[derive(Clone, Debug)] +pub(crate) struct BlockOpRef { + pub block: Arc, + pub change_index: usize, + pub op_index: usize, +} + +impl Deref for BlockOpRef { + type Target = Op; + + fn deref(&self) -> &Op { + &self.block.content.try_changes().unwrap()[self.change_index].ops[self.op_index] + } +} + +impl BlockOpRef { + pub fn lamport(&self) -> Lamport { + let change = &self.block.content.try_changes().unwrap()[self.change_index]; + let op = &change.ops[self.op_index]; + (op.counter - change.id.counter) as Lamport + change.lamport + } +} + +impl ChangesBlock { + fn from_bytes(bytes: Bytes) -> LoroResult { + let len = bytes.len(); + let bytes = ChangesBlockBytes::new(bytes); + bytes.ensure_header()?; + let header = bytes + .header + .get() + .expect("header should be initialized after ensure_header"); + let peer = header.peer; + let counter_range = ( + header.counter, + *header.counters.last().ok_or_else(|| { + LoroError::DecodeError("Decode block error: missing counters".into()) + })?, + ); + // `header.lamports` only stores the start lamport of each change (n entries), + // while `header.counters` has n + 1 entries ending with the block's end counter. + // The block's exclusive end lamport is the last change's start lamport plus its len. + let last_change_start = *header + .counters + .len() + .checked_sub(2) + .and_then(|i| header.counters.get(i)) + .ok_or_else(|| LoroError::DecodeError("Decode block error: missing counters".into()))?; + let last_change_len = counter_range.1 - last_change_start; + let lamport_range = ( + *header.lamports.first().ok_or_else(|| { + LoroError::DecodeError("Decode block error: missing lamports".into()) + })?, + header + .lamports + .last() + .ok_or_else(|| { + LoroError::DecodeError("Decode block error: missing lamports".into()) + })? + .checked_add(last_change_len as Lamport) + .ok_or_else(|| { + LoroError::DecodeError("Decode block error: lamport overflow".into()) + })?, + ); + let content = ChangesBlockContent::Bytes(bytes); + Ok(Self { + peer, + estimated_size: len, + counter_range, + lamport_range, + flushed: true, + content, + }) + } + + #[allow(dead_code)] + pub(crate) fn content(&self) -> &ChangesBlockContent { + &self.content + } + + fn new(change: Change, _a: &SharedArena) -> Self { + let atom_len = change.atom_len(); + let counter_range = (change.id.counter, change.id.counter + atom_len as Counter); + let lamport_range = (change.lamport, change.lamport + atom_len as Lamport); + let estimated_size = change.estimate_storage_size(); + let peer = change.id.peer; + let content = ChangesBlockContent::Changes(Arc::new(vec![change])); + Self { + peer, + counter_range, + lamport_range, + estimated_size, + content, + flushed: false, + } + } + + #[allow(unused)] + fn cmp_id(&self, id: ID) -> Ordering { + self.peer.cmp(&id.peer).then_with(|| { + if self.counter_range.0 > id.counter { + Ordering::Greater + } else if self.counter_range.1 <= id.counter { + Ordering::Less + } else { + Ordering::Equal + } + }) + } + + #[allow(unused)] + fn cmp_idlp(&self, idlp: (PeerID, Lamport)) -> Ordering { + self.peer.cmp(&idlp.0).then_with(|| { + if self.lamport_range.0 > idlp.1 { + Ordering::Greater + } else if self.lamport_range.1 <= idlp.1 { + Ordering::Less + } else { + Ordering::Equal + } + }) + } + + #[allow(unused)] + fn is_full(&self) -> bool { + self.estimated_size > MAX_BLOCK_SIZE + } + + #[allow(clippy::result_large_err)] + fn push_change( + self: &mut Arc, + change: Change, + new_change_size: usize, + merge_interval: i64, + a: &SharedArena, + ) -> Result<(), Change> { + if self.counter_range.1 != change.id.counter { + return Err(change); + } + + let atom_len = change.atom_len(); + let next_lamport = change.lamport + atom_len as Lamport; + let next_counter = change.id.counter + atom_len as Counter; + + let is_full = new_change_size + self.estimated_size > MAX_BLOCK_SIZE; + let this = Arc::make_mut(self); + let changes = this.content.changes_mut(a).unwrap(); + let changes = Arc::make_mut(changes); + match changes.last_mut() { + Some(last) + if last.can_merge_right(&change, merge_interval) + && (!is_full + || (change.ops.len() == 1 + && last.ops.last().unwrap().is_mergable(&change.ops[0], &()))) => + { + for op in change.ops.into_iter() { + let size = op.estimate_storage_size(); + if !last.ops.push(op) { + this.estimated_size += size; + } + } + } + _ => { + if is_full { + return Err(change); + } else { + this.estimated_size += new_change_size; + changes.push(change); + } + } + } + + this.flushed = false; + this.counter_range.1 = next_counter; + this.lamport_range.1 = next_lamport; + Ok(()) + } + + fn to_bytes(self: &mut Arc, a: &SharedArena) -> ChangesBlockBytes { + match &self.content { + ChangesBlockContent::Bytes(bytes) => bytes.clone(), + ChangesBlockContent::Both(_, bytes) => { + let bytes = bytes.clone(); + let this = Arc::make_mut(self); + this.content = ChangesBlockContent::Bytes(bytes.clone()); + bytes + } + ChangesBlockContent::Changes(changes) => { + let bytes = ChangesBlockBytes::serialize(changes, a); + let this = Arc::make_mut(self); + this.content = ChangesBlockContent::Bytes(bytes.clone()); + bytes + } + } + } + + fn ensure_changes(self: &mut Arc, a: &SharedArena) -> LoroResult<()> { + match &self.content { + ChangesBlockContent::Changes(_) => Ok(()), + ChangesBlockContent::Both(_, _) => Ok(()), + ChangesBlockContent::Bytes(bytes) => { + let changes = bytes.parse(a)?; + let b = bytes.clone(); + let this = Arc::make_mut(self); + this.content = ChangesBlockContent::Both(Arc::new(changes), b); + Ok(()) + } + } + } + + fn get_change_index_by_counter(&self, counter: Counter) -> Result { + let changes = self.content.try_changes().unwrap(); + changes.binary_search_by(|c| { + if c.id.counter > counter { + Ordering::Greater + } else if (c.id.counter + c.content_len() as Counter) <= counter { + Ordering::Less + } else { + Ordering::Equal + } + }) + } + + fn get_change_index_by_lamport_lte(&self, lamport: Lamport) -> Option { + let changes = self.content.try_changes().unwrap(); + let r = changes.binary_search_by(|c| { + if c.lamport > lamport { + Ordering::Greater + } else if (c.lamport + c.content_len() as Lamport) <= lamport { + Ordering::Less + } else { + Ordering::Equal + } + }); + + match r { + Ok(found) => Some(found), + Err(idx) => { + if idx == 0 { + None + } else { + Some(idx - 1) + } + } + } + } + + #[allow(unused)] + fn get_changes(&mut self, a: &SharedArena) -> LoroResult<&Vec> { + self.content.changes(a) + } + + #[allow(unused)] + fn id(&self) -> ID { + ID::new(self.peer, self.counter_range.0) + } + + pub fn change_num(&self) -> usize { + match &self.content { + ChangesBlockContent::Changes(c) => c.len(), + ChangesBlockContent::Bytes(b) => b.len_changes(), + ChangesBlockContent::Both(c, _) => c.len(), + } + } +} + +impl ChangesBlockContent { + // TODO: PERF: We can use Iter to replace Vec + pub fn iter_dag_nodes(&self) -> Vec { + let mut dag_nodes = Vec::new(); + match self { + ChangesBlockContent::Changes(c) | ChangesBlockContent::Both(c, _) => { + for change in c.iter() { + let new_node = AppDagNodeInner { + peer: change.id.peer, + cnt: change.id.counter, + lamport: change.lamport, + deps: change.deps.clone(), + vv: OnceCell::new(), + has_succ: false, + len: change.atom_len(), + } + .into(); + + dag_nodes.push_rle_element(new_node); + } + } + ChangesBlockContent::Bytes(b) => { + b.ensure_header().unwrap(); + let header = b.header.get().unwrap(); + let n = header.n_changes; + for i in 0..n { + let new_node = AppDagNodeInner { + peer: header.peer, + cnt: header.counters[i], + lamport: header.lamports[i], + deps: header.deps_groups[i].clone(), + vv: OnceCell::new(), + has_succ: false, + len: (header.counters[i + 1] - header.counters[i]) as usize, + } + .into(); + + dag_nodes.push_rle_element(new_node); + } + } + } + + dag_nodes + } + + #[allow(unused)] + pub fn changes(&mut self, a: &SharedArena) -> LoroResult<&Vec> { + match self { + ChangesBlockContent::Changes(changes) => Ok(changes), + ChangesBlockContent::Both(changes, _) => Ok(changes), + ChangesBlockContent::Bytes(bytes) => { + let changes = bytes.parse(a)?; + *self = ChangesBlockContent::Both(Arc::new(changes), bytes.clone()); + self.changes(a) + } + } + } + + /// Note that this method will invalidate the stored bytes + fn changes_mut(&mut self, a: &SharedArena) -> LoroResult<&mut Arc>> { + match self { + ChangesBlockContent::Changes(changes) => Ok(changes), + ChangesBlockContent::Both(changes, _) => { + *self = ChangesBlockContent::Changes(std::mem::take(changes)); + self.changes_mut(a) + } + ChangesBlockContent::Bytes(bytes) => { + let changes = bytes.parse(a)?; + *self = ChangesBlockContent::Changes(Arc::new(changes)); + self.changes_mut(a) + } + } + } + + pub(crate) fn try_changes(&self) -> Option<&Vec> { + match self { + ChangesBlockContent::Changes(changes) => Some(changes), + ChangesBlockContent::Both(changes, _) => Some(changes), + ChangesBlockContent::Bytes(_) => None, + } + } + + #[allow(dead_code)] + pub(crate) fn len_changes(&self) -> usize { + match self { + ChangesBlockContent::Changes(changes) => changes.len(), + ChangesBlockContent::Both(changes, _) => changes.len(), + ChangesBlockContent::Bytes(bytes) => bytes.len_changes(), + } + } +} + +impl std::fmt::Debug for ChangesBlockContent { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ChangesBlockContent::Changes(changes) => f + .debug_tuple("ChangesBlockContent::Changes") + .field(changes) + .finish(), + ChangesBlockContent::Bytes(_bytes) => { + f.debug_tuple("ChangesBlockContent::Bytes").finish() + } + ChangesBlockContent::Both(changes, _bytes) => f + .debug_tuple("ChangesBlockContent::Both") + .field(changes) + .finish(), + } + } +} + +impl ChangesBlockBytes { + fn new(bytes: Bytes) -> Self { + Self { + header: OnceCell::new(), + bytes, + } + } + + fn ensure_header(&self) -> LoroResult<()> { + self.header + .get_or_try_init(|| decode_header(&self.bytes).map(Arc::new))?; + Ok(()) + } + + fn parse(&self, a: &SharedArena) -> LoroResult> { + self.ensure_header()?; + let ans: Vec = decode_block(&self.bytes, a, self.header.get().map(|h| h.as_ref()))?; + for c in ans.iter() { + // PERF: This can be made faster (low priority) + register_container_and_parent_link(a, c) + } + + Ok(ans) + } + + fn serialize(changes: &[Change], a: &SharedArena) -> Self { + let bytes = encode_block(changes, a); + // TODO: Perf we can calculate header directly without parsing the bytes + let bytes = ChangesBlockBytes::new(Bytes::from(bytes)); + bytes.ensure_header().unwrap(); + bytes + } + + fn lamport_range(&mut self) -> LoroResult<(Lamport, Lamport)> { + if let Some(header) = self.header.get() { + Ok((header.lamports[0], *header.lamports.last().unwrap())) + } else { + decode_block_range(&self.bytes).map(|(_, lamport_range)| lamport_range) + } + } + + /// Length of the changes + fn len_changes(&self) -> usize { + self.ensure_header().unwrap(); + self.header.get().unwrap().n_changes + } +} + +#[cfg(test)] +mod test { + use crate::cursor::PosType; + use crate::{ + loro::ExportMode, oplog::convert_change_to_remote, state::TreeParentId, ListHandler, + LoroDoc, MovableListHandler, TextHandler, TreeHandler, + }; + + use super::*; + + fn test_encode_decode(doc: LoroDoc) { + doc.commit_then_renew(); + let oplog = doc.oplog().lock(); + let bytes = oplog + .change_store + .encode_all(oplog.vv(), oplog.dag.frontiers()); + let store = ChangeStore::new_for_test(); + let _ = store.import_all(bytes.clone()).unwrap(); + assert_eq!(store.external_kv.lock().export_all(), bytes); + let mut changes_parsed = Vec::new(); + let a = store.arena.clone(); + store.visit_all_changes(&mut |c| { + changes_parsed.push(convert_change_to_remote(&a, c)); + }); + let mut changes = Vec::new(); + oplog.change_store.visit_all_changes(&mut |c| { + changes.push(convert_change_to_remote(&oplog.arena, c)); + }); + assert_eq!(changes_parsed, changes); + } + + #[test] + fn decoded_block_lamport_range_matches_counter_range() { + // Regression test for a checkout hang after snapshot import. + // `ChangesBlock::from_bytes` used the start lamport of the block's last + // change as the block's end lamport, producing degenerate lamport + // ranges (empty for single-change blocks). The binary search in + // `get_change_by_lamport_lte` then misclassified the block containing + // the target lamport and looped forever. + let doc = LoroDoc::new_auto_commit(); + doc.set_peer_id(1).unwrap(); + // One big commit that splits into many blocks, with enough ops that + // lamport lookups engage the binary search path + // (lamport gap > MAX_BLOCK_SIZE * 8). + for i in 0..100 { + let text = doc.get_text(format!("t{i}").as_str()); + text.insert(0, &"x".repeat(30), PosType::Unicode).unwrap(); + } + doc.commit_then_renew(); + + let (bytes, end_counter) = { + let oplog = doc.oplog().lock(); + let end = oplog.vv().get(&1).copied().unwrap(); + let bytes = oplog + .change_store + .encode_all(oplog.vv(), oplog.dag.frontiers()); + (bytes, end) + }; + + let store = ChangeStore::new_for_test(); + let _ = store.import_all(bytes).unwrap(); + // Parse every block out of the external kv store + let mut c = 0; + while c < end_counter { + let change = store.get_change(ID::new(1, c)).unwrap(); + c = change.id.counter + change.atom_len() as Counter; + } + + { + let inner = store.inner.lock(); + assert!( + inner.mem_parsed_kv.len() > 1, + "the change should be split into multiple blocks" + ); + for (id, block) in inner.mem_parsed_kv.iter() { + // Single-peer linear history: lamport == counter for every op + assert_eq!(id.counter, block.counter_range.0); + assert_eq!(block.lamport_range.0 as Counter, block.counter_range.0); + assert_eq!(block.lamport_range.1 as Counter, block.counter_range.1); + } + } + + for l in (0..end_counter as Lamport).step_by(7) { + let change = store.get_change_by_lamport_lte(IdLp::new(1, l)).unwrap(); + assert!(change.lamport <= l); + assert!(l < change.lamport + change.atom_len() as Lamport); + } + } + + #[test] + fn lamport_lookup_finds_unflushed_mem_blocks() { + // Regression: when the lamport binary search bails out, the fallback + // used to scan only the external kv store. Local changes may exist + // solely in `mem_parsed_kv` before any flush, so a lookup targeting a + // lamport gap incorrectly returned `None`. + let doc = LoroDoc::new_auto_commit(); + doc.set_peer_id(1).unwrap(); + let text = doc.get_text("t"); + text.insert(0, &"x".repeat(500), PosType::Unicode).unwrap(); + doc.commit_then_renew(); + + // An independent large change from peer 2 pushes peer 1's next + // lamport far above its first change (gap > MAX_BLOCK_SIZE * 8, so + // lookups below engage the binary search path). + let doc2 = LoroDoc::new_auto_commit(); + doc2.set_peer_id(2).unwrap(); + let text2 = doc2.get_text("t2"); + text2 + .insert(0, &"y".repeat(3000), PosType::Unicode) + .unwrap(); + doc2.commit_then_renew(); + doc.import(&doc2.export(ExportMode::all_updates()).unwrap()) + .unwrap(); + + let text = doc.get_text("t"); + text.insert(0, &"z".repeat(500), PosType::Unicode).unwrap(); + doc.commit_then_renew(); + + // Query a lamport inside the gap between peer 1's two changes, + // without flushing the change store. The answer (the tail of peer 1's + // first commit) lives only in `mem_parsed_kv`. + let oplog = doc.oplog().lock(); + let change = oplog + .change_store + .get_change_by_lamport_lte(IdLp::new(1, 700)) + .expect("the change should be found in the unflushed mem blocks"); + assert_eq!(change.id.peer, 1); + assert!(change.lamport <= 700); + assert_eq!(change.id.counter + change.atom_len() as Counter, 500); + } + + #[test] + fn test_change_store() { + let doc = LoroDoc::new_auto_commit(); + doc.set_record_timestamp(true); + let t = doc.get_text("t"); + t.insert(0, "hello", PosType::Unicode).unwrap(); + doc.commit_then_renew(); + let t = doc.get_list("t"); + t.insert(0, "hello").unwrap(); + test_encode_decode(doc); + } + + #[test] + fn test_synced_doc() -> LoroResult<()> { + let doc_a = LoroDoc::new_auto_commit(); + let doc_b = LoroDoc::new_auto_commit(); + let doc_c = LoroDoc::new_auto_commit(); + + { + // A: Create initial structure + let map = doc_a.get_map("root"); + map.insert_container("text", TextHandler::new_detached())?; + map.insert_container("list", ListHandler::new_detached())?; + map.insert_container("tree", TreeHandler::new_detached())?; + } + + { + // Sync initial state to B and C + let initial_state = doc_a.export(ExportMode::all_updates()).unwrap(); + doc_b.import(&initial_state)?; + doc_c.import(&initial_state)?; + } + + { + // B: Edit text and list + let map = doc_b.get_map("root"); + let text = map + .insert_container("text", TextHandler::new_detached()) + .unwrap(); + text.insert(0, "Hello, ", PosType::Unicode)?; + + let list = map + .insert_container("list", ListHandler::new_detached()) + .unwrap(); + list.push("world")?; + } + + { + // C: Edit tree and movable list + let map = doc_c.get_map("root"); + let tree = map + .insert_container("tree", TreeHandler::new_detached()) + .unwrap(); + let node_id = tree.create(TreeParentId::Root)?; + tree.get_meta(node_id)?.insert("key", "value")?; + let node_b = tree.create(TreeParentId::Root)?; + tree.move_to(node_b, TreeParentId::Root, 0).unwrap(); + + let movable_list = map + .insert_container("movable", MovableListHandler::new_detached()) + .unwrap(); + movable_list.push("item1".into())?; + movable_list.push("item2".into())?; + movable_list.mov(0, 1)?; + } + + // Sync B's changes to A + let b_changes = doc_b + .export(ExportMode::updates(&doc_a.oplog_vv())) + .unwrap(); + doc_a.import(&b_changes)?; + + // Sync C's changes to A + let c_changes = doc_c + .export(ExportMode::updates(&doc_a.oplog_vv())) + .unwrap(); + doc_a.import(&c_changes)?; + + test_encode_decode(doc_a); + Ok(()) + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/oplog/change_store/block_encode.rs b/3rdparty/loro/crates/loro-internal/src/oplog/change_store/block_encode.rs new file mode 100644 index 0000000000..f18d1db8fb --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/oplog/change_store/block_encode.rs @@ -0,0 +1,830 @@ +//! # Encode +//! +//! ```log +//! ≈4KB after compression +//! +//! N = Number of Changes +//! +//! Peer_1 = This Peer +//! +//! +//! ┌────────────────────────┬─────────────────────────────────────┐ +//! │ LEB Counter Start&Len │ LEB Lamport Start&Len │◁───┐ +//! └────────────────────────┴─────────────────────────────────────┘ │ +//! ┌──────────────┬──────────────┬────────────────────────────────┐ │ +//! │ LEB N │ LEB Peer Num │ 8B Peer_1,...,Peer_x │◁───┤ +//! └──────────────┴──────────────┴────────────────────────────────┘ │ +//! ┌──────────────────────────────────────────────────────────────┐ │ +//! │ N LEB128 Change AtomLen │◁───┼───── Important metadata +//! └──────────────────────────────────────────────────────────────┘ │ +//! ┌───────────────────┬────────────────────────┬─────────────────┐ │ +//! │N DepOnSelf BoolRle│ N Delta Rle Deps Lens │ Dep IDs │◁───┤ +//! └───────────────────┴────────────────────────┴─────────────────┘ │ +//! ┌──────────────────────────────────────────────────────────────┐ │ +//! │ N LEB128 Delta Lamports │◁───┘ +//! └──────────────────────────────────────────────────────────────┘ +//! ┌──────────────────────────────────────────────────────────────┐ +//! │ N LEB128 DeltaOfDelta Timestamps │ +//! └──────────────────────────────────────────────────────────────┘ +//! ┌────────────────────────────────┬─────────────────────────────┐ +//! │ N Rle Commit Msg Lengths │ Commit Messages │ +//! └────────────────────────────────┴─────────────────────────────┘ +//! +//! ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ Encoded Operations ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ +//! +//! ┌────────────────────┬─────────────────────────────────────────┐ +//! │ ContainerIDs Size │ ContainerIDs │ +//! └────────────────────┴─────────────────────────────────────────┘ +//! ┌────────────────────┬─────────────────────────────────────────┐ +//! │ Key Strings Size │ Key Strings │ +//! └────────────────────┴─────────────────────────────────────────┘ +//! ┌────────────────────┬─────────────────────────────────────────┐ +//! │ Position Size │ Position │ +//! └────────────────────┴─────────────────────────────────────────┘ +//! ┌────────┬──────────┬──────────┬───────┬───────────────────────┐ +//! │ │ │ │ │ │ +//! │ │ │ │ │ │ +//! │ │ │ │ │ │ +//! │ Ops │ LEB128 │ RLE │ Delta │ │ +//! │ Size │ Lengths │Containers│ RLE │ ValueType │ +//! │ │ │ │ Props │ │ +//! │ │ │ │ │ │ +//! │ │ │ │ │ │ +//! │ │ │ │ │ │ +//! └────────┴──────────┴──────────┴───────┴───────────────────────┘ +//! ┌──────────────────────────────────────────────────────────────┐ +//! │ (Encoded with Ops By serde_columnar) │ +//! │ Delete Start IDs │ +//! │ │ +//! └──────────────────────────────────────────────────────────────┘ +//! ┌────────────────┬─────────────────────────────────────────────┐ +//! │ │ │ +//! │Value Bytes Size│ Value Bytes │ +//! │ │ │ +//! └────────────────┴─────────────────────────────────────────────┘ +//! ``` + +use std::borrow::Cow; +use std::collections::BTreeSet; +use std::io::Write; +use std::sync::Arc; + +use fractional_index::FractionalIndex; +use loro_common::{ + ContainerID, Counter, HasCounterSpan, HasLamportSpan, InternalString, Lamport, LoroError, + LoroResult, PeerID, TreeID, ID, +}; +use once_cell::sync::OnceCell; +use rle::HasLength; +use serde::{Deserialize, Serialize}; +use serde_columnar::{columnar, AnyRleDecoder, DeltaOfDeltaDecoder, Itertools}; +use tracing::info; + +use super::block_meta_encode::decode_changes_header; +use crate::arena::SharedArena; +use crate::change::{Change, Timestamp}; +use crate::container::tree::tree_op; +use crate::encoding::arena::{ContainerArena, PositionArena}; +use crate::encoding::value_register::ValueRegister; +use crate::encoding::{ + self, decode_op, encode_op, get_op_prop, EncodedDeleteStartId, IterableEncodedDeleteStartId, +}; +use crate::op::Op; + +#[derive(Debug, Serialize, Deserialize)] +struct EncodedBlock<'a> { + counter_start: u32, + counter_len: u32, + lamport_start: u32, + lamport_len: u32, + n_changes: u32, + #[serde(borrow)] + header: Cow<'a, [u8]>, + // timestamp and commit messages + #[serde(borrow)] + change_meta: Cow<'a, [u8]>, + // ---------------------- Ops ---------------------- + #[serde(borrow)] + cids: Cow<'a, [u8]>, + #[serde(borrow)] + keys: Cow<'a, [u8]>, + #[serde(borrow)] + positions: Cow<'a, [u8]>, + #[serde(borrow)] + ops: Cow<'a, [u8]>, + #[serde(borrow)] + delete_start_ids: Cow<'a, [u8]>, + #[serde(borrow)] + values: Cow<'a, [u8]>, +} + +fn diagnose_block(block: &EncodedBlock) { + info!("Diagnosing EncodedBlock:"); + info!(" header {} bytes", block.header.len()); + info!(" change_meta {} bytes", block.change_meta.len()); + info!(" cids: {} bytes", block.cids.len()); + info!(" keys: {} bytes", block.keys.len()); + info!(" positions: {} bytes", block.positions.len()); + info!(" ops: {} bytes", block.ops.len()); + info!(" delete_id_starts: {} bytes", block.delete_start_ids.len()); + info!(" values: {} bytes", block.values.len()); +} + +const VERSION: u16 = 0; + +// MARK: encode_block +/// It's assume that every change in the block share the same peer. +pub fn encode_block(block: &[Change], arena: &SharedArena) -> Vec { + if block.is_empty() { + panic!("Empty block") + } + + let mut peer_register: ValueRegister = ValueRegister::new(); + let peer = block[0].peer(); + peer_register.register(&peer); + + let cid_register: ValueRegister = ValueRegister::new(); + + let mut encoded_ops = Vec::new(); + let mut registers = Registers { + peer_register, + key_register: ValueRegister::new(), + cid_register, + position_register: ValueRegister::new(), + }; + + { + // Init position register, making it ordered by fractional index + let mut position_set = BTreeSet::default(); + for c in block { + for op in c.ops().iter() { + if let crate::op::InnerContent::Tree(tree_op) = &op.content { + match &**tree_op { + tree_op::TreeOp::Create { position, .. } => { + position_set.insert(position.clone()); + } + tree_op::TreeOp::Move { position, .. } => { + position_set.insert(position.clone()); + } + tree_op::TreeOp::Delete { .. } => {} + } + } + } + } + + for position in position_set { + registers.position_register.register(&position); + } + } + + let mut del_starts: Vec<_> = Vec::new(); + let mut value_writer = ValueWriter::new(); + for c in block { + for op in c.ops().iter() { + let cid = arena.get_container_id(op.container).unwrap(); + let cidx = registers.cid_register.register(&cid); + let prop = get_op_prop(op, &mut registers); + let value_kind = encode_op( + op, + arena, + &mut del_starts, + &mut value_writer, + &mut registers, + ); + encoded_ops.push(EncodedOp { + container_index: cidx as u32, + prop, + value_type: value_kind.to_u8(), + len: op.atom_len() as u32, + }) + } + } + + let cids = registers.cid_register.unwrap_vec(); + let container_arena = ContainerArena::from_containers( + cids, + &mut registers.peer_register, + &mut registers.key_register, + ); + + // Write to output + + // ┌────────────────────┬─────────────────────────────────────────┐ + // │ Key Strings Size │ Key Strings │ + // └────────────────────┴─────────────────────────────────────────┘ + let keys = registers.key_register.unwrap_vec(); + let keys_bytes = encode_keys(keys); + + // ┌────────────────────┬─────────────────────────────────────────┐ + // │ Position Size │ Position │ + // └────────────────────┴─────────────────────────────────────────┘ + let position_vec = registers.position_register.unwrap_vec(); + let positions = PositionArena::from_positions(position_vec.iter().map(|p| p.as_bytes())); + let position_bytes = positions.encode_v2(); + + // ┌──────────┬──────────┬───────┬────────────────────────────────┐ + // │ │ │ │ │ + // │ │ │ │ │ + // │ │ │ │ │ + // │ LEB128 │ RLE │ Delta │ │ + // │ Lengths │Containers│ RLE │ ValueType │ + // │ │ │ Props │ │ + // │ │ │ │ │ + // │ │ │ │ │ + // │ │ │ │ │ + // └──────────┴──────────┴───────┴────────────────────────────────┘ + + let ops_bytes = serde_columnar::to_vec(&EncodedOps { ops: encoded_ops }).unwrap(); + + let delete_id_starts_bytes = if del_starts.is_empty() { + Vec::new() + } else { + serde_columnar::to_vec(&EncodedDeleteStartIds { + delete_start_ids: del_starts, + }) + .unwrap() + }; + // ┌────────────────┬─────────────────────────────────────────────┐ + // │Value Bytes Size│ Value Bytes │ + // └────────────────┴─────────────────────────────────────────────┘ + + // PeerIDs + let mut peer_register = registers.peer_register; + // .unwrap_vec(); + // let peer_bytes: Vec = peers.iter().flat_map(|p| p.to_le_bytes()).collect(); + + // Change meta + let (header, change_meta) = encode_changes(block, &mut peer_register); + + let value_bytes = value_writer.finish(); + let out = EncodedBlock { + counter_start: block[0].id.counter as u32, + counter_len: (block.last().unwrap().ctr_end() - block[0].id.counter) as u32, + lamport_start: block[0].lamport(), + lamport_len: block.last().unwrap().lamport_end() - block[0].lamport(), + n_changes: block.len() as u32, + header: header.into(), + change_meta: change_meta.into(), + cids: container_arena.encode().into(), + keys: keys_bytes.into(), + positions: position_bytes.into(), + ops: ops_bytes.into(), + delete_start_ids: delete_id_starts_bytes.into(), + values: value_bytes.into(), + }; + + diagnose_block(&out); + postcard::to_allocvec(&out).unwrap() +} + +fn encode_keys(keys: Vec) -> Vec { + let mut keys_bytes = Vec::new(); + for key in keys { + let bytes = key.as_bytes(); + leb128::write::unsigned(&mut keys_bytes, bytes.len() as u64).unwrap(); + keys_bytes.write_all(bytes).unwrap(); + } + keys_bytes +} + +fn decode_keys(mut bytes: &[u8]) -> LoroResult> { + let mut keys = Vec::new(); + while !bytes.is_empty() { + let len = leb128::read::unsigned(&mut bytes) + .map_err(|_| LoroError::DecodeDataCorruptionError)? as usize; + if bytes.len() < len { + return Err(LoroError::DecodeDataCorruptionError); + } + let key = + std::str::from_utf8(&bytes[..len]).map_err(|_| LoroError::DecodeDataCorruptionError)?; + keys.push(key.into()); + bytes = &bytes[len..]; + } + + Ok(keys) +} + +struct Registers { + peer_register: ValueRegister, + key_register: ValueRegister, + cid_register: ValueRegister, + position_register: ValueRegister, +} + +use crate::encoding::value::{ + RawTreeMove, Value, ValueDecodedArenasTrait, ValueEncodeRegister, ValueKind, ValueReader, + ValueWriter, +}; +use crate::oplog::change_store::block_meta_encode::encode_changes; +use crate::version::Frontiers; +impl ValueEncodeRegister for Registers { + fn key_mut(&mut self) -> &mut ValueRegister { + &mut self.key_register + } + + fn peer_mut(&mut self) -> &mut ValueRegister { + &mut self.peer_register + } + + fn encode_tree_op(&mut self, op: &tree_op::TreeOp) -> encoding::value::Value<'static> { + match op { + tree_op::TreeOp::Create { + target, + parent, + position, + } => encoding::value::Value::RawTreeMove(RawTreeMove { + subject_peer_idx: self.peer_register.register(&target.peer), + subject_cnt: target.counter, + is_parent_null: parent.is_none(), + parent_peer_idx: parent.map_or(0, |p| self.peer_register.register(&p.peer)), + parent_cnt: parent.map_or(0, |p| p.counter), + position_idx: self.position_register.register(position), + }), + tree_op::TreeOp::Move { + target, + parent, + position, + } => encoding::value::Value::RawTreeMove(RawTreeMove { + subject_peer_idx: self.peer_register.register(&target.peer), + subject_cnt: target.counter, + is_parent_null: parent.is_none(), + parent_peer_idx: parent.map_or(0, |p| self.peer_register.register(&p.peer)), + parent_cnt: parent.map_or(0, |p| p.counter), + position_idx: self.position_register.register(position), + }), + tree_op::TreeOp::Delete { target } => { + let parent = TreeID::delete_root(); + encoding::value::Value::RawTreeMove(RawTreeMove { + subject_peer_idx: self.peer_register.register(&target.peer), + subject_cnt: target.counter, + is_parent_null: false, + parent_peer_idx: self.peer_register.register(&parent.peer), + parent_cnt: parent.counter, + position_idx: 0, + }) + } + } + } +} + +#[derive(Debug, Clone)] +pub(crate) struct ChangesBlockHeader { + pub peer: PeerID, + pub counter: Counter, + pub n_changes: usize, + pub peers: Vec, + /// This has n + 1 elements, where counters[n] is the end counter of the + /// last change in the block. + /// + /// You can infer the size of kth change by taking counters[k + 1] - counters[k] + pub counters: Vec, + /// This has n elements + pub lamports: Vec, + pub deps_groups: Vec, + pub keys: OnceCell>, + pub cids: OnceCell>, +} + +pub fn decode_header(m_bytes: &[u8]) -> LoroResult { + let doc = postcard::from_bytes(m_bytes) + .map_err(|e| LoroError::DecodeError(format!("Decode block error {e}").into_boxed_str()))?; + + decode_header_from_doc(&doc) +} + +// MARK: decode_block_from_doc +fn decode_header_from_doc(doc: &EncodedBlock) -> Result { + let EncodedBlock { + n_changes, + header, + counter_len, + counter_start, + lamport_len, + lamport_start, + .. + } = doc; + let ans: ChangesBlockHeader = decode_changes_header( + header, + *n_changes as usize, + *counter_start as Counter, + *counter_len as Counter, + *lamport_start, + *lamport_len, + )?; + Ok(ans) +} + +#[columnar(vec, ser, de, iterable)] +#[derive(Debug, Clone)] +struct EncodedOp { + #[columnar(strategy = "DeltaRle")] + container_index: u32, + #[columnar(strategy = "DeltaRle")] + prop: i32, + #[columnar(strategy = "Rle")] + value_type: u8, + #[columnar(strategy = "Rle")] + len: u32, +} + +#[columnar(ser, de)] +struct EncodedOps { + #[columnar(class = "vec", iter = "EncodedOp")] + ops: Vec, +} + +#[columnar(ser, de)] +struct EncodedDeleteStartIds { + #[columnar(class = "vec", iter = "EncodedDeleteStartId")] + delete_start_ids: Vec, +} + +struct ValueDecodeArena<'a> { + peers: &'a [PeerID], + keys: &'a [InternalString], +} + +impl ValueDecodedArenasTrait for ValueDecodeArena<'_> { + fn keys(&self) -> &[InternalString] { + self.keys + } + + fn peers(&self) -> &[PeerID] { + self.peers + } + + fn decode_tree_op( + &self, + _positions: &[Vec], + _op: encoding::value::EncodedTreeMove, + _id: ID, + ) -> LoroResult { + Err(LoroError::DecodeDataCorruptionError) + } +} + +pub fn decode_block_range( + mut bytes: &[u8], +) -> LoroResult<((Counter, Counter), (Lamport, Lamport))> { + let version = leb128::read::unsigned(&mut bytes).map_err(|e| { + LoroError::DecodeError(format!("Failed to read version: {e}").into_boxed_str()) + })?; + + if version as u16 != VERSION { + return Err(LoroError::DecodeError( + "Version mismatch".to_string().into_boxed_str(), + )); + } + + let counter_start = leb128::read::unsigned(&mut bytes).map_err(|e| { + LoroError::DecodeError(format!("Failed to read counter start: {e}").into_boxed_str()) + })? as Counter; + let counter_len = leb128::read::unsigned(&mut bytes).map_err(|e| { + LoroError::DecodeError(format!("Failed to read counter length: {e}").into_boxed_str()) + })? as Counter; + let lamport_start = leb128::read::unsigned(&mut bytes).map_err(|e| { + LoroError::DecodeError(format!("Failed to read lamport start: {e}").into_boxed_str()) + })? as Lamport; + let lamport_len = leb128::read::unsigned(&mut bytes).map_err(|e| { + LoroError::DecodeError(format!("Failed to read lamport length: {e}").into_boxed_str()) + })? as Lamport; + Ok(( + (counter_start, counter_start + counter_len), + (lamport_start, lamport_start + lamport_len), + )) +} + +/// Ensure the cids in header are decoded +#[allow(unused)] +pub fn decode_cids( + bytes: &[u8], + header: Option, +) -> LoroResult { + let doc = postcard::from_bytes(bytes) + .map_err(|e| LoroError::DecodeError(format!("Decode block error {e}").into_boxed_str()))?; + let header = if let Some(h) = header { + h + } else { + let doc = postcard::from_bytes(bytes).map_err(|e| { + LoroError::DecodeError(format!("Decode block error {e}").into_boxed_str()) + })?; + decode_header_from_doc(&doc)? + }; + + if header.cids.get().is_some() { + return Ok(header); + } + + let EncodedBlock { cids, keys, .. } = doc; + let keys = header.keys.get_or_try_init(|| decode_keys(&keys))?; + let decode_arena = ValueDecodeArena { + peers: &header.peers, + keys, + }; + + header.cids.get_or_try_init(|| { + ContainerArena::decode(&cids)? + .iter() + .map(|x| x.as_container_id(&decode_arena)) + .try_collect() + })?; + Ok(header) +} + +// MARK: decode_block +pub fn decode_block( + m_bytes: &[u8], + shared_arena: &SharedArena, + header: Option<&ChangesBlockHeader>, +) -> LoroResult> { + let doc = postcard::from_bytes(m_bytes) + .map_err(|e| LoroError::DecodeError(format!("Decode block error {e}").into_boxed_str()))?; + let header_on_stack; + let header = if let Some(header) = header { + header + } else { + header_on_stack = decode_header_from_doc(&doc)?; + &header_on_stack + }; + let EncodedBlock { + n_changes, + counter_start: first_counter, + change_meta, + cids, + keys, + ops, + delete_start_ids, + values, + positions, + .. + } = doc; + let n_changes = n_changes as usize; + let mut changes = Vec::with_capacity(n_changes); + let timestamp_decoder = DeltaOfDeltaDecoder::::new(&change_meta) + .map_err(|_| LoroError::DecodeDataCorruptionError)?; + let (timestamps, bytes) = timestamp_decoder + .take_n_finalize(n_changes) + .map_err(|_| LoroError::DecodeDataCorruptionError)?; + let commit_msg_len_decoder = AnyRleDecoder::::new(bytes); + let (commit_msg_lens, commit_msgs) = commit_msg_len_decoder + .take_n_finalize(n_changes) + .map_err(|_| LoroError::DecodeDataCorruptionError)?; + let mut commit_msg_index = 0u32; + let keys = header.keys.get_or_try_init(|| decode_keys(&keys))?; + let decode_arena = ValueDecodeArena { + peers: &header.peers, + keys, + }; + let positions = PositionArena::decode_v2(&positions)?; + let positions = positions.try_parse_to_positions()?; + let cids: &Vec = header.cids.get_or_try_init(|| { + ContainerArena::decode(&cids)? + .iter() + .map(|x| x.as_container_id(&decode_arena)) + .try_collect() + })?; + let mut value_reader = ValueReader::new(&values); + let encoded_ops_iters = serde_columnar::iter_from_bytes::(&ops)?; + let op_iter = encoded_ops_iters.ops; + let encoded_delete_id_starts: EncodedDeleteStartIds = if delete_start_ids.is_empty() { + EncodedDeleteStartIds { + delete_start_ids: Vec::new(), + } + } else { + serde_columnar::from_bytes(&delete_start_ids)? + }; + let mut del_iter = encoded_delete_id_starts + .delete_start_ids + .into_iter() + .map(Ok); + for i in 0..n_changes { + let commit_msg: Option> = { + let len = *commit_msg_lens + .get(i) + .ok_or(LoroError::DecodeDataCorruptionError)?; + if len == 0 { + None + } else { + let end = commit_msg_index + .checked_add(len) + .ok_or(LoroError::DecodeDataCorruptionError)?; + if commit_msgs.len() < end as usize { + return Err(LoroError::DecodeDataCorruptionError); + } + match std::str::from_utf8(&commit_msgs[commit_msg_index as usize..end as usize]) { + Ok(s) => { + commit_msg_index = end; + Some(Arc::from(s)) + } + Err(_) => { + tracing::error!("Invalid UTF8 String"); + return LoroResult::Err(LoroError::DecodeDataCorruptionError); + } + } + } + }; + changes.push(Change { + ops: Default::default(), + deps: header + .deps_groups + .get(i) + .ok_or(LoroError::DecodeDataCorruptionError)? + .clone(), + id: ID::new( + header.peer, + *header + .counters + .get(i) + .ok_or(LoroError::DecodeDataCorruptionError)?, + ), + lamport: *header + .lamports + .get(i) + .ok_or(LoroError::DecodeDataCorruptionError)?, + timestamp: *timestamps + .get(i) + .ok_or(LoroError::DecodeDataCorruptionError)? as Timestamp, + commit_msg, + }) + } + + let mut counter = first_counter as Counter; + let mut change_index = 0; + let peer = header.peer; + for op in op_iter { + let EncodedOp { + container_index, + prop, + value_type, + len, + } = op?; + let value = Value::decode( + ValueKind::from_u8(value_type), + &mut value_reader, + &decode_arena, + ID::new(peer, counter), + )?; + + let cid = cids + .get(container_index as usize) + .ok_or(LoroError::DecodeDataCorruptionError)?; + let content = decode_op( + cid, + value, + &mut del_iter, + shared_arena, + &decode_arena, + &positions, + prop, + ID::new(peer, counter), + )?; + + let c_idx = shared_arena.register_container(cid); + let op = Op { + counter, + container: c_idx, + content, + }; + + changes + .get_mut(change_index) + .ok_or(LoroError::DecodeDataCorruptionError)? + .ops + .push(op); + counter = counter + .checked_add(len as Counter) + .ok_or(LoroError::DecodeDataCorruptionError)?; + let next_counter = header + .counters + .get(change_index + 1) + .ok_or(LoroError::DecodeDataCorruptionError)?; + if counter >= *next_counter { + change_index += 1; + } + } + Ok(changes) +} + +#[cfg(test)] +mod test { + use std::{borrow::Cow, panic::AssertUnwindSafe}; + + use loro_common::ContainerType; + + use crate::encoding::value::{EncodedTreeMove, Value}; + use crate::{cursor::PosType, delta::DeltaValue, loro::ExportMode, LoroDoc}; + + use super::*; + + #[test] + fn decode_block_rejects_corrupt_payload_without_panic() { + let doc = LoroDoc::new_auto_commit(); + doc.get_map("map").insert("x", 100).unwrap(); + doc.commit_then_renew(); + + let oplog = doc.oplog.lock(); + let mut changes = Vec::new(); + oplog + .change_store() + .visit_all_changes(&mut |change| changes.push(change.clone())); + let block_bytes = encode_block(&changes, &oplog.arena); + let mut encoded: EncodedBlock = postcard::from_bytes(&block_bytes).unwrap(); + encoded.change_meta = Cow::Owned(vec![0x80]); + let corrupt = postcard::to_allocvec(&encoded).unwrap(); + + let result = std::panic::catch_unwind(AssertUnwindSafe(|| { + decode_block(&corrupt, &oplog.arena, None) + })); + assert!(result.is_ok(), "corrupt block payload should not panic"); + assert!(result.unwrap().is_err()); + } + + #[test] + fn tree_move_payload_is_rejected_without_panic() { + let arena = SharedArena::new(); + let value_arena = ValueDecodeArena { + peers: &[1], + keys: &[], + }; + let cid = ContainerID::Root { + name: "tree".into(), + container_type: ContainerType::Tree, + }; + let value = Value::TreeMove(EncodedTreeMove { + target_idx: 0, + is_parent_null: true, + parent_idx: 0, + position: 0, + }); + let mut deletes = std::iter::empty(); + + let result = std::panic::catch_unwind(AssertUnwindSafe(|| { + decode_op( + &cid, + value, + &mut deletes, + &arena, + &value_arena, + &[], + 0, + ID::new(1, 0), + ) + })); + + assert!(result.is_ok(), "tree move payload should not panic"); + assert!(matches!( + result.unwrap(), + Err(LoroError::DecodeDataCorruptionError) + )); + } + + #[test] + pub fn encode_single_text_edit() { + let doc = LoroDoc::new(); + doc.start_auto_commit(); + doc.get_map("map").insert("x", 100).unwrap(); + // doc.get_text("text").insert(0, "Hi").unwrap(); + // let node = doc.get_tree("tree").create(None).unwrap(); + // doc.get_tree("tree").create(node).unwrap(); + diagnose(&doc); + doc.get_map("map").insert("y", 20).unwrap(); + diagnose(&doc); + doc.get_map("map").insert("z", 1000).unwrap(); + diagnose(&doc); + doc.get_text("text") + .insert(0, "Hello", PosType::Unicode) + .unwrap(); + diagnose(&doc); + doc.get_text("text") + .insert(2, "He", PosType::Unicode) + .unwrap(); + diagnose(&doc); + doc.get_text("text").delete(1, 4, PosType::Unicode).unwrap(); + diagnose(&doc); + doc.get_text("text").delete(0, 2, PosType::Unicode).unwrap(); + diagnose(&doc); + } + + fn diagnose(doc: &LoroDoc) { + let bytes = doc.export(ExportMode::all_updates()).unwrap(); + println!("Old Update bytes {:?}", dev_utils::ByteSize(bytes.length())); + + let bytes = doc.export(ExportMode::all_updates()).unwrap(); + println!("Update bytes {:?}", dev_utils::ByteSize(bytes.length())); + // assert!(bytes.len() < 30); + + let bytes = doc.export(ExportMode::Snapshot).unwrap(); + println!("Snapshot bytes {:?}", dev_utils::ByteSize(bytes.length())); + // assert!(bytes.len() < 30); + + let json = doc.export_json_updates(&Default::default(), &doc.oplog_vv(), true); + let json_string = serde_json::to_string(&json.changes).unwrap(); + println!( + "JSON string bytes {:?}", + dev_utils::ByteSize(json_string.len()) + ); + let bytes = postcard::to_allocvec(&json.changes).unwrap(); + println!("JSON update bytes {:?}", dev_utils::ByteSize(bytes.len())); + println!("\n\n") + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/oplog/change_store/block_meta_encode.rs b/3rdparty/loro/crates/loro-internal/src/oplog/change_store/block_meta_encode.rs new file mode 100644 index 0000000000..ef9695f756 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/oplog/change_store/block_meta_encode.rs @@ -0,0 +1,247 @@ +use loro_common::{Counter, Lamport, LoroError, LoroResult, PeerID, ID}; +use once_cell::sync::OnceCell; +use rle::HasLength; +use serde_columnar::{ + AnyRleDecoder, AnyRleEncoder, BoolRleDecoder, BoolRleEncoder, DeltaOfDeltaDecoder, + DeltaOfDeltaEncoder, +}; + +use crate::{change::Change, encoding::value_register::ValueRegister, version::Frontiers}; + +use super::block_encode::ChangesBlockHeader; + +pub(crate) fn encode_changes( + block: &[Change], + peer_register: &mut ValueRegister, +) -> (Vec, Vec) { + let peer = block[0].peer(); + let mut timestamp_encoder = DeltaOfDeltaEncoder::new(); + let mut lamport_encoder = DeltaOfDeltaEncoder::new(); + let mut commit_msg_len_encoder = AnyRleEncoder::::new(); + let mut commit_msgs = String::new(); + let mut dep_self_encoder = BoolRleEncoder::new(); + let mut dep_len_encoder = AnyRleEncoder::::new(); + let mut encoded_deps = EncodedDeps { + peer_idx: AnyRleEncoder::new(), + counter: DeltaOfDeltaEncoder::new(), + }; + // First Counter + Change Len + let mut lengths_bytes = Vec::new(); + let mut counter = vec![]; + + for (i, c) in block.iter().enumerate() { + counter.push(c.id.counter); + let is_last = i == block.len() - 1; + if !is_last { + leb128::write::unsigned(&mut lengths_bytes, c.atom_len() as u64).unwrap(); + lamport_encoder.append(c.lamport() as i64).unwrap(); + } + timestamp_encoder.append(c.timestamp()).unwrap(); + if let Some(msg) = c.commit_msg.as_ref() { + commit_msg_len_encoder.append(msg.len() as u32).unwrap(); + commit_msgs.push_str(msg); + } else { + commit_msg_len_encoder.append(0).unwrap(); + } + + let mut dep_on_self = false; + for dep in c.deps().iter() { + if dep.peer == peer { + dep_on_self = true; + } else { + let peer_idx = peer_register.register(&dep.peer); + encoded_deps.peer_idx.append(peer_idx as u32).unwrap(); + encoded_deps.counter.append(dep.counter as i64).unwrap(); + } + } + + dep_self_encoder.append(dep_on_self).unwrap(); + dep_len_encoder + .append(if dep_on_self { + c.deps().len() - 1 + } else { + c.deps().len() + }) + .unwrap(); + } + + // TODO: capacity + let mut ans = Vec::with_capacity(block.len() * 15); + let _ = leb128::write::unsigned(&mut ans, peer_register.vec().len() as u64); + ans.extend(peer_register.vec().iter().flat_map(|p| p.to_le_bytes())); + ans.append(&mut lengths_bytes); + ans.append(&mut dep_self_encoder.finish().unwrap()); + ans.append(&mut dep_len_encoder.finish().unwrap()); + ans.append(&mut encoded_deps.peer_idx.finish().unwrap()); + ans.append(&mut encoded_deps.counter.finish().unwrap()); + ans.append(&mut lamport_encoder.finish().unwrap()); + + let mut t = timestamp_encoder.finish().unwrap(); + let mut cml = commit_msg_len_encoder.finish().unwrap(); + let mut cms = commit_msgs.into_bytes(); + let mut meta = Vec::with_capacity(t.len() + cml.len() + cms.len()); + meta.append(&mut t); + meta.append(&mut cml); + meta.append(&mut cms); + + (ans, meta) +} + +pub(crate) fn decode_changes_header( + mut bytes: &[u8], + n_changes: usize, + first_counter: Counter, + counter_len: Counter, + lamport_start: Lamport, + lamport_len: Lamport, +) -> LoroResult { + if n_changes == 0 { + return Err(LoroError::DecodeError( + "Decode block error: empty change block".into(), + )); + } + + let mut this_counter = first_counter; + let peer_num = leb128::read::unsigned(&mut bytes) + .map_err(|e| LoroError::DecodeError(format!("Decode block error {e}").into_boxed_str()))? + as usize; + let peers_bytes_len = peer_num + .checked_mul(8) + .ok_or_else(|| LoroError::DecodeError("Decode block error: peer length overflow".into()))?; + if peer_num == 0 || bytes.len() < peers_bytes_len { + return Err(LoroError::DecodeError( + "Decode block error: invalid peer table".into(), + )); + } + let mut peers = Vec::with_capacity(peer_num); + for i in 0..peer_num { + let start = 8 * i; + let peer_id = PeerID::from_le_bytes( + bytes[start..start + 8] + .try_into() + .expect("peer byte range should be checked"), + ); + peers.push(peer_id); + } + let mut bytes = &bytes[peers_bytes_len..]; + + // ┌───────────────────┬──────────────────────────────────────────┐ │ + // │ LEB First Counter │ N LEB128 Change AtomLen │◁───┼───── Important metadata + // └───────────────────┴──────────────────────────────────────────┘ │ + + let mut lengths = Vec::with_capacity(n_changes); + for _ in 0..n_changes - 1 { + lengths.push(leb128::read::unsigned(&mut bytes).map_err(|e| { + LoroError::DecodeError(format!("Decode block error {e}").into_boxed_str()) + })? as Counter); + } + let known_len = lengths.iter().try_fold(0i32, |acc, len| { + acc.checked_add(*len).ok_or_else(|| { + LoroError::DecodeError("Decode block error: counter length overflow".into()) + }) + })?; + lengths.push(counter_len.checked_sub(known_len).ok_or_else(|| { + LoroError::DecodeError("Decode block error: invalid counter length".into()) + })?); + + // ┌───────────────────┬────────────────────────┬─────────────────┐ │ + // │N DepOnSelf BoolRle│ N Delta Rle Deps Lens │ N Dep IDs │◁───┘ + // └───────────────────┴────────────────────────┴─────────────────┘ + + let dep_self_decoder = BoolRleDecoder::new(bytes); + let (dep_self, bytes) = dep_self_decoder + .take_n_finalize(n_changes) + .map_err(|_| LoroError::DecodeError("Decode block error: invalid deps".into()))?; + let dep_len_decoder = AnyRleDecoder::::new(bytes); + let (deps_len, bytes) = dep_len_decoder + .take_n_finalize(n_changes) + .map_err(|_| LoroError::DecodeError("Decode block error: invalid deps".into()))?; + let other_dep_num = deps_len.iter().sum::(); + let dep_peer_decoder = AnyRleDecoder::::new(bytes); + let (dep_peers, bytes) = dep_peer_decoder + .take_n_finalize(other_dep_num) + .map_err(|_| LoroError::DecodeError("Decode block error: invalid deps".into()))?; + let mut deps_peers_iter = dep_peers.into_iter(); + let dep_counter_decoder = DeltaOfDeltaDecoder::::new(bytes) + .map_err(|_| LoroError::DecodeError("Decode block error: invalid deps".into()))?; + let (dep_counters, bytes) = dep_counter_decoder + .take_n_finalize(other_dep_num) + .map_err(|_| LoroError::DecodeError("Decode block error: invalid deps".into()))?; + let mut deps_counters_iter = dep_counters.into_iter(); + let mut deps = Vec::with_capacity(n_changes); + for i in 0..n_changes { + let mut f = Frontiers::default(); + if dep_self[i] { + let dep_counter = this_counter.checked_sub(1).ok_or_else(|| { + LoroError::DecodeError("Decode block error: invalid self dependency".into()) + })?; + f.push(ID::new(peers[0], dep_counter)) + } + + let len = deps_len[i]; + for _ in 0..len { + let peer_idx = deps_peers_iter + .next() + .ok_or_else(|| LoroError::DecodeError("Decode block error: invalid deps".into()))?; + let peer = peers.get(peer_idx).copied().ok_or_else(|| { + LoroError::DecodeError("Decode block error: invalid peer index".into()) + })?; + let counter = deps_counters_iter + .next() + .ok_or_else(|| LoroError::DecodeError("Decode block error: invalid deps".into()))? + as Counter; + f.push(ID::new(peer, counter)); + } + + deps.push(f); + this_counter = this_counter + .checked_add(lengths[i]) + .ok_or_else(|| LoroError::DecodeError("Decode block error: counter overflow".into()))?; + } + let mut counters = Vec::with_capacity(n_changes); + let mut last = first_counter; + for len in lengths.iter() { + counters.push(last); + last = last + .checked_add(*len) + .ok_or_else(|| LoroError::DecodeError("Decode block error: counter overflow".into()))?; + } + + let lamport_decoder = DeltaOfDeltaDecoder::new(bytes) + .map_err(|_| LoroError::DecodeError("Decode block error: invalid lamport".into()))?; + let (mut lamports, rest) = lamport_decoder + .take_n_finalize(n_changes.saturating_sub(1)) + .map_err(|_| LoroError::DecodeError("Decode block error: invalid lamport".into()))?; + // the last lamport + let last_len = *lengths.last().unwrap_or(&0) as u32; + let last_lamport = lamport_start + .checked_add(lamport_len) + .and_then(|end| end.checked_sub(last_len)) + .ok_or_else(|| LoroError::DecodeError("Decode block error: invalid lamport".into()))?; + lamports.push(last_lamport as Lamport); + + // we need counter range, so encode + counters.push( + first_counter + .checked_add(counter_len) + .ok_or_else(|| LoroError::DecodeError("Decode block error: counter overflow".into()))?, + ); + debug_assert!(rest.is_empty()); + + Ok(ChangesBlockHeader { + peer: peers[0], + counter: first_counter, + n_changes, + peers, + counters, + deps_groups: deps, + lamports, + keys: OnceCell::new(), + cids: OnceCell::new(), + }) +} + +struct EncodedDeps { + peer_idx: AnyRleEncoder, + counter: DeltaOfDeltaEncoder, +} diff --git a/3rdparty/loro/crates/loro-internal/src/oplog/change_store/iter.rs b/3rdparty/loro/crates/loro-internal/src/oplog/change_store/iter.rs new file mode 100644 index 0000000000..153eaf0a2b --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/oplog/change_store/iter.rs @@ -0,0 +1,139 @@ +#![allow(dead_code)] + +use std::{ + collections::{BinaryHeap, VecDeque}, + sync::Arc, +}; + +use loro_common::{CounterSpan, HasIdSpan}; + +use crate::{OpLog, VersionVector}; + +use super::{BlockChangeRef, ChangesBlock}; + +pub(crate) struct PeerChangesIter { + blocks: VecDeque>, + change_index: usize, + counter_range: CounterSpan, +} + +impl PeerChangesIter { + fn new_change_iter_rev(mut changes: VecDeque>, counter: CounterSpan) -> Self { + let mut index = changes + .back() + .map(|x| x.content().len_changes().saturating_sub(1)) + .unwrap_or(0); + + while let Some(block) = changes.back() { + if let Some(change) = block.content().try_changes().unwrap().get(index) { + if change.id.counter < counter.end { + break; + } + } else if index == 0 { + changes.pop_back(); + } else { + index -= 1; + } + } + + PeerChangesIter { + blocks: changes, + change_index: index, + counter_range: counter, + } + } + + fn current_weight(&self) -> i32 { + if self.blocks.is_empty() { + return 0; + } + + self.blocks + .back() + .map(|x| { + x.content() + .try_changes() + .unwrap() + .get(self.change_index) + .unwrap() + .lamport as i32 + }) + .unwrap_or(0) + } +} + +impl Ord for PeerChangesIter { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.current_weight().cmp(&other.current_weight()) + } +} + +impl PartialOrd for PeerChangesIter { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl PartialEq for PeerChangesIter { + fn eq(&self, other: &Self) -> bool { + self.current_weight() == other.current_weight() + } +} + +impl Eq for PeerChangesIter {} + +impl Iterator for PeerChangesIter { + type Item = BlockChangeRef; + + fn next(&mut self) -> Option { + if self.blocks.is_empty() { + return None; + } + + let c = BlockChangeRef { + block: self.blocks.back().unwrap().clone(), + change_index: self.change_index, + }; + + if c.id_last().counter < self.counter_range.start { + return None; + } + + let ans = Some(c); + if self.change_index == 0 { + self.blocks.pop_back(); + } else { + self.change_index -= 1; + } + + ans + } +} + +pub(crate) struct MergedChangeIter { + heap: BinaryHeap, +} + +impl Iterator for MergedChangeIter { + type Item = BlockChangeRef; + + fn next(&mut self) -> Option { + let mut iter = self.heap.pop()?; + let ans = iter.next(); + self.heap.push(iter); + ans + } +} + +impl MergedChangeIter { + pub fn new_change_iter_rev(oplog: &OpLog, from: &VersionVector, to: &VersionVector) -> Self { + let mut heap = BinaryHeap::new(); + for span in to.sub_iter(from) { + let blocks = oplog.change_store.get_blocks_in_range(span); + let iter = PeerChangesIter::new_change_iter_rev(blocks, span.counter); + heap.push(iter); + } + + Self { heap } + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/oplog/loro_dag.rs b/3rdparty/loro/crates/loro-internal/src/oplog/loro_dag.rs new file mode 100644 index 0000000000..3772d92990 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/oplog/loro_dag.rs @@ -0,0 +1,1475 @@ +use crate::change::{Change, Lamport}; +use crate::dag::{Dag, DagNode}; +use crate::id::{Counter, ID}; +use crate::span::{HasId, HasLamport}; +use crate::sync::Mutex; +use crate::version::{shrink_frontiers, Frontiers, ImVersionVector, VersionVector}; +use loro_common::{HasCounter, HasCounterSpan, HasIdSpan, HasLamportSpan, PeerID}; +use once_cell::sync::OnceCell; +use rle::{HasIndex, HasLength, Mergable, Sliceable}; +use rustc_hash::FxHashSet; +use smallvec::SmallVec; +use std::cmp::Ordering; +use std::collections::{BTreeMap, BTreeSet, BinaryHeap}; +use std::fmt::Display; +use std::ops::{ControlFlow, Deref}; +use std::sync::atomic::{AtomicBool, Ordering as AtomicOrdering}; +use std::sync::Arc; +use tracing::instrument; + +use super::change_store::BatchDecodeInfo; +use super::ChangeStore; + +/// [AppDag] maintains the causal graph of the app. +/// It's faster to answer the question like what's the LCA version +#[derive(Debug)] +pub struct AppDag { + change_store: ChangeStore, + /// It only contains nodes that are already parsed. + /// + /// - All the unparsed op ids must be included in `unparsed_vv`. + /// - All the parsed and unparsed op ids must be included in `vv`. + map: Mutex>, + /// The latest known frontiers + frontiers: Frontiers, + /// The latest known version vector + vv: VersionVector, + /// The earliest known frontiers + shallow_since_frontiers: Frontiers, + /// The deps of the shallow frontiers + shallow_root_frontiers_deps: Frontiers, + /// The vv of shallow_frontiers_deps + shallow_since_vv: ImVersionVector, + /// Ops included in the version vector but not parsed yet + /// + /// # Invariants + /// + /// - `vv` >= `unparsed_vv` + unparsed_vv: Mutex, + /// It's a set of points which are deps of some parsed ops. + /// But the ops in this set are not parsed yet. When they are parsed, + /// we need to make sure it breaks at the given point. + unhandled_dep_points: Mutex>, + pending_txn_node: Option, + import_rollback_has_journal: AtomicBool, + import_rollback: Mutex>, +} + +#[derive(Debug)] +pub(crate) struct AppDagRollback { + frontiers: Frontiers, + vv: VersionVector, + unparsed_vv: VersionVector, + shallow_since_frontiers: Frontiers, + shallow_root_frontiers_deps: Frontiers, + shallow_since_vv: ImVersionVector, + pending_txn_node: Option, + map_entries_before_mutation: BTreeMap>, + unhandled_dep_point_log: Vec, +} + +#[derive(Debug)] +enum UnhandledDepPointLog { + Added(ID), + Removed(ID), +} + +#[derive(Debug, Clone)] +pub struct AppDagNode { + inner: Arc, +} + +impl Deref for AppDagNode { + type Target = AppDagNodeInner; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl AppDagNode { + pub fn new(inner: AppDagNodeInner) -> Self { + Self { + inner: Arc::new(inner), + } + } +} + +#[derive(Debug, Clone)] +pub struct AppDagNodeInner { + pub(crate) peer: PeerID, + pub(crate) cnt: Counter, + pub(crate) lamport: Lamport, + pub(crate) deps: Frontiers, + pub(crate) vv: OnceCell, + /// A flag indicating whether any other nodes from a different peer depend on this node. + /// The calculation of frontiers is based on the property that a node does not depend + /// on the middle of other nodes. + pub(crate) has_succ: bool, + pub(crate) len: usize, +} + +impl From for AppDagNode { + fn from(inner: AppDagNodeInner) -> Self { + AppDagNode { + inner: Arc::new(inner), + } + } +} + +impl AppDag { + pub(super) fn new(change_store: ChangeStore) -> Self { + Self { + change_store, + map: Mutex::new(BTreeMap::new()), + frontiers: Frontiers::default(), + vv: VersionVector::default(), + unparsed_vv: Mutex::new(VersionVector::default()), + unhandled_dep_points: Mutex::new(BTreeSet::new()), + shallow_since_frontiers: Default::default(), + shallow_root_frontiers_deps: Default::default(), + shallow_since_vv: Default::default(), + pending_txn_node: None, + import_rollback_has_journal: AtomicBool::new(false), + import_rollback: Mutex::new(None), + } + } + + pub fn frontiers(&self) -> &Frontiers { + &self.frontiers + } + + pub fn vv(&self) -> &VersionVector { + &self.vv + } + + pub fn shallow_since_vv(&self) -> &ImVersionVector { + &self.shallow_since_vv + } + + pub fn shallow_since_frontiers(&self) -> &Frontiers { + &self.shallow_since_frontiers + } + + pub(crate) fn begin_import_rollback(&mut self) { + let old_vv_is_empty = self.vv.is_empty(); + let mut rollback = self.import_rollback.lock(); + debug_assert!(rollback.is_none()); + *rollback = Some(AppDagRollback { + frontiers: self.frontiers.clone(), + vv: self.vv.clone(), + unparsed_vv: self.unparsed_vv.lock().clone(), + shallow_since_frontiers: self.shallow_since_frontiers.clone(), + shallow_root_frontiers_deps: self.shallow_root_frontiers_deps.clone(), + shallow_since_vv: self.shallow_since_vv.clone(), + pending_txn_node: self.pending_txn_node.clone(), + map_entries_before_mutation: BTreeMap::new(), + unhandled_dep_point_log: Vec::new(), + }); + self.import_rollback_has_journal + .store(!old_vv_is_empty, AtomicOrdering::Relaxed); + } + + pub(crate) fn commit_import_rollback(&mut self) { + self.import_rollback_has_journal + .store(false, AtomicOrdering::Relaxed); + *self.import_rollback.lock() = None; + } + + pub(crate) fn rollback_import(&mut self) { + self.import_rollback_has_journal + .store(false, AtomicOrdering::Relaxed); + let Some(checkpoint) = self.import_rollback.lock().take() else { + return; + }; + + let imported_spans = self.vv.sub_iter(&checkpoint.vv).collect::>(); + let mut map = self.map.lock(); + for span in imported_spans { + let start = ID::new(span.peer, span.counter.start); + let end = ID::new(span.peer, span.counter.end); + let keys = map.range(start..end).map(|(id, _)| *id).collect::>(); + for key in keys { + map.remove(&key); + } + } + + for (id, node) in checkpoint.map_entries_before_mutation { + if let Some(node) = node { + map.insert(id, node); + } else { + map.remove(&id); + } + } + drop(map); + + self.frontiers = checkpoint.frontiers; + self.vv = checkpoint.vv.clone(); + self.shallow_since_frontiers = checkpoint.shallow_since_frontiers; + self.shallow_root_frontiers_deps = checkpoint.shallow_root_frontiers_deps; + self.shallow_since_vv = checkpoint.shallow_since_vv; + *self.unparsed_vv.lock() = checkpoint.unparsed_vv; + + let mut unhandled_dep_points = self.unhandled_dep_points.lock(); + if checkpoint.vv.is_empty() { + unhandled_dep_points.clear(); + } + for item in checkpoint.unhandled_dep_point_log.into_iter().rev() { + match item { + UnhandledDepPointLog::Added(id) => { + unhandled_dep_points.remove(&id); + } + UnhandledDepPointLog::Removed(id) => { + unhandled_dep_points.insert(id); + } + } + } + drop(unhandled_dep_points); + + self.pending_txn_node = checkpoint.pending_txn_node; + } + + fn record_map_entry_before_mutation(&self, map: &BTreeMap, id: ID) { + if !self + .import_rollback_has_journal + .load(AtomicOrdering::Relaxed) + { + return; + } + + self.record_map_entry_before_replacement(id, map.get(&id).cloned()); + } + + fn record_map_entry_before_replacement(&self, id: ID, old: Option) { + if !self + .import_rollback_has_journal + .load(AtomicOrdering::Relaxed) + { + return; + } + + let mut rollback = self.import_rollback.lock(); + let Some(rollback) = rollback.as_mut() else { + return; + }; + + let old_end = rollback.vv.get(&id.peer).copied().unwrap_or(0); + if id.counter >= old_end { + return; + } + + rollback + .map_entries_before_mutation + .entry(id) + .or_insert(old); + } + + fn record_unhandled_dep_point_added(&self, id: ID) { + if !self + .import_rollback_has_journal + .load(AtomicOrdering::Relaxed) + { + return; + } + + if let Some(rollback) = self.import_rollback.lock().as_mut() { + rollback + .unhandled_dep_point_log + .push(UnhandledDepPointLog::Added(id)); + } + } + + fn record_unhandled_dep_point_removed(&self, id: ID) { + if !self + .import_rollback_has_journal + .load(AtomicOrdering::Relaxed) + { + return; + } + + if let Some(rollback) = self.import_rollback.lock().as_mut() { + rollback + .unhandled_dep_point_log + .push(UnhandledDepPointLog::Removed(id)); + } + } + + pub fn is_empty(&self) -> bool { + self.vv.is_empty() + } + + #[tracing::instrument(skip_all, name = "handle_new_change")] + pub(super) fn handle_new_change( + &mut self, + change: &Change, + from_local: bool, + rollback_old_vv: Option<&VersionVector>, + ) { + let len = change.content_len(); + self.update_version_on_new_change(change, from_local); + #[cfg(debug_assertions)] + { + let unhandled_dep_points = self.unhandled_dep_points.lock(); + let c = unhandled_dep_points + .range(change.id_start()..change.id_end()) + .count(); + assert!(c == 0); + } + + let mut inserted = false; + if change.deps_on_self() { + let record_old_node_before_merge = rollback_old_vv + .and_then(|vv| vv.get(&change.id.peer).copied()) + .filter(|old_end| *old_end == change.id.counter); + // We may not need to push new element to dag because it only depends on itself + inserted = + self.with_last_mut_of_peer(change.id.peer, record_old_node_before_merge, |last| { + let (_, last) = last.unwrap(); + if last.has_succ { + // Don't merge the node if there are other nodes depending on it + return false; + } + + assert_eq!(last.peer, change.id.peer, "peer id is not the same"); + assert_eq!( + last.cnt + last.len as Counter, + change.id.counter, + "counter is not continuous" + ); + assert_eq!( + last.lamport + last.len as Lamport, + change.lamport, + "lamport is not continuous" + ); + let last = Arc::make_mut(&mut last.inner); + last.len = (change.id.counter - last.cnt) as usize + len; + last.has_succ = false; + true + }); + } + + if !inserted { + let node: AppDagNode = AppDagNodeInner { + vv: OnceCell::new(), + peer: change.id.peer, + cnt: change.id.counter, + lamport: change.lamport, + deps: change.deps.clone(), + has_succ: false, + len, + } + .into(); + + let mut map = self.map.lock(); + map.insert(node.id_start(), node); + self.handle_deps_break_points(change.deps.iter(), change.id.peer, Some(&mut map)); + } + } + + fn try_with_node_mut( + &self, + map: &mut BTreeMap, + id: ID, + f: impl FnOnce(Option<(ID, &mut AppDagNode)>) -> R, + ) -> R { + let x = map.range_mut(..=id).next_back(); + if let Some((node_id, node)) = x { + if node.contains_id(id) { + f(Some((*node_id, node))) + } else { + f(None) + } + } else { + f(None) + } + } + + /// If the lamport of change can be calculated, return Ok, otherwise, Err + pub(crate) fn calc_unknown_lamport_change(&self, change: &mut Change) -> Result<(), ()> { + for dep in change.deps.iter() { + match self.get_lamport(&dep) { + Some(lamport) => { + change.lamport = change.lamport.max(lamport + 1); + } + None => return Err(()), + } + } + Ok(()) + } + + pub(crate) fn find_deps_of_id(&self, id: ID) -> Frontiers { + let Some(node) = self.get(id) else { + return Frontiers::default(); + }; + + let offset = id.counter - node.cnt; + if offset == 0 { + node.deps.clone() + } else { + ID::new(id.peer, node.cnt + offset - 1).into() + } + } + + pub(crate) fn with_last_mut_of_peer( + &mut self, + peer: PeerID, + record_if_before: Option, + f: impl FnOnce(Option<(ID, &mut AppDagNode)>) -> R, + ) -> R { + self.lazy_load_last_of_peer(peer); + let mut binding = self.map.lock(); + let last = binding + .range_mut(..=ID::new(peer, Counter::MAX)) + .next_back() + .map(|(id, v)| { + if record_if_before.is_some_and(|old_end| id.counter < old_end) { + self.record_map_entry_before_replacement(*id, Some(v.clone())); + } + (*id, v) + }); + f(last) + } + + fn update_version_on_new_change(&mut self, change: &Change, from_local: bool) { + if from_local { + assert!(self.pending_txn_node.take().is_some()); + assert_eq!( + self.vv.get(&change.id.peer).copied().unwrap_or(0), + change.ctr_end() + ); + } else { + let id_last = change.id_last(); + self.frontiers + .update_frontiers_on_new_change(id_last, &change.deps); + assert!(self.pending_txn_node.is_none()); + assert_eq!( + self.vv.get(&change.id.peer).copied().unwrap_or(0), + change.id.counter + ); + self.vv.extend_to_include_last_id(id_last); + } + } + + pub(super) fn lazy_load_last_of_peer(&mut self, peer: u64) { + let unparsed_vv = self.unparsed_vv.lock(); + if !unparsed_vv.contains_key(&peer) || self.vv[&peer] >= unparsed_vv[&peer] { + return; + } + + let Some(nodes) = self.change_store.get_last_dag_nodes_for_peer(peer) else { + panic!("unparsed vv don't match with change store. Peer:{peer} is not in change store") + }; + + self.lazy_load_nodes_internal(nodes, peer, None); + } + + fn lazy_load_nodes_internal( + &self, + nodes: Vec, + peer: u64, + map_input: Option<&mut BTreeMap>, + ) { + assert!(!nodes.is_empty()); + let mut map_guard = None; + let map = map_input.unwrap_or_else(|| { + map_guard = Some(self.map.lock()); + map_guard.as_mut().unwrap() + }); + let new_dag_start_counter_for_the_peer = nodes[0].cnt; + let nodes_cnt_end = nodes.last().unwrap().ctr_end(); + let mut unparsed_vv = self.unparsed_vv.lock(); + let end_counter = unparsed_vv[&peer]; + assert!(end_counter <= nodes_cnt_end); + let mut deps_on_others = Vec::new(); + let mut break_point_set = self.unhandled_dep_points.lock(); + for mut node in nodes { + if node.cnt >= end_counter { + // skip already parsed nodes + break; + } + + if node.cnt + node.len as Counter > end_counter { + node = node.slice(0, (end_counter - node.cnt) as usize); + // This is unlikely to happen + } + + for dep in node.deps.iter() { + if dep.peer != peer { + deps_on_others.push(dep); + } + } + + // PERF: we can try to merge the node with the previous node + let break_point_ends: Vec<_> = break_point_set + .range(node.id_start()..node.id_end()) + .map(|id| (id.counter - node.cnt) as usize + 1) + .collect(); + if break_point_ends.is_empty() { + self.record_map_entry_before_mutation(map, node.id_start()); + map.insert(node.id_start(), node); + } else { + let mut slice_start = 0; + for slice_end in break_point_ends.iter().copied() { + let mut slice_node = node.slice(slice_start, slice_end); + let inner = Arc::make_mut(&mut slice_node.inner); + inner.has_succ = true; + self.record_map_entry_before_mutation(map, slice_node.id_start()); + map.insert(slice_node.id_start(), slice_node); + slice_start = slice_end; + } + + let last_break_point = break_point_ends.last().copied().unwrap(); + if last_break_point != node.len { + let slice_node = node.slice(last_break_point, node.len); + self.record_map_entry_before_mutation(map, slice_node.id_start()); + map.insert(slice_node.id_start(), slice_node); + } + + for break_point in break_point_ends.into_iter() { + let id = node.id_start().inc(break_point as Counter - 1); + if break_point_set.remove(&id) { + self.record_unhandled_dep_point_removed(id); + } + } + } + } + + if new_dag_start_counter_for_the_peer == 0 { + unparsed_vv.remove(&peer); + } else { + unparsed_vv.insert(peer, new_dag_start_counter_for_the_peer); + } + drop(unparsed_vv); + drop(break_point_set); + self.handle_deps_break_points(deps_on_others.iter().copied(), peer, Some(map)); + } + + fn handle_deps_break_points( + &self, + ids: impl IntoIterator, + skip_peer: PeerID, + map: Option<&mut BTreeMap>, + ) { + let mut map_guard = None; + let map = map.unwrap_or_else(|| { + map_guard = Some(self.map.lock()); + map_guard.as_mut().unwrap() + }); + for id in ids { + if id.peer == skip_peer { + continue; + } + + let mut handled = false; + let ans = self.try_with_node_mut(map, id, |target| { + // We don't need to break the dag node if it's not loaded yet + let (target_id, target) = target?; + if target.ctr_last() == id.counter { + self.record_map_entry_before_replacement(target_id, Some(target.clone())); + let target = Arc::make_mut(&mut target.inner); + handled = true; + target.has_succ = true; + None + } else { + // We need to split the target node into two part + // so that we can ensure the new change depends on the + // last id of a dag node. + + let new_node = + target.slice(id.counter as usize - target.cnt as usize + 1, target.len); + self.record_map_entry_before_replacement(target_id, Some(target.clone())); + let target = Arc::make_mut(&mut target.inner); + target.len -= new_node.len; + Some(new_node) + } + }); + + if let Some(new_node) = ans { + self.record_map_entry_before_mutation(map, new_node.id_start()); + map.insert(new_node.id_start(), new_node); + } else if !handled { + let mut unhandled_dep_points = self.unhandled_dep_points.lock(); + if unhandled_dep_points.insert(id) { + self.record_unhandled_dep_point_added(id); + } + } + } + } + + fn ensure_lazy_load_node(&self, id: ID) { + if self.shallow_since_vv.includes_id(id) { + return; + } + + loop { + // We need to load all the dag nodes that has the same peer and greater counter than the given `id` + // Because we only record the end counter of the unparsed version on `unparsed_vv` + let unparsed_end = { + let unparsed_vv = self.unparsed_vv.lock(); + unparsed_vv.get(&id.peer).copied().unwrap_or(0) + }; + if unparsed_end <= id.counter { + return; + } + + let last_unparsed_id = ID::new(id.peer, unparsed_end - 1); + let Some(nodes) = self + .change_store + .get_dag_nodes_that_contains(last_unparsed_id) + else { + panic!("unparsed vv don't match with change store. Id:{id} is not in change store") + }; + + self.lazy_load_nodes_internal(nodes, id.peer, None); + } + } + + pub fn total_parsed_dag_node(&self) -> usize { + self.map.lock().len() + } + + pub(crate) fn set_version_by_fast_snapshot_import(&mut self, v: BatchDecodeInfo) { + assert!(self.vv.is_empty()); + *self.unparsed_vv.lock() = v.vv.clone(); + self.vv = v.vv; + self.frontiers = v.frontiers; + if let Some((vv, f)) = v.start_version { + if !f.is_empty() { + assert!(f.len() == 1); + let id = f.as_single().unwrap(); + let node = self.get(id).unwrap(); + assert!(node.cnt == id.counter); + self.shallow_root_frontiers_deps = node.deps.clone(); + } + self.shallow_since_frontiers = f; + self.shallow_since_vv = ImVersionVector::from_vv(&vv); + } + } + + /// This method is slow and should only be used for debugging and testing. + /// + /// It will check the following properties: + /// + /// 1. Counter is continuous + /// 2. A node always depends of the last ids of other nodes + /// 3. Lamport is correctly calculated + /// 4. VV for each node is correctly calculated + /// 5. Frontiers are correctly calculated + #[instrument(skip(self))] + pub fn check_dag_correctness(&self) { + { + // parse all nodes + let unparsed_vv = self.unparsed_vv.lock().clone(); + for (peer, cnt) in unparsed_vv.iter() { + if *cnt == 0 { + continue; + } + + let mut end_cnt = *cnt; + let init_counter = self.shallow_since_vv.get(peer).copied().unwrap_or(0); + while end_cnt > init_counter { + let cnt = end_cnt - 1; + self.ensure_lazy_load_node(ID::new(*peer, cnt)); + end_cnt = self.unparsed_vv.lock().get(peer).copied().unwrap_or(0); + } + } + + self.unparsed_vv.lock().clear(); + } + { + // check property 1: Counter is continuous + let map = self.map.lock(); + let mut last_end_id = ID::new(0, 0); + for (&id, node) in map.iter() { + let init_counter = self.shallow_since_vv.get(&id.peer).copied().unwrap_or(0); + if id.peer == last_end_id.peer { + assert!(id.counter == last_end_id.counter); + } else { + assert_eq!(id.counter, init_counter); + } + + last_end_id = id.inc(node.len as Counter); + } + } + { + // check property 2: A node always depends of the last ids of other nodes + let map = self.map.lock(); + check_always_dep_on_last_id(&map); + } + { + // check property 3: Lamport is correctly calculated + let map = self.map.lock(); + 'outer: for (_, node) in map.iter() { + let mut this_lamport = 0; + for dep in node.deps.iter() { + if self.shallow_since_vv.includes_id(dep) { + continue 'outer; + } + + let (_, dep_node) = map.range(..=dep).next_back().unwrap(); + this_lamport = this_lamport.max(dep_node.lamport_end()); + } + + assert_eq!(this_lamport, node.lamport); + } + } + { + // check property 4: VV for each node is correctly calculated + let map = self.map.lock().clone(); + 'outer: for (_, node) in map.iter() { + let actual_vv = self.ensure_vv_for(node); + let mut expected_vv = ImVersionVector::default(); + for dep in node.deps.iter() { + if self.shallow_since_vv.includes_id(dep) { + continue 'outer; + } + + let (_, dep_node) = map.range(..=dep).next_back().unwrap(); + self.ensure_vv_for(dep_node); + expected_vv.extend_to_include_vv(dep_node.vv.get().unwrap().iter()); + expected_vv.extend_to_include_last_id(dep); + } + + assert_eq!(actual_vv, expected_vv); + } + } + { + // check property 5: Frontiers are correctly calculated + let mut maybe_frontiers = FxHashSet::default(); + let map = self.map.lock(); + for (_, node) in map.iter() { + maybe_frontiers.insert(node.id_last()); + } + + for (_, node) in map.iter() { + for dep in node.deps.iter() { + maybe_frontiers.remove(&dep); + } + } + + let frontiers = self.frontiers.iter().collect::>(); + assert_eq!(maybe_frontiers, frontiers); + } + } + + pub(crate) fn can_export_shallow_snapshot_on(&self, deps: &Frontiers) -> bool { + for id in deps.iter() { + if !self.vv.includes_id(id) { + return false; + } + } + + if self.is_before_shallow_root(deps) { + return false; + } + + true + } + + pub(crate) fn is_before_shallow_root(&self, deps: &Frontiers) -> bool { + // trace!("Is on shallow history? deps={:?}", deps); + // trace!("self.shallow_since_vv {:?}", &self.shallow_since_vv); + // trace!("self.shallow_frontiers {:?}", &self.shallow_since_frontiers); + + if self.shallow_since_vv.is_empty() { + return false; + } + + if deps.is_empty() { + return true; + } + + if deps.iter().any(|x| self.shallow_since_vv.includes_id(x)) { + return true; + } + + if deps + .iter() + .any(|x| self.shallow_since_frontiers.contains(&x)) + { + return deps != &self.shallow_since_frontiers; + } + + false + } + + pub(crate) fn import_deps_before_shallow_root(&self, deps: &Frontiers) -> bool { + if self.shallow_since_vv.is_empty() { + return false; + } + + if deps.is_empty() { + return true; + } + + let shallow_vv = VersionVector::from_im_vv(&self.shallow_since_vv); + if let Some(vv) = self.frontiers_to_vv(deps) { + return !vv.includes_vv(&shallow_vv); + } + + // Import only needs to reject updates whose causal source is older than + // the shallow root. A dependency set that touches the retained boundary + // can still be a valid post-root update, even when the rest of the deps + // are imported later in the same batch. + if deps + .iter() + .any(|id| self.shallow_since_frontiers.contains(&id)) + { + return false; + } + + deps.iter().any(|id| self.shallow_since_vv.includes_id(id)) + } + + /// Travel the ancestors of the given id, and call the callback for each node + /// + /// It will travel the ancestors in the reverse order (from the greatest lamport to the smallest) + pub(crate) fn travel_ancestors( + &self, + id: ID, + f: &mut dyn FnMut(&AppDagNode) -> ControlFlow<()>, + ) { + struct PendingNode(AppDagNode); + impl PartialEq for PendingNode { + fn eq(&self, other: &Self) -> bool { + self.0.lamport_last() == other.0.lamport_last() && self.0.peer == other.0.peer + } + } + impl Eq for PendingNode {} + impl PartialOrd for PendingNode { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } + } + impl Ord for PendingNode { + fn cmp(&self, other: &Self) -> Ordering { + self.0 + .lamport_last() + .cmp(&other.0.lamport_last()) + .then_with(|| self.0.peer.cmp(&other.0.peer)) + } + } + + let mut visited = FxHashSet::default(); + let mut pending: BinaryHeap = BinaryHeap::new(); + pending.push(PendingNode(self.get(id).unwrap())); + while let Some(PendingNode(node)) = pending.pop() { + if f(&node).is_break() { + break; + } + + for dep in node.deps.iter() { + let Some(dep_node) = self.get(dep) else { + continue; + }; + if visited.contains(&dep_node.id_start()) { + continue; + } + + visited.insert(dep_node.id_start()); + pending.push(PendingNode(dep_node)); + } + } + } + + pub(crate) fn update_version_on_new_local_op( + &mut self, + deps: &Frontiers, + start_id: ID, + start_lamport: Lamport, + len: usize, + ) { + let last_id = start_id.inc(len as Counter - 1); + // PERF: we can cache this last_id - this is a hot path + self.vv.set_last(last_id); + self.frontiers.update_frontiers_on_new_change(last_id, deps); + match &mut self.pending_txn_node { + Some(node) => { + assert!( + node.peer == start_id.peer + && node.cnt + node.len as Counter == start_id.counter + && deps.len() == 1 + && deps.as_single().unwrap().peer == start_id.peer + ); + let inner = Arc::make_mut(&mut node.inner); + inner.len += len; + } + None => { + let node = AppDagNode { + inner: Arc::new(AppDagNodeInner { + peer: start_id.peer, + cnt: start_id.counter, + lamport: start_lamport, + deps: deps.clone(), + vv: OnceCell::new(), + has_succ: false, + len, + }), + }; + self.pending_txn_node = Some(node); + } + } + } + + pub(crate) fn latest_vv_contains_peer(&self, peer: PeerID) -> bool { + self.vv.contains_key(&peer) && *self.vv.get(&peer).unwrap() > 0 + } +} + +fn check_always_dep_on_last_id(map: &BTreeMap) { + for (_, node) in map.iter() { + for dep in node.deps.iter() { + let Some((&dep_id, dep_node)) = map.range(..=dep).next_back() else { + // It's shallow + continue; + }; + assert_eq!(dep_node.id_start(), dep_id); + if dep_node.contains_id(dep) { + assert_eq!(dep_node.id_last(), dep); + } + } + } +} + +impl HasIndex for AppDagNode { + type Int = Counter; + fn get_start_index(&self) -> Self::Int { + self.cnt + } + + fn get_end_index(&self) -> Self::Int { + self.cnt + self.len as Counter + } +} + +impl Sliceable for AppDagNode { + fn slice(&self, from: usize, to: usize) -> Self { + AppDagNodeInner { + peer: self.peer, + cnt: self.cnt + from as Counter, + lamport: self.lamport + from as Lamport, + deps: if from > 0 { + Frontiers::from_id(self.id_start().inc(from as Counter - 1)) + } else { + self.deps.clone() + }, + vv: if let Some(vv) = self.vv.get() { + let mut new = vv.clone(); + new.insert(self.peer, self.cnt + from as Counter); + OnceCell::with_value(new) + } else { + OnceCell::new() + }, + has_succ: if to == self.len { self.has_succ } else { true }, + len: to - from, + } + .into() + } +} + +impl HasId for AppDagNode { + fn id_start(&self) -> ID { + ID { + peer: self.peer, + counter: self.cnt, + } + } +} + +impl HasCounter for AppDagNode { + fn ctr_start(&self) -> Counter { + self.cnt + } +} + +impl HasLength for AppDagNode { + fn atom_len(&self) -> usize { + self.len + } + + fn content_len(&self) -> usize { + self.len + } +} + +impl Mergable for AppDagNode { + fn is_mergable(&self, other: &Self, _conf: &()) -> bool + where + Self: Sized, + { + !self.has_succ + && self.peer == other.peer + && self.cnt + self.len as Counter == other.cnt + && other.deps.len() == 1 + && self.lamport + self.len as Lamport == other.lamport + && other.deps.as_single().unwrap().peer == self.peer + } + + fn merge(&mut self, other: &Self, _conf: &()) + where + Self: Sized, + { + assert_eq!( + other.deps.as_single().unwrap().counter, + self.cnt + self.len as Counter - 1 + ); + let this = Arc::make_mut(&mut self.inner); + this.len += other.len; + this.has_succ = other.has_succ; + } +} + +impl HasLamport for AppDagNode { + fn lamport(&self) -> Lamport { + self.lamport + } +} + +impl DagNode for AppDagNode { + fn deps(&self) -> &Frontiers { + &self.deps + } +} + +impl Dag for AppDag { + type Node = AppDagNode; + + fn frontier(&self) -> &Frontiers { + &self.frontiers + } + + fn get(&self, id: ID) -> Option { + self.ensure_lazy_load_node(id); + let binding = self.map.lock(); + if let Some(x) = binding.range(..=id).next_back() { + if x.1.contains_id(id) { + // PERF: do we need to optimize clone like this? + // by adding another layer of Arc? + return Some(x.1.clone()); + } + } + + if let Some(node) = &self.pending_txn_node { + if node.peer == id.peer && node.cnt <= id.counter { + assert!(node.cnt + node.len as Counter > id.counter); + return Some(node.clone()); + } + } + + None + } + + fn vv(&self) -> &VersionVector { + &self.vv + } + + fn contains(&self, id: ID) -> bool { + self.vv.includes_id(id) + } +} + +impl AppDag { + // PERF: this may be painfully slow + /// get the version vector for a certain op. + /// It's the version when the op is applied + pub fn get_vv(&self, id: ID) -> Option { + self.get(id).map(|x| { + let mut vv = self.ensure_vv_for(&x); + vv.insert(id.peer, id.counter + 1); + vv + }) + } + + pub(crate) fn ensure_vv_for(&self, target_node: &AppDagNode) -> ImVersionVector { + if target_node.vv.get().is_none() { + // Iterative DFS. When the DAG contains a diamond, a dep can end + // up on the stack multiple times (once for each ancestor that + // re-queued the current node before its deps were ready). We + // skip nodes whose vv has already been computed — the original + // code eagerly called `OnceCell::set(..).unwrap()` on every pop + // and panicked on the second visit (see loro-dev/loro#929). + let mut stack: SmallVec<[AppDagNode; 4]> = smallvec::smallvec![target_node.clone()]; + while let Some(top_node) = stack.pop() { + if top_node.vv.get().is_some() { + continue; + } + + let mut ans_vv = ImVersionVector::default(); + if top_node.deps == self.shallow_root_frontiers_deps { + for (&p, &c) in self.shallow_since_vv.iter() { + ans_vv.insert(p, c); + } + } else { + let mut all_deps_processed = true; + for id in top_node.deps.iter() { + let Some(node) = self.get(id) else { + if self.shallow_since_vv.includes_id(id) { + continue; + } + + panic!("deps should be in the dag"); + }; + if node.vv.get().is_none() { + if all_deps_processed { + stack.push(top_node.clone()); + } + all_deps_processed = false; + stack.push(node); + continue; + }; + } + + if !all_deps_processed { + continue; + } + + for id in top_node.deps.iter() { + let Some(node) = self.get(id) else { + if self.shallow_since_vv.includes_id(id) { + ans_vv.extend_to_include_vv(self.shallow_since_vv.iter()); + continue; + } + + panic!("deps should be in the dag"); + }; + let dep_vv = node.vv.get().unwrap(); + if ans_vv.is_empty() { + ans_vv = dep_vv.clone(); + } else { + ans_vv.extend_to_include_vv(dep_vv.iter()); + } + + ans_vv.insert(node.peer, node.ctr_end()); + } + } + + // Tolerate a racing set from the diamond case above: if + // another path already initialized this cell, trust that + // value (it was computed from the same DAG) and move on. + let _ = top_node.vv.set(ans_vv); + } + } + + target_node.vv.get().unwrap().clone() + } + + /// Compare the causal order of two versions. + /// If None, two versions are concurrent to each other + pub fn cmp_version(&self, a: ID, b: ID) -> Option { + if a.peer == b.peer { + return Some(a.counter.cmp(&b.counter)); + } + + let a = self.get_vv(a).unwrap(); + let b = self.get_vv(b).unwrap(); + a.partial_cmp(&b) + } + + pub fn get_lamport(&self, id: &ID) -> Option { + self.get(*id).and_then(|node| { + assert!(id.counter >= node.cnt); + if node.cnt + node.len as Counter > id.counter { + Some(node.lamport + (id.counter - node.cnt) as Lamport) + } else { + None + } + }) + } + + pub fn get_change_lamport_from_deps(&self, deps: &Frontiers) -> Option { + let mut lamport = 0; + for id in deps.iter() { + let x = self.get_lamport(&id)?; + lamport = lamport.max(x + 1); + } + + Some(lamport) + } + + /// Convert a frontiers to a version vector + /// + /// If the frontiers version is not found in the dag, return None + pub fn frontiers_to_vv(&self, frontiers: &Frontiers) -> Option { + if frontiers == &self.shallow_root_frontiers_deps { + let vv = VersionVector::from_im_vv(&self.shallow_since_vv); + return Some(vv); + } + + let mut vv: VersionVector = Default::default(); + for id in frontiers.iter() { + let x = self.get(id)?; + let target_vv = self.ensure_vv_for(&x); + vv.extend_to_include_vv(target_vv.iter()); + vv.extend_to_include_last_id(id); + } + + Some(vv) + } + + #[allow(unused)] + pub(crate) fn frontiers_to_im_vv(&self, frontiers: &Frontiers) -> ImVersionVector { + if frontiers.is_empty() { + return Default::default(); + } + + let mut iter = frontiers.iter(); + let mut vv = { + let id = iter.next().unwrap(); + let Some(x) = self.get(id) else { + unreachable!() + }; + let mut vv = self.ensure_vv_for(&x); + vv.extend_to_include_last_id(id); + vv + }; + + for id in iter { + let Some(x) = self.get(id) else { + unreachable!() + }; + let x = self.ensure_vv_for(&x); + vv.extend_to_include_vv(x.iter()); + vv.extend_to_include_last_id(id); + } + + vv + } + + pub fn im_vv_to_frontiers(&self, vv: &ImVersionVector) -> Frontiers { + if vv.is_empty() { + return Default::default(); + } + + let this = vv; + let last_ids: Frontiers = this + .iter() + .filter_map(|(client_id, cnt)| { + if *cnt <= 0 { + return None; + } + + if self + .shallow_since_vv + .includes_id(ID::new(*client_id, *cnt - 1)) + { + return None; + } + + Some(ID::new(*client_id, cnt - 1)) + }) + .collect(); + + if last_ids.is_empty() { + return self.shallow_since_frontiers.clone(); + } + + shrink_frontiers(&last_ids, self).unwrap() + } + + pub fn vv_to_frontiers(&self, vv: &VersionVector) -> Frontiers { + if vv.is_empty() { + return Default::default(); + } + + let this = vv; + let last_ids: Frontiers = this + .iter() + .filter_map(|(client_id, cnt)| { + if *cnt <= 0 { + return None; + } + + if self + .shallow_since_vv + .includes_id(ID::new(*client_id, *cnt - 1)) + { + return None; + } + + Some(ID::new(*client_id, cnt - 1)) + }) + .collect(); + + if last_ids.is_empty() { + return self.shallow_since_frontiers.clone(); + } + + shrink_frontiers(&last_ids, self).unwrap() + } + + pub(crate) fn frontiers_to_next_lamport(&self, frontiers: &Frontiers) -> Lamport { + if frontiers.is_empty() { + return 0; + } + + let mut iter = frontiers.iter(); + let mut lamport = { + let id = iter.next().unwrap(); + let Some(x) = self.get(id) else { + unreachable!() + }; + assert!(id.counter >= x.cnt); + (id.counter - x.cnt) as Lamport + x.lamport + 1 + }; + + for id in iter { + let Some(x) = self.get(id) else { + unreachable!() + }; + assert!(id.counter >= x.cnt); + lamport = lamport.max((id.counter - x.cnt) as Lamport + x.lamport + 1); + } + + lamport + } + + pub fn get_frontiers(&self) -> &Frontiers { + &self.frontiers + } + + /// - Ordering::Less means self is less than target or parallel + /// - Ordering::Equal means versions equal + /// - Ordering::Greater means self's version is greater than target + pub fn cmp_with_frontiers(&self, other: &Frontiers) -> Ordering { + if &self.frontiers == other { + Ordering::Equal + } else if other.iter().all(|id| self.vv.includes_id(id)) { + Ordering::Greater + } else { + Ordering::Less + } + } + + // PERF + /// Compare two [Frontiers] causally. + /// + /// If one of the [Frontiers] are not included, it will return [FrontiersNotIncluded]. + pub fn cmp_frontiers( + &self, + a: &Frontiers, + b: &Frontiers, + ) -> Result, FrontiersNotIncluded> { + let a = self.frontiers_to_vv(a).ok_or(FrontiersNotIncluded)?; + let b = self.frontiers_to_vv(b).ok_or(FrontiersNotIncluded)?; + Ok(a.partial_cmp(&b)) + } +} + +#[derive(Debug, PartialEq, Eq)] +pub struct FrontiersNotIncluded; +impl Display for FrontiersNotIncluded { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("The given Frontiers are not included by the doc") + } +} + +#[cfg(test)] +mod ensure_vv_for_tests { + use super::*; + use crate::arena::SharedArena; + use std::sync::atomic::AtomicI64; + + fn make_dag_node(peer: PeerID, cnt: Counter, len: usize, deps: Frontiers) -> AppDagNode { + AppDagNodeInner { + vv: OnceCell::new(), + peer, + cnt, + lamport: cnt as Lamport, + deps, + has_succ: false, + len, + } + .into() + } + + fn make_shallow_dag_for_import_deps() -> AppDag { + let change_store = ChangeStore::new_mem(&SharedArena::new(), Arc::new(AtomicI64::new(0))); + let mut dag = AppDag::new(change_store); + let root_deps = Frontiers::from_id(ID::new(1, 1)); + let boundary = make_dag_node(1, 2, 1, root_deps.clone()); + + { + let mut map = dag.map.lock(); + map.insert(boundary.id_start(), boundary); + } + + dag.shallow_since_vv.insert(1, 2); + dag.shallow_since_frontiers = Frontiers::from_id(ID::new(1, 2)); + dag.shallow_root_frontiers_deps = root_deps; + dag + } + + /// Regression for loro-dev/loro#929: when computing the vv for a node + /// whose DAG fan-in contains a shared ancestor reached through multiple + /// paths, the iterative DFS used to push the shared ancestor onto the + /// stack twice. The second visit would then call `OnceCell::set(..) + /// .unwrap()` on an already-initialized cell and panic with + /// "called `Result::unwrap()` on an `Err` value: ImVersionVector(..)". + /// + /// Topology: + /// x (peer 1, counter 0) + /// | + /// y (peer 2, counter 0, deps = [x]) + /// | + /// z (peer 3, counter 0, deps = [x, y]) + /// + /// When we call `ensure_vv_for(z)`, z pushes x and y. Processing y then + /// re-pushes x (still uninitialized), so the stack ends up with two + /// copies of x; the second pop must be a no-op after the fix. + #[test] + fn diamond_dep_ensure_vv_for_does_not_double_set() { + let change_store = ChangeStore::new_mem(&SharedArena::new(), Arc::new(AtomicI64::new(0))); + let dag = AppDag::new(change_store); + + let x = make_dag_node(1, 0, 1, Frontiers::default()); + let y = make_dag_node(2, 0, 1, Frontiers::from_id(ID::new(1, 0))); + let mut z_deps = Frontiers::default(); + z_deps.push(ID::new(1, 0)); + z_deps.push(ID::new(2, 0)); + let z = make_dag_node(3, 0, 1, z_deps); + + { + let mut map = dag.map.lock(); + map.insert(x.id_start(), x); + map.insert(y.id_start(), y); + map.insert(z.id_start(), z.clone()); + } + + // Historically this panicked at the inner `vv.set(...).unwrap()`. + // `ensure_vv_for` returns the vv *at* the node: it covers every + // peer in the causal past, but not the node's own peer counter. + let vv = dag.ensure_vv_for(&z); + assert_eq!(vv.get(&1).copied(), Some(1)); + assert_eq!(vv.get(&2).copied(), Some(1)); + assert!(vv.get(&3).is_none()); + } + + #[test] + fn import_deps_before_shallow_root_rejects_trimmed_history_dep() { + let dag = make_shallow_dag_for_import_deps(); + let deps = Frontiers::from_id(ID::new(1, 0)); + + assert!(dag.frontiers_to_vv(&deps).is_none()); + assert!(dag.import_deps_before_shallow_root(&deps)); + } + + #[test] + fn import_deps_before_shallow_root_allows_boundary_with_missing_peer() { + let dag = make_shallow_dag_for_import_deps(); + let mut deps = Frontiers::default(); + deps.push(ID::new(1, 2)); + deps.push(ID::new(2, 0)); + + assert!(dag.frontiers_to_vv(&deps).is_none()); + assert!(!dag.import_deps_before_shallow_root(&deps)); + } + + #[test] + fn import_deps_before_shallow_root_allows_missing_non_trimmed_dep() { + let dag = make_shallow_dag_for_import_deps(); + let deps = Frontiers::from_id(ID::new(2, 0)); + + assert!(dag.frontiers_to_vv(&deps).is_none()); + assert!(!dag.import_deps_before_shallow_root(&deps)); + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/oplog/pending_changes.rs b/3rdparty/loro/crates/loro-internal/src/oplog/pending_changes.rs new file mode 100644 index 0000000000..8d84e2b372 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/oplog/pending_changes.rs @@ -0,0 +1,428 @@ +use std::{collections::BTreeMap, ops::Deref}; + +use crate::{ + change::Change, + version::{ImVersionVector, VersionRange}, + OpLog, VersionVector, +}; +use loro_common::{ + ContainerType, Counter, CounterSpan, HasCounterSpan, HasIdSpan, LoroResult, PeerID, ID, +}; +use rustc_hash::FxHashMap; + +#[derive(Debug, Clone)] +pub enum PendingChange { + // The lamport of the change decoded by `enhanced` is unknown. + // we need calculate it when the change can be applied + Unknown(Change), + // TODO: Refactor, remove this? + #[allow(unused)] + Known(Change), +} + +impl Deref for PendingChange { + type Target = Change; + fn deref(&self) -> &Self::Target { + match self { + Self::Unknown(a) => a, + Self::Known(a) => a, + } + } +} + +#[derive(Debug, Default)] +pub(crate) struct PendingChanges { + changes: FxHashMap>>, +} + +impl PendingChanges { + pub(crate) fn has_state_apply_rollback_ops(&self) -> bool { + self.changes.values().any(|tree| { + tree.values().any(|changes| { + changes.iter().any(|change| { + change.ops.iter().any(|op| { + matches!( + op.container.get_type(), + ContainerType::List | ContainerType::Tree + ) + }) + }) + }) + }) + } + + pub(crate) fn version_range(&self) -> VersionRange { + let mut range = VersionRange::default(); + for tree in self.changes.values() { + for pending_changes in tree.values() { + for pending_change in pending_changes { + range.extends_to_include_id_span(pending_change.id_span()); + } + } + } + + range + } +} + +#[cfg(test)] +#[allow(dead_code)] +impl PendingChanges { + pub(crate) fn len(&self) -> usize { + self.changes + .values() + .map(|tree| tree.values().map(Vec::len).sum::()) + .sum() + } +} + +#[derive(Debug, Default)] +pub(crate) struct PendingChangesRollback { + added: Vec<(PeerID, Counter)>, + removed: Vec<(PeerID, Counter, Vec)>, +} + +impl PendingChangesRollback { + fn record_added(&mut self, id: ID) { + self.added.push((id.peer, id.counter)); + } + + fn record_removed(&mut self, peer: PeerID, counter: Counter, changes: Vec) { + self.removed.push((peer, counter, changes)); + } + + pub(crate) fn rollback(self, pending_changes: &mut PendingChanges) { + for (peer, counter) in self.added.into_iter().rev() { + let Some(tree) = pending_changes.changes.get_mut(&peer) else { + continue; + }; + let Some(changes) = tree.get_mut(&counter) else { + continue; + }; + changes.pop(); + if changes.is_empty() { + tree.remove(&counter); + } + if tree.is_empty() { + pending_changes.changes.remove(&peer); + } + } + + for (peer, counter, changes) in self.removed.into_iter().rev() { + pending_changes + .changes + .entry(peer) + .or_default() + .insert(counter, changes); + } + } +} + +impl OpLog { + fn push_pending_change(&mut self, missing_dep: ID, change: PendingChange) { + if let Some(rollback) = self.import_rollback.as_mut() { + rollback.pending.record_added(missing_dep); + } + + self.pending_changes + .changes + .entry(missing_dep.peer) + .or_default() + .entry(missing_dep.counter) + .or_default() + .push(change); + } + + pub(super) fn extend_pending_changes_with_unknown_lamport( + &mut self, + remote_changes: Vec, + ) -> LoroResult<()> { + for change in remote_changes { + let local_change = PendingChange::Unknown(change); + match remote_change_apply_state(self.vv(), self.shallow_since_vv(), &local_change) { + ChangeState::AwaitingMissingDependency(miss_dep) => { + self.push_pending_change(miss_dep, local_change) + } + ChangeState::Applied => unreachable!("already applied"), + ChangeState::CanApplyDirectly => unreachable!("can apply directly"), + } + } + + Ok(()) + } +} + +impl OpLog { + /// Try to apply pending changes. + /// + /// `new_ids` are the ID of the op that is just applied. + pub(crate) fn try_apply_pending( + &mut self, + mut new_ids: Vec, + mut would_affect: Option<&mut VersionRange>, + ) { + while let Some(id) = new_ids.pop() { + let Some(tree) = self.pending_changes.changes.get_mut(&id.peer) else { + continue; + }; + + let mut to_remove = Vec::new(); + for (cnt, _) in tree.range_mut(0..=id.counter) { + to_remove.push(*cnt); + } + + let mut pending_set = Vec::with_capacity(to_remove.len()); + for cnt in to_remove { + let pending_changes = tree.remove(&cnt).unwrap(); + if let Some(rollback) = self.import_rollback.as_mut() { + rollback + .pending + .record_removed(id.peer, cnt, pending_changes.clone()); + } + pending_set.push(pending_changes); + } + + if tree.is_empty() { + self.pending_changes.changes.remove(&id.peer); + } + + for pending_changes in pending_set { + for pending_change in pending_changes { + match remote_change_apply_state( + self.dag.vv(), + self.dag.shallow_since_vv(), + &pending_change, + ) { + ChangeState::CanApplyDirectly => { + new_ids.push(pending_change.id_last()); + self.apply_change_from_remote( + pending_change, + would_affect.as_deref_mut(), + ); + } + ChangeState::Applied => {} + ChangeState::AwaitingMissingDependency(miss_dep) => { + self.push_pending_change(miss_dep, pending_change) + } + } + } + } + } + } + + pub(super) fn apply_change_from_remote( + &mut self, + change: PendingChange, + would_affect: Option<&mut VersionRange>, + ) { + let change = match change { + PendingChange::Known(mut c) => { + self.dag.calc_unknown_lamport_change(&mut c).unwrap(); + c + } + PendingChange::Unknown(mut c) => { + self.dag.calc_unknown_lamport_change(&mut c).unwrap(); + c + } + }; + + let Some(change) = self.trim_the_known_part_of_change(change) else { + return; + }; + + if let Some(w) = would_affect { + w.extends_to_include_id_span(change.id_span()); + } + self.insert_new_change(change, false); + } +} + +enum ChangeState { + Applied, + CanApplyDirectly, + // The id of first missing dep + AwaitingMissingDependency(ID), +} + +fn remote_change_apply_state( + vv: &VersionVector, + _shallow_vv: &ImVersionVector, + change: &Change, +) -> ChangeState { + let peer = change.id.peer; + let CounterSpan { start, end } = change.ctr_span(); + let vv_latest_ctr = vv.get(&peer).copied().unwrap_or(0); + if vv_latest_ctr >= end { + return ChangeState::Applied; + } + + if vv_latest_ctr < start { + return ChangeState::AwaitingMissingDependency(change.id.inc(-1)); + } + + for dep in change.deps.iter() { + let dep_vv_latest_ctr = vv.get(&dep.peer).copied().unwrap_or(0); + if dep_vv_latest_ctr - 1 < dep.counter { + return ChangeState::AwaitingMissingDependency(dep); + } + } + + ChangeState::CanApplyDirectly +} + +#[cfg(test)] +mod test { + use crate::{cursor::PosType, loro::ExportMode, LoroDoc, ToJson, VersionVector}; + + #[test] + fn import_pending() { + let a = LoroDoc::new_auto_commit(); + a.set_peer_id(1).unwrap(); + let b = LoroDoc::new_auto_commit(); + b.set_peer_id(2).unwrap(); + let text_a = a.get_text("text"); + text_a.insert(0, "a", PosType::Unicode).unwrap(); + let update1 = a + .export(ExportMode::updates(&VersionVector::default())) + .unwrap(); + let version1 = a.oplog_vv(); + text_a.insert(0, "b", PosType::Unicode).unwrap(); + let update2 = a.export(ExportMode::updates(&version1)).unwrap(); + let version2 = a.oplog_vv(); + text_a.insert(0, "c", PosType::Unicode).unwrap(); + let update3 = a.export(ExportMode::updates(&version2)).unwrap(); + let version3 = a.oplog_vv(); + text_a.insert(0, "d", PosType::Unicode).unwrap(); + let update4 = a.export(ExportMode::updates(&version3)).unwrap(); + // let version4 = a.oplog_vv(); + text_a.insert(0, "e", PosType::Unicode).unwrap(); + let update3_5 = a.export(ExportMode::updates(&version2)).unwrap(); + b.import(&update3_5).unwrap(); + b.import(&update4).unwrap(); + b.import(&update2).unwrap(); + b.import(&update3).unwrap(); + b.import(&update1).unwrap(); + assert_eq!(a.get_deep_value(), b.get_deep_value()); + } + + #[test] + fn pending_import_snapshot() { + let a = LoroDoc::new_auto_commit(); + a.set_peer_id(1).unwrap(); + let b = LoroDoc::new_auto_commit(); + b.set_peer_id(2).unwrap(); + let text_a = a.get_text("text"); + text_a.insert(0, "a", PosType::Unicode).unwrap(); + let update1 = a.export(ExportMode::Snapshot).unwrap(); + let version1 = a.oplog_vv(); + text_a.insert(0, "b", PosType::Unicode).unwrap(); + let update2 = a.export(ExportMode::updates(&version1)).unwrap(); + let _version2 = a.oplog_vv(); + b.import(&update2).unwrap(); + // snapshot will be converted to updates + b.import(&update1).unwrap(); + assert_eq!(a.get_deep_value(), b.get_deep_value()); + } + + #[test] + fn need_deps_pending_import() { + // a: a1 <--- a2 + // \ / + // b: b1 + let a = LoroDoc::new_auto_commit(); + a.set_peer_id(1).unwrap(); + let b = LoroDoc::new_auto_commit(); + b.set_peer_id(2).unwrap(); + let c = LoroDoc::new_auto_commit(); + c.set_peer_id(3).unwrap(); + let d = LoroDoc::new_auto_commit(); + d.set_peer_id(4).unwrap(); + let text_a = a.get_text("text"); + let text_b = b.get_text("text"); + text_a.insert(0, "a", PosType::Unicode).unwrap(); + let version_a1 = a.oplog_vv(); + let update_a1 = a + .export(ExportMode::updates(&VersionVector::default())) + .unwrap(); + b.import(&update_a1).unwrap(); + text_b.insert(1, "b", PosType::Unicode).unwrap(); + let update_b1 = b.export(ExportMode::updates(&version_a1)).unwrap(); + a.import(&update_b1).unwrap(); + let version_a1b1 = a.oplog_vv(); + text_a.insert(2, "c", PosType::Unicode).unwrap(); + let update_a2 = a.export(ExportMode::updates(&version_a1b1)).unwrap(); + c.import(&update_a2).unwrap(); + assert_eq!(c.get_deep_value().to_json(), "{\"text\":\"\"}"); + c.import(&update_a1).unwrap(); + assert_eq!(c.get_deep_value().to_json(), "{\"text\":\"a\"}"); + c.import(&update_b1).unwrap(); + assert_eq!(a.get_deep_value(), c.get_deep_value()); + + d.import(&update_a2).unwrap(); + assert_eq!(d.get_deep_value().to_json(), "{\"text\":\"\"}"); + d.import(&update_b1).unwrap(); + assert_eq!(d.get_deep_value().to_json(), "{\"text\":\"\"}"); + d.import(&update_a1).unwrap(); + assert_eq!(a.get_deep_value(), d.get_deep_value()); + } + + #[test] + fn should_activate_pending_change_when() { + // 0@a <- 0@b + // 0@a <- 1@a, where 0@a and 1@a will be merged + // In this case, c apply b's change first, then apply all the changes from a. + // C is expected to have the same content as a, after a imported b's change + let a = LoroDoc::new_auto_commit(); + a.set_peer_id(1).unwrap(); + let b = LoroDoc::new_auto_commit(); + b.set_peer_id(2).unwrap(); + let c = LoroDoc::new_auto_commit(); + c.set_peer_id(3).unwrap(); + let text_a = a.get_text("text"); + let text_b = b.get_text("text"); + text_a.insert(0, "1", PosType::Unicode).unwrap(); + b.import(&a.export(ExportMode::Snapshot).unwrap()).unwrap(); + text_b.insert(0, "1", PosType::Unicode).unwrap(); + let b_change = b.export(ExportMode::updates(&a.oplog_vv())).unwrap(); + text_a.insert(0, "1", PosType::Unicode).unwrap(); + c.import(&b_change).unwrap(); + c.import(&a.export(ExportMode::Snapshot).unwrap()).unwrap(); + a.import(&b_change).unwrap(); + assert_eq!(c.get_deep_value(), a.get_deep_value()); + } + + // Change cannot be merged now + // #[test] + // fn pending_changes_may_deps_merged_change() { + // // a: (a1 <-- a2 <-- a3) <-- a4 a1~a3 is a merged change + // // \ / + // // b: b1 + // let a = LoroDoc::new(); + // a.set_peer_id(1); + // let b = LoroDoc::new(); + // b.set_peer_id(2); + // let c = LoroDoc::new(); + // c.set_peer_id(3); + // let text_a = a.get_text("text"); + // let text_b = b.get_text("text"); + // a.with_txn(|txn| text_a.insert(txn, 0, "a")).unwrap(); + // a.with_txn(|txn| text_a.insert(txn, 1, "b")).unwrap(); + // let version_a12 = a.oplog_vv(); + // let updates_a12 = a.export(ExportMode::Snapshot); + // a.with_txn(|txn| text_a.insert(txn, 2, "c")).unwrap(); + // let updates_a123 = a.export(ExportMode::Snapshot); + // b.import(&updates_a12).unwrap(); + // b.with_txn(|txn| text_b.insert(txn, 2, "d")).unwrap(); + // let update_b1 = b.export(ExportMode::updates(&version_a12)).unwrap(); + // a.import(&update_b1).unwrap(); + // let version_a123_b1 = a.oplog_vv(); + // a.with_txn(|txn| text_a.insert(txn, 4, "e")).unwrap(); + // let update_a4 = a.export(ExportMode::updates(&version_a123_b1)).unwrap(); + // c.import(&update_b1).unwrap(); + // assert_eq!(c.get_deep_value().to_json(), "{\"text\":\"\"}"); + // c.import(&update_a4).unwrap(); + // assert_eq!(c.get_deep_value().to_json(), "{\"text\":\"\"}"); + // c.import(&updates_a123).unwrap(); + // assert_eq!(c.get_deep_value(), a.get_deep_value()); + // } +} diff --git a/3rdparty/loro/crates/loro-internal/src/parent.rs b/3rdparty/loro/crates/loro-internal/src/parent.rs new file mode 100644 index 0000000000..371950f328 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/parent.rs @@ -0,0 +1,88 @@ +//! This file contains the methods to maintain the parent-child relationship in Loro. +//! The child-parent relationship is established when the child container is created. +//! It won't be changed. So the depth of a container is fixed. +//! +//! + +use loro_common::LoroValue; + +use crate::{ + arena::SharedArena, + change::Change, + container::{list::list_op::ListOp, map::MapSet}, + op::{ListSlice, RawOp, RawOpContent}, + DocState, OpLog, +}; + +impl OpLog { + /// Establish the link between the child container and the parent container. + /// This relationship will never be broken. + /// It's used at the entry of creating new ops to ensure all the valid containers + /// in Loro are properly linked. + pub(super) fn register_container_and_parent_link(&self, change: &Change) { + let arena = &self.arena; + register_container_and_parent_link(arena, change); + } +} + +pub(super) fn register_container_and_parent_link(arena: &SharedArena, change: &Change) { + for op in change.ops.iter() { + op.content.visit_created_children(arena, &mut |c| { + let idx = arena.register_container(c); + arena.set_parent(idx, Some(op.container)); + }); + } +} + +impl DocState { + /// This is used in txn to short cut the process of applying an op to the state. + /// So the op here has not been registered on the oplog yet. + pub(super) fn set_container_parent_by_raw_op(&self, raw_op: &RawOp) { + let container = raw_op.container; + match &raw_op.content { + RawOpContent::List(op) => { + if let ListOp::Insert { + slice: ListSlice::RawData(list), + .. + } = op + { + let list = match list { + std::borrow::Cow::Borrowed(list) => list.iter(), + std::borrow::Cow::Owned(list) => list.iter(), + }; + for value in list { + if let LoroValue::Container(c) = value { + let idx = self.arena.register_container(c); + self.arena.set_parent(idx, Some(container)); + } + } + } + if let ListOp::Set { + elem_id: _, + value: LoroValue::Container(c), + } = op + { + let idx = self.arena.register_container(c); + self.arena.set_parent(idx, Some(container)); + } + } + RawOpContent::Map(MapSet { key: _, value }) => { + if let Some(LoroValue::Container(c)) = value { + let idx = self.arena.register_container(c); + self.arena.set_parent(idx, Some(container)); + } + } + RawOpContent::Tree(tree) => { + let target = tree.target(); + // create associated metadata container + // TODO: maybe we could create map container only when setting metadata + let container_id = target.associated_meta_container(); + let child_idx = self.arena.register_container(&container_id); + self.arena.set_parent(child_idx, Some(container)); + } + #[cfg(feature = "counter")] + RawOpContent::Counter(_) => {} + RawOpContent::Unknown { .. } => {} + } + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/pre_commit.rs b/3rdparty/loro/crates/loro-internal/src/pre_commit.rs new file mode 100644 index 0000000000..48c32ecda9 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/pre_commit.rs @@ -0,0 +1,69 @@ +use std::sync::Arc; + +use crate::sync::Mutex; +use crate::{ + change::{Change, Timestamp}, + oplog::get_timestamp_now_txn, + ChangeMeta, +}; +use loro_common::PeerID; + +/// The callback of the first commit from a peer. +pub type FirstCommitFromPeerCallback = + Box bool + Send + Sync + 'static>; +pub type PreCommitCallback = Box bool + Send + Sync + 'static>; + +/// The payload of the pre commit callback. +#[derive(Debug, Clone)] +pub struct PreCommitCallbackPayload { + /// The metadata of the change which will be committed. + pub change_meta: ChangeMeta, + /// The origin of the commit. + pub origin: String, + /// The modifier of the change. You can modify the change in the callback. + pub modifier: ChangeModifier, +} + +/// The payload of the first commit from a peer callback. +#[derive(Debug, Clone)] +pub struct FirstCommitFromPeerPayload { + /// The peer id of the first commit. + pub peer: PeerID, +} + +#[derive(Debug, Clone, Default)] +pub struct ChangeModifier(Arc>); + +#[derive(Debug, Default)] +struct ChangeModifierInner { + new_msg: Option>, + new_timestamp: Option, +} + +impl ChangeModifier { + pub fn set_message(&self, msg: &str) -> &Self { + self.0.lock().new_msg = Some(Arc::from(msg)); + self + } + + pub fn set_timestamp(&self, timestamp: Timestamp) -> &Self { + self.0.lock().new_timestamp = Some(timestamp); + self + } + + pub fn set_timestamp_now(&self) -> &Self { + self.0.lock().new_timestamp = Some(get_timestamp_now_txn()); + self + } + + pub(crate) fn modify_change(&self, change: &mut Change) { + let m = self.0.lock(); + if let Some(msg) = &m.new_msg { + change.commit_msg = Some(msg.clone()); + } + + if let Some(timestamp) = m.new_timestamp { + change.timestamp = timestamp; + } + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/span.rs b/3rdparty/loro/crates/loro-internal/src/span.rs new file mode 100644 index 0000000000..b111219389 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/span.rs @@ -0,0 +1,3 @@ +pub use loro_common::{ + CounterSpan, HasCounter, HasCounterSpan, HasId, HasIdSpan, HasLamport, HasLamportSpan, IdSpan, +}; diff --git a/3rdparty/loro/crates/loro-internal/src/state.rs b/3rdparty/loro/crates/loro-internal/src/state.rs new file mode 100644 index 0000000000..0e5009d997 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/state.rs @@ -0,0 +1,2151 @@ +use crate::sync::{AtomicU64, Mutex, RwLock}; +#[cfg(test)] +use std::cell::Cell; +use std::sync::{Arc, Weak}; +use std::{borrow::Cow, io::Write, sync::atomic::Ordering}; + +use container_store::ContainerStore; +use dead_containers_cache::DeadContainersCache; +use enum_as_inner::EnumAsInner; +use enum_dispatch::enum_dispatch; +use itertools::Itertools; +use loro_common::{ContainerID, Lamport, LoroError, LoroResult, TreeID}; +use loro_delta::DeltaItem; +use rustc_hash::{FxHashMap, FxHashSet}; +use tracing::{info_span, instrument, warn}; + +use crate::{ + configure::{Configure, DefaultRandom, SecureRandomGenerator}, + container::{idx::ContainerIdx, richtext::config::StyleConfigMap}, + cursor::{Cursor, PosType}, + delta::TreeExternalDiff, + diff_calc::{DiffCalculator, DiffMode}, + event::{Diff, EventTriggerKind, Index, InternalContainerDiff, InternalDiff}, + fx_map, + handler::ValueOrHandler, + id::PeerID, + lock::{LoroLockGroup, LoroMutex}, + op::{Op, RawOp}, + version::Frontiers, + ContainerDiff, ContainerType, DocDiff, InternalString, LoroDocInner, LoroValue, OpLog, +}; + +pub(crate) mod analyzer; +pub(crate) mod container_store; +#[cfg(feature = "counter")] +mod counter_state; +mod dead_containers_cache; +mod list_state; +mod map_state; +mod mergeable; +mod movable_list_state; +mod richtext_state; +mod tree_state; +mod unknown_state; + +pub(crate) use self::movable_list_state::{IndexType, MovableListState}; +pub(crate) use container_store::GcStore; +pub(crate) use list_state::ListState; +pub(crate) use map_state::MapState; +pub(crate) use richtext_state::RichtextState; +pub(crate) use tree_state::FiIfNotConfigured; +pub(crate) use tree_state::{get_meta_value, FractionalIndexGenResult, NodePosition, TreeState}; +pub use tree_state::{TreeNode, TreeNodeWithChildren, TreeParentId}; + +use self::{container_store::ContainerWrapper, unknown_state::UnknownState}; + +#[cfg(feature = "counter")] +use self::counter_state::CounterState; + +use super::{arena::SharedArena, event::InternalDocDiff}; + +#[cfg(test)] +thread_local! { + static FAIL_NEXT_IMPORT_STATE_APPLY: Cell = Cell::new(false); +} + +#[cfg(test)] +pub(crate) fn fail_next_import_state_apply_for_test() { + FAIL_NEXT_IMPORT_STATE_APPLY.with(|fail| fail.set(true)); +} + +fn visible_container_value_is_empty(kind: ContainerType, value: &LoroValue) -> bool { + match kind { + ContainerType::Text => value.as_string().is_some_and(|value| value.is_empty()), + ContainerType::Map | ContainerType::List | ContainerType::MovableList => { + value.is_empty_collection() + } + ContainerType::Tree => value.as_list().is_some_and(|value| value.is_empty()), + #[cfg(feature = "counter")] + ContainerType::Counter => false, + ContainerType::Unknown(_) => false, + } +} + +fn deleted_root_container_value_is_cleared(kind: ContainerType, value: &LoroValue) -> bool { + match kind { + #[cfg(feature = "counter")] + ContainerType::Counter => value.as_double().is_some_and(|value| *value == 0.0), + _ => visible_container_value_is_empty(kind, value), + } +} + +fn state_decode_error(message: impl Into>) -> LoroError { + LoroError::DecodeError(message.into()) +} + +fn decode_peer_table(bytes: &mut &[u8], context: &str) -> LoroResult> { + let peer_num = leb128::read::unsigned(bytes) + .map_err(|_| state_decode_error(format!("{context}: invalid peer table length")))?; + let peer_num = usize::try_from(peer_num) + .map_err(|_| state_decode_error(format!("{context}: peer table length overflow")))?; + let peer_bytes_len = peer_num + .checked_mul(std::mem::size_of::()) + .ok_or_else(|| state_decode_error(format!("{context}: peer table byte length overflow")))?; + if bytes.len() < peer_bytes_len { + return Err(state_decode_error(format!( + "{context}: truncated peer table" + ))); + } + + let peer_bytes = &bytes[..peer_bytes_len]; + let peers = peer_bytes + .chunks_exact(std::mem::size_of::()) + .map(|chunk| { + let mut buf = [0u8; std::mem::size_of::()]; + buf.copy_from_slice(chunk); + PeerID::from_le_bytes(buf) + }) + .collect(); + *bytes = &bytes[peer_bytes_len..]; + Ok(peers) +} + +fn decode_peer_from_table(peers: &[PeerID], peer_idx: usize, context: &str) -> LoroResult { + peers + .get(peer_idx) + .copied() + .ok_or_else(|| state_decode_error(format!("{context}: peer index out of range"))) +} + +fn read_state_leb_u64(bytes: &mut &[u8], context: &str) -> LoroResult { + leb128::read::unsigned(bytes) + .map_err(|_| state_decode_error(format!("{context}: invalid integer"))) +} + +fn decode_counter(counter: i32, context: &str) -> LoroResult { + if counter < 0 { + return Err(state_decode_error(format!("{context}: negative counter"))); + } + + Ok(counter) +} + +fn decode_lamport_from_delta( + counter: i32, + lamport_sub_counter: i32, + context: &str, +) -> LoroResult { + decode_counter(counter, context)?; + let lamport = counter + .checked_add(lamport_sub_counter) + .ok_or_else(|| state_decode_error(format!("{context}: lamport overflow")))?; + u32::try_from(lamport).map_err(|_| state_decode_error(format!("{context}: negative lamport"))) +} + +pub struct DocState { + pub(super) peer: Arc, + + pub(super) frontiers: Frontiers, + // pub(super) states: FxHashMap, + pub(super) store: ContainerStore, + pub(super) arena: SharedArena, + pub(crate) config: Configure, + // resolve event stuff + doc: Weak, + // txn related stuff + in_txn: bool, + changed_idx_in_txn: FxHashSet, + + // diff related stuff + event_recorder: EventRecorder, + + dead_containers_cache: DeadContainersCache, +} + +impl std::fmt::Debug for DocState { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DocState") + .field("peer", &self.peer) + .finish() + } +} + +#[derive(Clone, Copy)] +pub(crate) struct ContainerCreationContext<'a> { + pub configure: &'a Configure, + pub peer: PeerID, +} + +pub(crate) struct DiffApplyContext<'a> { + pub mode: DiffMode, + pub doc: &'a Weak, +} + +pub(crate) trait FastStateSnapshot { + fn encode_snapshot_fast(&mut self, w: W); + fn decode_value(bytes: &[u8]) -> LoroResult<(LoroValue, &[u8])>; + fn decode_snapshot_fast( + idx: ContainerIdx, + v: (LoroValue, &[u8]), + ctx: ContainerCreationContext, + ) -> LoroResult + where + Self: Sized; +} + +#[derive(Debug, Clone, Default)] +pub(crate) struct ApplyLocalOpReturn { + pub deleted_containers: Vec, +} + +#[enum_dispatch] +pub(crate) trait ContainerState { + fn container_idx(&self) -> ContainerIdx; + + fn is_state_empty(&self) -> bool; + + #[must_use] + fn apply_diff_and_convert(&mut self, diff: InternalDiff, ctx: DiffApplyContext) -> Diff; + + /// Apply a state diff. Returns `Err` when the diff is malformed in a + /// way that would corrupt the container (e.g. a `Retain`/`Insert` that + /// overruns the current state). The error is propagated up to + /// `doc.import` so the caller can observe the failure without the + /// doc-state panicking inside a lock. + fn apply_diff(&mut self, diff: InternalDiff, ctx: DiffApplyContext) -> LoroResult<()>; + + /// Validate a state diff before it is applied. Implementations should keep + /// this check proportional to the diff size and must not mutate container + /// state. + fn validate_diff(&self, _diff: &InternalDiff) -> LoroResult<()> { + Ok(()) + } + + fn apply_local_op(&mut self, raw_op: &RawOp, op: &Op) -> LoroResult; + /// Convert a state to a diff, such that an empty state will be transformed into the same as this state when it's applied. + fn to_diff(&mut self, doc: &Weak) -> Diff; + + fn get_value(&mut self) -> LoroValue; + + /// Get the index of the child container + #[allow(unused)] + fn get_child_index(&self, id: &ContainerID) -> Option; + + #[allow(unused)] + fn contains_child(&self, id: &ContainerID) -> bool; + + #[allow(unused)] + fn get_child_containers(&self) -> Vec; + + fn fork(&self, config: &Configure) -> Self; +} + +impl FastStateSnapshot for Box { + fn encode_snapshot_fast(&mut self, w: W) { + self.as_mut().encode_snapshot_fast(w) + } + + fn decode_value(bytes: &[u8]) -> LoroResult<(LoroValue, &[u8])> { + T::decode_value(bytes) + } + + fn decode_snapshot_fast( + idx: ContainerIdx, + v: (LoroValue, &[u8]), + ctx: ContainerCreationContext, + ) -> LoroResult + where + Self: Sized, + { + T::decode_snapshot_fast(idx, v, ctx).map(|x| Box::new(x)) + } +} + +impl ContainerState for Box { + fn container_idx(&self) -> ContainerIdx { + self.as_ref().container_idx() + } + + fn is_state_empty(&self) -> bool { + self.as_ref().is_state_empty() + } + + fn apply_diff_and_convert(&mut self, diff: InternalDiff, ctx: DiffApplyContext) -> Diff { + self.as_mut().apply_diff_and_convert(diff, ctx) + } + + fn apply_diff(&mut self, diff: InternalDiff, ctx: DiffApplyContext) -> LoroResult<()> { + self.as_mut().apply_diff(diff, ctx) + } + + fn validate_diff(&self, diff: &InternalDiff) -> LoroResult<()> { + self.as_ref().validate_diff(diff) + } + + fn apply_local_op(&mut self, raw_op: &RawOp, op: &Op) -> LoroResult { + self.as_mut().apply_local_op(raw_op, op) + } + + #[doc = r" Convert a state to a diff, such that an empty state will be transformed into the same as this state when it's applied."] + fn to_diff(&mut self, doc: &Weak) -> Diff { + self.as_mut().to_diff(doc) + } + + fn get_value(&mut self) -> LoroValue { + self.as_mut().get_value() + } + + #[doc = r" Get the index of the child container"] + #[allow(unused)] + fn get_child_index(&self, id: &ContainerID) -> Option { + self.as_ref().get_child_index(id) + } + + fn contains_child(&self, id: &ContainerID) -> bool { + self.as_ref().contains_child(id) + } + + #[allow(unused)] + fn get_child_containers(&self) -> Vec { + self.as_ref().get_child_containers() + } + + fn fork(&self, config: &Configure) -> Self { + Box::new(self.as_ref().fork(config)) + } +} + +#[allow(clippy::enum_variant_names)] +#[enum_dispatch(ContainerState)] +#[derive(EnumAsInner, Debug)] +pub enum State { + ListState(Box), + MovableListState(Box), + MapState(Box), + RichtextState(Box), + TreeState(Box), + #[cfg(feature = "counter")] + CounterState(Box), + UnknownState(UnknownState), +} + +impl From for State { + fn from(s: ListState) -> Self { + Self::ListState(Box::new(s)) + } +} + +impl From for State { + fn from(s: RichtextState) -> Self { + Self::RichtextState(Box::new(s)) + } +} + +impl From for State { + fn from(s: MovableListState) -> Self { + Self::MovableListState(Box::new(s)) + } +} + +impl From for State { + fn from(s: MapState) -> Self { + Self::MapState(Box::new(s)) + } +} + +impl From for State { + fn from(s: TreeState) -> Self { + Self::TreeState(Box::new(s)) + } +} + +#[cfg(feature = "counter")] +impl From for State { + fn from(s: CounterState) -> Self { + Self::CounterState(Box::new(s)) + } +} + +impl State { + pub fn new_list(idx: ContainerIdx) -> Self { + Self::ListState(Box::new(ListState::new(idx))) + } + + pub fn new_map(idx: ContainerIdx) -> Self { + Self::MapState(Box::new(MapState::new(idx))) + } + + pub fn new_richtext(idx: ContainerIdx, config: Arc>) -> Self { + Self::RichtextState(Box::new(RichtextState::new(idx, config))) + } + + pub fn new_tree(idx: ContainerIdx, peer: PeerID) -> Self { + Self::TreeState(Box::new(TreeState::new(idx, peer))) + } + + pub fn new_unknown(idx: ContainerIdx) -> Self { + Self::UnknownState(UnknownState::new(idx)) + } + + pub fn encode_snapshot_fast(&mut self, mut w: W) { + match self { + State::ListState(s) => s.encode_snapshot_fast(&mut w), + State::MovableListState(s) => s.encode_snapshot_fast(&mut w), + State::MapState(s) => s.encode_snapshot_fast(&mut w), + State::RichtextState(s) => s.encode_snapshot_fast(&mut w), + State::TreeState(s) => s.encode_snapshot_fast(&mut w), + #[cfg(feature = "counter")] + State::CounterState(s) => s.encode_snapshot_fast(&mut w), + State::UnknownState(s) => s.encode_snapshot_fast(&mut w), + } + } + + pub fn fork(&self, config: &Configure) -> Self { + match self { + State::ListState(list_state) => State::ListState(list_state.fork(config)), + State::MovableListState(movable_list_state) => { + State::MovableListState(movable_list_state.fork(config)) + } + State::MapState(map_state) => State::MapState(map_state.fork(config)), + State::RichtextState(richtext_state) => { + State::RichtextState(richtext_state.fork(config)) + } + State::TreeState(tree_state) => State::TreeState(tree_state.fork(config)), + #[cfg(feature = "counter")] + State::CounterState(counter_state) => State::CounterState(counter_state.fork(config)), + State::UnknownState(unknown_state) => State::UnknownState(unknown_state.fork(config)), + } + } +} + +impl DocState { + #[inline] + pub fn new_arc( + doc: Weak, + arena: SharedArena, + config: Configure, + lock_group: &LoroLockGroup, + ) -> Arc> { + let peer = DefaultRandom.next_u64(); + // TODO: maybe we should switch to certain version in oplog? + + let peer = Arc::new(AtomicU64::new(peer)); + Arc::new(lock_group.new_lock( + Self { + store: ContainerStore::new(arena.clone(), config.clone(), peer.clone()), + peer, + arena, + frontiers: Frontiers::default(), + doc, + config, + in_txn: false, + changed_idx_in_txn: FxHashSet::default(), + event_recorder: Default::default(), + dead_containers_cache: Default::default(), + }, + crate::lock::LockKind::DocState, + )) + } + + pub fn fork_with_new_peer_id( + &mut self, + doc: Weak, + arena: SharedArena, + config: Configure, + ) -> Arc> { + let peer = Arc::new(AtomicU64::new(DefaultRandom.next_u64())); + let store = self.store.fork(arena.clone(), peer.clone(), config.clone()); + Arc::new(Mutex::new(Self { + peer, + frontiers: self.frontiers.clone(), + store, + arena, + config, + doc, + in_txn: false, + changed_idx_in_txn: FxHashSet::default(), + event_recorder: Default::default(), + dead_containers_cache: Default::default(), + })) + } + + pub fn start_recording(&mut self) { + if self.is_recording() { + return; + } + + self.event_recorder.recording_diff = true; + self.event_recorder.diff_start_version = Some(self.frontiers.clone()); + } + + #[inline(always)] + pub fn stop_and_clear_recording(&mut self) { + self.event_recorder = Default::default(); + } + + #[inline(always)] + pub fn is_recording(&self) -> bool { + self.event_recorder.recording_diff + } + + pub fn refresh_peer_id(&mut self) { + self.peer.store( + DefaultRandom.next_u64(), + std::sync::atomic::Ordering::Relaxed, + ); + } + + /// Take all the diffs that are recorded and convert them to events. + pub fn take_events(&mut self) -> Vec { + if !self.is_recording() { + return vec![]; + } + + self.convert_current_batch_diff_into_event(); + std::mem::take(&mut self.event_recorder.events) + } + + /// Record the next diff. + /// Caller should call [pre_txn] before calling this. + /// + /// # Panic + /// + /// Panic when the diff cannot be merged with the previous diff. + /// Caller should call [pre_txn] before calling this to avoid panic. + fn record_diff(&mut self, diff: InternalDocDiff) { + if !self.event_recorder.recording_diff || diff.diff.is_empty() { + return; + } + + let Some(last_diff) = self.event_recorder.diffs.last_mut() else { + self.event_recorder.diffs.push(diff.into_owned()); + return; + }; + + if last_diff.can_merge(&diff) { + self.event_recorder.diffs.push(diff.into_owned()); + return; + } + + panic!("should call pre_txn before record_diff") + } + + /// This should be called when DocState is going to apply a transaction / a diff. + fn pre_txn(&mut self, next_origin: InternalString, next_trigger: EventTriggerKind) { + if !self.is_recording() { + return; + } + + let Some(last_diff) = self.event_recorder.diffs.last() else { + return; + }; + + if last_diff.origin == next_origin && last_diff.by == next_trigger { + return; + } + + // current diff batch cannot merge with the incoming diff, + // need to convert all the current diffs into event + self.convert_current_batch_diff_into_event() + } + + fn convert_current_batch_diff_into_event(&mut self) { + let recorder = &mut self.event_recorder; + if recorder.diffs.is_empty() { + return; + } + + let diffs = std::mem::take(&mut recorder.diffs); + let start = recorder.diff_start_version.take().unwrap(); + recorder.diff_start_version = Some((*diffs.last().unwrap().new_version).to_owned()); + let event = self.diffs_to_event(diffs, start); + self.event_recorder.events.push(event); + } + + /// Change the peer id of this doc state. + /// It changes the peer id for the future txn on this AppState + #[inline] + pub fn set_peer_id(&mut self, peer: PeerID) { + self.peer.store(peer, std::sync::atomic::Ordering::Relaxed); + } + + pub fn peer_id(&self) -> PeerID { + self.peer.load(std::sync::atomic::Ordering::Relaxed) + } + + /// It's expected that diff only contains [`InternalDiff`] + /// + /// Returns `Err` if any container state rejects its diff as malformed + /// (see `ContainerState::validate_diff`). Validation runs before mutating + /// any container state, so a returned error leaves the doc-state view at + /// the previous version. + #[instrument(skip_all)] + pub(crate) fn apply_diff( + &mut self, + mut diff: InternalDocDiff<'static>, + diff_mode: DiffMode, + ) -> LoroResult<()> { + if self.in_txn { + return Err(LoroError::TransactionError( + "apply_diff should not be called in a transaction" + .to_string() + .into_boxed_str(), + )); + } + + let is_recording = self.is_recording(); + let Cow::Owned(mut diffs) = std::mem::take(&mut diff.diff) else { + unreachable!() + }; + self.validate_diff_batch(&diffs)?; + #[cfg(test)] + if diff.origin.as_str() == "__loro_fail_import_state_apply" { + return Err(LoroError::internal("state apply failpoint")); + } + #[cfg(test)] + if diff.by == EventTriggerKind::Import { + let should_fail = FAIL_NEXT_IMPORT_STATE_APPLY.with(|fail| { + let should_fail = fail.get(); + if should_fail { + fail.set(false); + } + should_fail + }); + if should_fail { + return Err(LoroError::internal("state apply failpoint")); + } + } + match diff_mode { + DiffMode::Checkout => { + self.dead_containers_cache.clear(); + } + _ => { + self.dead_containers_cache.clear_alive(); + } + } + self.pre_txn(diff.origin.clone(), diff.by); + + // # Revival + // + // A Container, if it is deleted from its parent Container, will still exist + // in the internal state of Loro; whereas on the user side, a tree structure + // is maintained following Events, and at this point, the corresponding state + // is considered deleted. + // + // Sometimes, this "pseudo-dead" Container may be revived (for example, through + // backtracking or parallel editing), and the user side should receive an Event + // that restores the consistency between the revived Container and the internal + // state of Loro. This Event is required to restore the pseudo-dead Container + // State to its current state on Loro, and we refer to this process as "revival". + // + // Revival occurs during the application of the internal diff, and this operation + // is necessary when it needs to be converted into an external Event. + // + // We can utilize the output of the Diff to determine which child nodes should be revived. + // + // For nodes that are to be revived, we can disregard the Events output by their + // round of apply_diff_and_convert, and instead, directly convert their state into + // an Event once their application is complete. + // + // Suppose A is revived and B is A's child, and B also needs to be revived; therefore, + // we should process each level alternately. + + // We need to ensure diff is processed in order + diffs.sort_by_cached_key(|diff| self.arena.get_depth(diff.idx)); + let mut to_revive_in_next_layer: FxHashSet = FxHashSet::default(); + let mut to_revive_in_this_layer: FxHashSet = FxHashSet::default(); + let mut last_depth = 0; + let len = diffs.len(); + for mut diff in std::mem::replace(&mut diffs, Vec::with_capacity(len)) { + let Some(depth) = self.arena.get_depth(diff.idx) else { + warn!("{:?} is not in arena. It could be a dangling container that was deleted before the shallow start version.", self.arena.idx_to_id(diff.idx)); + continue; + }; + let this_depth = depth.get(); + while this_depth > last_depth { + // Clear `to_revive` when we are going to process a new level + // so that we can process the revival of the next level + let to_create = std::mem::take(&mut to_revive_in_this_layer); + to_revive_in_this_layer = std::mem::take(&mut to_revive_in_next_layer); + for new in to_create { + let state = self.store.get_or_create_mut(new); + if state.is_state_empty() { + continue; + } + + let external_diff = state.to_diff(&self.doc); + trigger_on_new_container( + &external_diff, + |cid| { + to_revive_in_this_layer.insert(cid); + }, + &self.arena, + ); + + diffs.push(InternalContainerDiff { + idx: new, + bring_back: true, + diff: external_diff.into(), + diff_mode: DiffMode::Checkout, + }); + } + + last_depth += 1; + } + + let idx = diff.idx; + let internal_diff = std::mem::take(&mut diff.diff); + match &internal_diff { + crate::event::DiffVariant::None => { + if is_recording { + let state = self.store.get_or_create_mut(diff.idx); + let extern_diff = state.to_diff(&self.doc); + trigger_on_new_container( + &extern_diff, + |cid| { + to_revive_in_next_layer.insert(cid); + }, + &self.arena, + ); + diff.diff = extern_diff.into(); + } + } + crate::event::DiffVariant::Internal(_) => { + let cid = self.arena.idx_to_id(idx).unwrap(); + info_span!("apply diff on", container_id = ?cid).in_scope( + || -> LoroResult<()> { + if self.in_txn { + self.changed_idx_in_txn.insert(idx); + } + let state = self.store.get_or_create_mut(idx); + if is_recording { + // process bring_back before apply + let external_diff = + if diff.bring_back || to_revive_in_this_layer.contains(&idx) { + state.apply_diff( + internal_diff.into_internal().unwrap(), + DiffApplyContext { + mode: diff.diff_mode, + doc: &self.doc, + }, + )?; + state.to_diff(&self.doc) + } else { + state.apply_diff_and_convert( + internal_diff.into_internal().unwrap(), + DiffApplyContext { + mode: diff.diff_mode, + doc: &self.doc, + }, + ) + }; + trigger_on_new_container( + &external_diff, + |cid| { + to_revive_in_next_layer.insert(cid); + }, + &self.arena, + ); + diff.diff = external_diff.into(); + } else { + state.apply_diff( + internal_diff.into_internal().unwrap(), + DiffApplyContext { + mode: diff.diff_mode, + doc: &self.doc, + }, + )?; + } + Ok(()) + }, + )?; + } + crate::event::DiffVariant::External(_) => unreachable!(), + } + + to_revive_in_this_layer.remove(&idx); + if !diff.diff.is_empty() { + diffs.push(diff); + } + } + + // Revive the last several layers + while !to_revive_in_this_layer.is_empty() || !to_revive_in_next_layer.is_empty() { + let to_create = std::mem::take(&mut to_revive_in_this_layer); + for new in to_create { + let state = self.store.get_or_create_mut(new); + if state.is_state_empty() { + continue; + } + + let external_diff = state.to_diff(&self.doc); + trigger_on_new_container( + &external_diff, + |cid| { + to_revive_in_next_layer.insert(cid); + }, + &self.arena, + ); + + if !external_diff.is_empty() { + diffs.push(InternalContainerDiff { + idx: new, + bring_back: true, + diff: external_diff.into(), + diff_mode: DiffMode::Checkout, + }); + } + } + + to_revive_in_this_layer = std::mem::take(&mut to_revive_in_next_layer); + } + + diff.diff = diffs.into(); + self.frontiers = diff.new_version.clone().into_owned(); + + if self.is_recording() { + self.record_diff(diff) + } + Ok(()) + } + + fn validate_diff_batch(&mut self, diffs: &[InternalContainerDiff]) -> LoroResult<()> { + for diff in diffs { + let crate::event::DiffVariant::Internal(internal_diff) = &diff.diff else { + continue; + }; + if let Some(state) = self.store.get_container(diff.idx) { + state.validate_diff(internal_diff)?; + } else { + let state = create_state_(diff.idx, &self.config, self.peer_id()); + state.validate_diff(internal_diff)?; + } + } + + Ok(()) + } + + pub fn apply_local_op(&mut self, raw_op: &RawOp, op: &Op) -> LoroResult<()> { + // set parent first, `MapContainer` will only be created for TreeID that does not contain + self.set_container_parent_by_raw_op(raw_op); + let state = self.store.get_or_create_mut(op.container); + if self.in_txn { + self.changed_idx_in_txn.insert(op.container); + } + let ret = state.apply_local_op(raw_op, op)?; + if !ret.deleted_containers.is_empty() { + self.dead_containers_cache.clear_alive(); + } + + Ok(()) + } + + pub(crate) fn start_txn(&mut self, origin: InternalString, trigger: EventTriggerKind) { + self.pre_txn(origin, trigger); + self.in_txn = true; + } + + pub(crate) fn abort_txn(&mut self) { + self.in_txn = false; + } + + pub fn iter_and_decode_all(&mut self) -> impl Iterator { + self.store.iter_and_decode_all() + } + + pub(crate) fn iter_all_containers_mut( + &mut self, + ) -> impl Iterator { + self.store.iter_all_containers() + } + + pub fn does_container_exist(&mut self, id: &ContainerID) -> bool { + // A container may exist even if not yet registered in the arena. + // Check arena first, then fall back to KV presence in the store. + let is_mergeable = id.is_mergeable(); + if id.is_root() && !is_mergeable { + return true; + } + + if !is_mergeable { + if let Some(idx) = self.arena.id_to_idx(id) { + if self.arena.get_depth(idx).is_some() { + return true; + } + } + } + + if self.store.contains_id(id) { + return true; + } + + // An ensured-but-empty mergeable child has no ops or KV state of its own yet; + // the parent map's binary child ref is the source of truth for its existence. + is_mergeable && self.get_reachable(id) + } + + pub(crate) fn commit_txn(&mut self, new_frontiers: Frontiers, diff: Option) { + self.in_txn = false; + self.frontiers = new_frontiers; + if let Some(diff) = diff { + if self.is_recording() { + self.record_diff(diff); + } + } + } + + /// Ensure the container is created and will be encoded in the next `encode` call + #[inline] + pub(crate) fn ensure_container(&mut self, id: &ContainerID) { + self.store.ensure_container(id); + } + + /// Ensure all alive containers are created in DocState and will be encoded in the next `encode` call + pub(crate) fn ensure_all_alive_containers(&mut self) -> FxHashSet { + // TODO: PERF This can be optimized because we shouldn't need to call get_value for + // all the containers every time we export + let ans = self.get_all_alive_containers(); + for id in ans.iter() { + self.ensure_container(id); + } + + ans + } + + pub(crate) fn get_value_by_idx(&mut self, container_idx: ContainerIdx) -> LoroValue { + self.store + .get_value(container_idx) + .unwrap_or_else(|| container_idx.get_type().default_value()) + } + + pub(crate) fn get_map_value_by_key( + &mut self, + container_idx: ContainerIdx, + key: &str, + ) -> Option { + self.store.map_get(container_idx, key) + } + + pub(crate) fn get_map_len(&mut self, container_idx: ContainerIdx) -> usize { + self.store.map_len(container_idx) + } + + pub(crate) fn get_map_keys(&mut self, container_idx: ContainerIdx) -> Vec { + self.store.map_keys(container_idx) + } + + pub(crate) fn get_map_entries( + &mut self, + container_idx: ContainerIdx, + ) -> Vec<(InternalString, LoroValue)> { + self.store.map_entries(container_idx) + } + + pub(crate) fn get_list_value_at( + &mut self, + container_idx: ContainerIdx, + index: usize, + ) -> Option { + self.store.list_get(container_idx, index) + } + + pub(crate) fn get_list_len(&mut self, container_idx: ContainerIdx) -> usize { + self.store.list_len(container_idx) + } + + pub(crate) fn get_list_values(&mut self, container_idx: ContainerIdx) -> Vec { + self.store.list_values(container_idx) + } + + pub(crate) fn get_text_unicode_len(&mut self, container_idx: ContainerIdx) -> usize { + self.store.text_unicode_len(container_idx).unwrap_or(0) + } + + pub(crate) fn get_text_utf16_len(&mut self, container_idx: ContainerIdx) -> usize { + self.store.text_utf16_len(container_idx).unwrap_or(0) + } + + pub(crate) fn get_text_utf8_len(&mut self, container_idx: ContainerIdx) -> usize { + self.store.text_utf8_len(container_idx).unwrap_or(0) + } + + pub(crate) fn has_decoded_container_state(&mut self, container_idx: ContainerIdx) -> bool { + self.store.has_decoded_state(container_idx) + } + + /// Length of a text container in the given `pos_type`, taking a single + /// `DocState` lock and a single container-store lookup. + /// + /// The per-`pos_type` store helpers already branch on decoded-vs-lazy + /// internally, and their lazy branch reads the cheap length metadata + /// without materializing the full richtext state — preserving the + /// lazy-snapshot memory behavior. Callers previously took two separate + /// locks (one to check decoded-ness, one to query), which showed up as a + /// per-op regression on the text editing hot path. Only `Entity` length has + /// no store helper and falls back to the state path. + pub(crate) fn get_text_len(&mut self, container_idx: ContainerIdx, pos_type: PosType) -> usize { + match pos_type { + PosType::Unicode => self.get_text_unicode_len(container_idx), + PosType::Utf16 => self.get_text_utf16_len(container_idx), + PosType::Event if cfg!(feature = "wasm") => self.get_text_utf16_len(container_idx), + PosType::Event => self.get_text_unicode_len(container_idx), + PosType::Bytes => self.get_text_utf8_len(container_idx), + PosType::Entity => self.with_state_mut(container_idx, |state| { + state.as_richtext_state_mut().unwrap().len(PosType::Entity) + }), + } + } + + /// Set the state of the container with the given container idx. + /// This is only used for decode. + /// + /// # Panic + /// + /// If the state is not empty. + pub(super) fn init_with_states_and_version( + &mut self, + frontiers: Frontiers, + oplog: &OpLog, + unknown_containers: Vec, + need_to_register_parent: bool, + origin: InternalString, + ) -> LoroResult<()> { + self.pre_txn(Default::default(), EventTriggerKind::Import); + if need_to_register_parent { + for state in self.store.iter_and_decode_all() { + let idx = state.container_idx(); + let s = state; + for child_id in s.get_child_containers() { + let child_idx = self.arena.register_container(&child_id); + self.arena.set_parent(child_idx, Some(idx)); + } + } + } + + if !unknown_containers.is_empty() { + let mut diff_calc = DiffCalculator::new(false); + let stack_vv; + let vv = if oplog.frontiers() == &frontiers { + oplog.vv() + } else { + stack_vv = oplog.dag().frontiers_to_vv(&frontiers); + stack_vv.as_ref().unwrap() + }; + + let (unknown_diffs, _diff_mode) = diff_calc.calc_diff_internal( + oplog, + &Default::default(), + &Default::default(), + vv, + &frontiers, + Some(&|idx| !idx.is_unknown() && unknown_containers.contains(&idx)), + ); + self.apply_diff( + InternalDocDiff { + origin: origin.clone(), + by: EventTriggerKind::Import, + diff: unknown_diffs.into(), + new_version: Cow::Owned(frontiers.clone()), + }, + DiffMode::Checkout, + )?; + } + + if self.is_recording() { + let diff: Vec<_> = self + .store + .iter_all_containers() + .map(|(idx, state)| InternalContainerDiff { + idx, + bring_back: false, + diff: state + .get_state_mut( + idx, + ContainerCreationContext { + configure: &self.config, + peer: self.peer.load(Ordering::Relaxed), + }, + ) + .to_diff(&self.doc) + .into(), + diff_mode: DiffMode::Checkout, + }) + .collect(); + + self.record_diff(InternalDocDiff { + origin, + by: EventTriggerKind::Import, + diff: diff.into(), + new_version: Cow::Borrowed(&frontiers), + }); + } + + self.frontiers = frontiers; + Ok(()) + } + + #[inline(always)] + #[allow(unused)] + pub(crate) fn with_state(&mut self, idx: ContainerIdx, f: F) -> R + where + F: FnOnce(&State) -> R, + { + let depth = self.arena.get_depth(idx).unwrap().get() as usize; + let state = self.store.get_or_create_imm(idx); + f(state) + } + + #[inline(always)] + pub(crate) fn with_state_mut(&mut self, idx: ContainerIdx, f: F) -> R + where + F: FnOnce(&mut State) -> R, + { + let state = self.store.get_or_create_mut(idx); + f(state) + } + + pub(super) fn is_in_txn(&self) -> bool { + self.in_txn + } + + pub fn can_import_snapshot(&self) -> bool { + !self.in_txn && self.arena.can_import_snapshot() && self.store.can_import_snapshot() + } + + pub(crate) fn reset_to_empty_for_failed_snapshot_import(&mut self) { + let was_recording = self.is_recording(); + self.frontiers = Frontiers::default(); + self.store = + ContainerStore::new(self.arena.clone(), self.config.clone(), self.peer.clone()); + self.in_txn = false; + self.changed_idx_in_txn.clear(); + self.event_recorder = Default::default(); + if was_recording { + self.start_recording(); + } + self.dead_containers_cache = Default::default(); + } + + pub fn get_value(&mut self) -> LoroValue { + let roots = self.preferred_root_containers(); + let ans: loro_common::LoroMapValue = roots + .into_iter() + .map(|idx| { + let id = self.arena.idx_to_id(idx).unwrap(); + let ContainerID::Root { + name, + container_type: _, + } = &id + else { + unreachable!() + }; + (name.to_string(), LoroValue::Container(id)) + }) + .collect(); + LoroValue::Map(ans) + } + + pub fn get_deep_value(&mut self) -> LoroValue { + let roots = self.preferred_root_containers(); + let mut ans = FxHashMap::with_capacity_and_hasher(roots.len(), Default::default()); + let binding = self.config.deleted_root_containers.clone(); + let deleted_root_container = binding.lock(); + let should_hide_empty_root_container = self + .config + .hide_empty_root_containers + .load(Ordering::Relaxed); + for root_idx in roots { + let id = self.arena.idx_to_id(root_idx).unwrap(); + match &id { + loro_common::ContainerID::Root { name, .. } => { + let v = self.get_container_deep_value(root_idx); + if should_hide_empty_root_container + && visible_container_value_is_empty(root_idx.get_type(), &v) + { + continue; + } + + if deleted_root_container.contains(&id) + && deleted_root_container_value_is_cleared(root_idx.get_type(), &v) + { + continue; + } + + ans.insert(name.to_string(), v); + } + loro_common::ContainerID::Normal { .. } => { + unreachable!() + } + } + } + + LoroValue::Map(ans.into()) + } + + pub fn get_deep_value_with_id(&mut self) -> LoroValue { + let roots = self.preferred_root_containers(); + let mut ans = FxHashMap::with_capacity_and_hasher(roots.len(), Default::default()); + for root_idx in roots { + let id = self.arena.idx_to_id(root_idx).unwrap(); + match id.clone() { + loro_common::ContainerID::Root { name, .. } => { + ans.insert( + name.to_string(), + self.get_container_deep_value_with_id(root_idx, Some(id)), + ); + } + loro_common::ContainerID::Normal { .. } => { + unreachable!() + } + } + } + + LoroValue::Map(ans.into()) + } + + pub(crate) fn preferred_root_containers(&mut self) -> Vec { + let flag = self.store.load_root_containers(); + // Mergeable cids live in a private namespace and are logically children of a regular + // Map — they must not appear here. `top_level_root_containers` already excludes them, + // so this loop is O(top_level_roots) regardless of the number of mergeable cids. + let roots = self.arena.top_level_root_containers(flag); + let mut selected = FxHashMap::default(); + let mut names = Vec::new(); + + for idx in roots { + let Some(id) = self.arena.idx_to_id(idx) else { + continue; + }; + if !self.store.contains_id(&id) { + continue; + } + let Some(name) = self.root_container_name(idx) else { + continue; + }; + let is_empty = self.root_container_is_empty(idx); + match selected.entry(name.clone()) { + std::collections::hash_map::Entry::Vacant(entry) => { + names.push(name); + entry.insert((idx, is_empty)); + } + std::collections::hash_map::Entry::Occupied(mut entry) => { + let (_, selected_is_empty) = entry.get(); + // Keep the previous last-root-wins behavior, except an empty + // root should not hide a non-empty root with the same name. + if *selected_is_empty || !is_empty { + entry.insert((idx, is_empty)); + } + } + } + } + + names + .into_iter() + .filter_map(|name| selected.remove(&name).map(|(idx, _)| idx)) + .collect() + } + + pub(crate) fn preferred_root_container_idx_by_key( + &mut self, + root_index: &InternalString, + ) -> Option { + let flag = self.store.load_root_containers(); + // Same reasoning as `preferred_root_containers`: only top-level roots are eligible. + let roots = self.arena.top_level_root_containers(flag); + let mut selected = None; + + for idx in roots { + let Some(id) = self.arena.idx_to_id(idx) else { + continue; + }; + if !self.store.contains_id(&id) { + continue; + } + let Some(name) = self.root_container_name(idx) else { + continue; + }; + if &name != root_index { + continue; + } + + let is_empty = self.root_container_is_empty(idx); + match selected { + None => selected = Some((idx, is_empty)), + Some((_, selected_is_empty)) => { + if selected_is_empty || !is_empty { + selected = Some((idx, is_empty)); + } + } + } + } + + selected.map(|(idx, _)| idx) + } + + fn root_container_name(&self, idx: ContainerIdx) -> Option { + match self.arena.idx_to_id(idx)? { + ContainerID::Root { name, .. } => Some(name), + ContainerID::Normal { .. } => None, + } + } + + fn root_container_is_empty(&mut self, idx: ContainerIdx) -> bool { + let value = self + .store + .get_value(idx) + .unwrap_or_else(|| idx.get_type().default_value()); + visible_container_value_is_empty(idx.get_type(), &value) + } + + pub fn get_all_container_value_flat(&mut self) -> LoroValue { + let mut map = FxHashMap::default(); + self.store.iter_and_decode_all().for_each(|c| { + let value = c.get_value(); + let cid = self.arena.idx_to_id(c.container_idx()).unwrap().to_string(); + map.insert(cid, value); + }); + + LoroValue::Map(map.into()) + } + + pub(crate) fn get_container_deep_value_with_id( + &mut self, + container: ContainerIdx, + id: Option, + ) -> LoroValue { + let id = id.unwrap_or_else(|| self.arena.idx_to_id(container).unwrap()); + let Some(value) = self.store.get_value(container) else { + return container.get_type().default_value(); + }; + let cid_str = LoroValue::String(format!("idx:{}, id:{}", container.to_index(), id).into()); + match value { + LoroValue::Container(_) => unreachable!(), + LoroValue::List(mut list) => { + if container.get_type() == ContainerType::Tree { + get_meta_value(list.make_mut(), self); + } else { + if list.iter().all(|x| !x.is_container()) { + return LoroValue::Map( + (fx_map!( + "cid".into() => cid_str, + "value".into() => LoroValue::List(list) + )) + .into(), + ); + } + + let list_mut = list.make_mut(); + for item in list_mut.iter_mut() { + if item.is_container() { + let container = item.as_container().unwrap(); + let container_idx = self.arena.register_container(container); + let value = self.get_container_deep_value_with_id( + container_idx, + Some(container.clone()), + ); + *item = value; + } + } + } + LoroValue::Map( + (fx_map!( + "cid".into() => cid_str, + "value".into() => LoroValue::List(list) + )) + .into(), + ) + } + LoroValue::Map(mut map) => { + // A map's mergeable children are encoded as compact binary markers in the map's + // own value table (loro-dev/loro#759), so they are already present in `map` here. + // Derive the active children straight from that value — re-fetching the + // `MapState` would force the snapshot-backed container to decode and break the + // lazy-value invariant for roots. + let mergeable_children = self.mergeable_children_from_value(&id, &map); + + let map_mut = map.make_mut(); + for (_key, value) in map_mut.iter_mut() { + if value.is_container() { + let container = value.as_container().unwrap(); + let container_idx = self.arena.register_container(container); + let new_value = self.get_container_deep_value_with_id( + container_idx, + Some(container.clone()), + ); + *value = new_value; + } + } + // Replace each marker with the nested deep value of its resolved child, keyed + // under the same logical key. + for (key, cid) in mergeable_children { + let child_idx = self.arena.register_container(&cid); + let new_value = self.get_container_deep_value_with_id(child_idx, Some(cid)); + map_mut.insert(key.to_string(), new_value); + } + + LoroValue::Map( + (fx_map!( + "cid".into() => cid_str, + "value".into() => LoroValue::Map(map) + )) + .into(), + ) + } + _ => LoroValue::Map( + (fx_map!( + "cid".into() => cid_str, + "value".into() => value + )) + .into(), + ), + } + } + + pub fn get_container_deep_value(&mut self, container: ContainerIdx) -> LoroValue { + let Some(value) = self.store.get_value(container) else { + return container.get_type().default_value(); + }; + match value { + LoroValue::Container(_) => unreachable!(), + LoroValue::List(mut list) => { + if container.get_type() == ContainerType::Tree { + // Each tree node has an associated map container to represent + // the metadata of this node. When the user get the deep value, + // we need to add a field named `meta` to the tree node, + // whose value is deep value of map container. + get_meta_value(list.make_mut(), self); + } else { + if list.iter().all(|x| !x.is_container()) { + return LoroValue::List(list); + } + + let list_mut = list.make_mut(); + for item in list_mut.iter_mut() { + if item.is_container() { + let container = item.as_container().unwrap(); + let container_idx = self.arena.register_container(container); + let value = self.get_container_deep_value(container_idx); + *item = value; + } + } + } + LoroValue::List(list) + } + LoroValue::Map(mut map) => { + // A map's mergeable children are encoded as compact binary markers in the map's + // own value table (loro-dev/loro#759), so they are already present in `map` here. + // Derive the active children straight from that value — re-fetching the + // `MapState` would force the snapshot-backed container to decode and break the + // lazy-value invariant for roots. + let mergeable_children = self + .arena + .idx_to_id(container) + .map(|parent_id| self.mergeable_children_from_value(&parent_id, &map)) + .unwrap_or_default(); + + if mergeable_children.is_empty() && map.iter().all(|x| !x.1.is_container()) { + return LoroValue::Map(map); + } + + let map_mut = map.make_mut(); + for (_key, value) in map_mut.iter_mut() { + if value.is_container() { + let container = value.as_container().unwrap(); + let container_idx = self.arena.register_container(container); + let new_value = self.get_container_deep_value(container_idx); + *value = new_value; + } + } + // Replace each marker with the nested deep value of its resolved child, keyed + // under the same logical key. + for (key, cid) in mergeable_children { + let child_idx = self.arena.register_container(&cid); + let new_value = self.get_container_deep_value(child_idx); + map_mut.insert(key.to_string(), new_value); + } + LoroValue::Map(map) + } + _ => value, + } + } + + pub(crate) fn get_all_alive_containers(&mut self) -> FxHashSet { + let flag = self.store.load_all(); + let mut ans = FxHashSet::default(); + let mut to_visit = self + .arena + .root_containers(flag) + .iter() + .map(|x| self.arena.get_container_id(*x).unwrap()) + .filter(|id| self.store.contains_id(id)) + .collect_vec(); + + while let Some(id) = to_visit.pop() { + self.get_alive_children_of(&id, &mut to_visit); + ans.insert(id); + } + + ans + } + + pub(crate) fn get_alive_children_of(&mut self, id: &ContainerID, ans: &mut Vec) { + let idx = self.arena.register_container(id); + let Some(value) = self.store.get_value(idx) else { + return; + }; + + match value { + LoroValue::Container(_) => unreachable!(), + LoroValue::List(list) => { + if idx.get_type() == ContainerType::Tree { + // Each tree node has an associated map container to represent + // the metadata of this node. When the user get the deep value, + // we need to add a field named `meta` to the tree node, + // whose value is deep value of map container. + let mut list = list.unwrap(); + while let Some(node) = list.pop() { + let map = node.as_map().unwrap(); + let meta = map.get("meta").unwrap(); + let id = meta.as_container().unwrap(); + ans.push(id.clone()); + let children = map.get("children").unwrap(); + let children = children.as_list().unwrap(); + for child in children.iter() { + list.push(child.clone()); + } + } + } else { + for item in list.iter() { + if let LoroValue::Container(id) = item { + ans.push(id.clone()); + } + } + } + } + LoroValue::Map(map) => { + for (_key, value) in map.iter() { + if let LoroValue::Container(id) = value { + ans.push(id.clone()); + } + } + // Mergeable children are resolved from compact binary markers stored in this + // map's own value table (loro-dev/loro#759): the active child at each key is the + // deterministic cid for whichever marker the map's regular LWW resolved to. + // Derive them from the value we already fetched lazily above — re-reading the + // decoded `MapState` would force a snapshot-backed map to materialize its full + // state. Pull them in so alive-container walks (notably shallow snapshot export) + // include mergeable cids and don't filter their KV out by `retain_keys`. + let mergeable_cids: Vec = self + .arena + .idx_to_id(idx) + .map(|parent_id| { + self.mergeable_children_from_value(&parent_id, &map) + .into_iter() + .map(|(_key, cid)| cid) + .collect() + }) + .unwrap_or_default(); + for cid in mergeable_cids { + self.arena.register_container(&cid); + ans.push(cid); + } + } + _ => {} + } + } + + // Because we need to calculate path based on [DocState], so we cannot extract + // the event recorder to a separate module. + fn diffs_to_event(&mut self, diffs: Vec>, from: Frontiers) -> DocDiff { + if diffs.is_empty() { + panic!("diffs is empty"); + } + + let triggered_by = diffs[0].by; + debug_assert!(diffs.iter().all(|x| x.by == triggered_by)); + let mut containers = FxHashMap::default(); + let to = (*diffs.last().unwrap().new_version).to_owned(); + let origin = diffs[0].origin.clone(); + for diff in diffs { + #[allow(clippy::unnecessary_to_owned)] + for container_diff in diff.diff.into_owned() { + let Some((last_container_diff, _)) = containers.get_mut(&container_diff.idx) else { + if let Some(path) = self.get_path(container_diff.idx) { + containers.insert(container_diff.idx, (container_diff.diff, path)); + } else { + // if we cannot find the path to the container, the container must be overwritten afterwards. + // So we can ignore the diff from it. + let _container_id = self + .arena + .idx_to_id(container_diff.idx) + .map(|x| x.to_string()) + .unwrap_or_else(|| "unknown".to_string()); + #[cfg(feature = "logging")] + loro_common::warn!( + "⚠️ WARNING: ignore event because cannot find its path {:#?} container id:{}", + &container_diff, + _container_id + ); + } + + continue; + }; + // Compose in place. Cloning the accumulated diff here made a + // batch of N same-container fragments O(N^2) (each compose + // cloned the growing accumulator), which is hit whenever a + // subscriber is attached and many edits land on one container + // in a single event batch. + let prev = std::mem::take(last_container_diff); + *last_container_diff = prev.compose(container_diff.diff).unwrap(); + } + } + let mut diff: Vec<_> = containers + .into_iter() + .map(|(container, (diff, path))| { + let idx = container; + let id = self.arena.get_container_id(idx).unwrap(); + let is_unknown = id.is_unknown(); + + ContainerDiff { + id, + idx, + diff: diff.into_external().unwrap(), + is_unknown, + path, + } + }) + .collect(); + + // Sort by path length, so caller can apply the diff from the root to the leaf. + // Otherwise, the caller may use a wrong path to apply the diff. + + diff.sort_by_key(|x| { + ( + x.path.len(), + match &x.id { + ContainerID::Root { .. } => 0, + ContainerID::Normal { counter, .. } => *counter + 1, + }, + ) + }); + DocDiff { + from, + to, + origin, + by: triggered_by, + diff, + } + } + + pub(crate) fn get_reachable(&mut self, id: &ContainerID) -> bool { + if id.is_root() && !id.is_mergeable() { + return true; + } + + // If not registered yet, check KV presence for ordinary containers, then register lazily. + // Mergeable children can be active through a parent marker before they have their + // own KV state, so their reachability must be resolved by the logical edge walk below. + if self.arena.id_to_idx(id).is_none() { + if !id.is_mergeable() && !self.does_container_exist(id) { + return false; + } + self.arena.register_container(id); + } + + let mut idx = self.arena.id_to_idx(id).unwrap(); + loop { + let id = self.arena.idx_to_id(idx).unwrap(); + if let Some(parent_idx) = self.arena.get_parent(idx) { + if !self.contains_logical_child(parent_idx, &id) { + return false; + } + idx = parent_idx; + } else { + if id.is_root() && !id.is_mergeable() { + return true; + } + + return false; + } + } + } + + // the container may be override, so it may return None + pub(super) fn get_path(&mut self, idx: ContainerIdx) -> Option> { + let mut ans = Vec::new(); + let mut idx = idx; + loop { + let id = self.arena.idx_to_id(idx).unwrap(); + if let Some(parent_idx) = self.arena.get_parent(idx) { + let Some(prop) = self.get_logical_child_index(parent_idx, &id) else { + tracing::warn!("Missing in parent's children"); + return None; + }; + ans.push((id, prop)); + idx = parent_idx; + } else { + // this container may be deleted + if id.is_mergeable() { + tracing::info!(id = %id, "Missing parent - mergeable container is inactive"); + return None; + } + let Ok(prop) = id.clone().into_root() else { + let id = format!("{}", &id); + tracing::info!(?id, "Missing parent - container is deleted"); + return None; + }; + ans.push((id, Index::Key(prop.0))); + break; + } + } + + ans.reverse(); + + Some(ans) + } + + pub(crate) fn check_before_decode_snapshot(&self) -> LoroResult<()> { + if self.is_in_txn() { + return Err(LoroError::DecodeError( + "State is in txn".to_string().into_boxed_str(), + )); + } + + if !self.can_import_snapshot() { + return Err(LoroError::DecodeError( + "State is not empty, cannot import snapshot directly" + .to_string() + .into_boxed_str(), + )); + } + + Ok(()) + } + + /// Check whether two [DocState]s are the same. Panic if not. + /// + /// Compared to check equality on `get_deep_value`, this function also checks the equality on richtext + /// styles and states that are not reachable from the root. + /// + /// This is only used for test. + pub(crate) fn check_is_the_same(&mut self, other: &mut Self) { + fn get_entries_for_state( + arena: &SharedArena, + state: &mut State, + ) -> Option<(ContainerID, (ContainerIdx, LoroValue))> { + if state.is_state_empty() { + return None; + } + + let id = arena.idx_to_id(state.container_idx()).unwrap(); + let value = match state { + State::RichtextState(s) => s.get_richtext_value(), + _ => state.get_value(), + }; + if match &value { + LoroValue::List(l) => l.is_empty(), + LoroValue::Map(m) => m.is_empty(), + _ => false, + } { + return None; + } + #[cfg(feature = "counter")] + if id.container_type() == ContainerType::Counter { + if let LoroValue::Double(c) = value { + if c.abs() < f64::EPSILON { + return None; + } + } + } + + Some((id, (state.container_idx(), value))) + } + + let self_id_to_states: FxHashMap = self + .store + .iter_and_decode_all() + .filter_map(|state: &mut State| { + let arena = &self.arena; + get_entries_for_state(arena, state) + }) + .collect(); + let mut other_id_to_states: FxHashMap = other + .store + .iter_and_decode_all() + .filter_map(|state: &mut State| { + let arena = &other.arena; + get_entries_for_state(arena, state) + }) + .collect(); + for (id, (idx, this_value)) in self_id_to_states { + let (_, other_value) = match other_id_to_states.remove(&id) { + Some(x) => x, + None => { + panic!( + "id: {:?}, path: {:?} is missing, value={:?}", + id, + self.get_path(idx), + &this_value + ); + } + }; + + pretty_assertions::assert_eq!( + this_value, + other_value, + "[self!=other] id: {:?}, path: {:?}", + id, + self.get_path(idx) + ); + } + + if !other_id_to_states.is_empty() { + panic!("other has more states {:#?}", &other_id_to_states); + } + } + + pub fn create_state(&self, idx: ContainerIdx) -> State { + let config = &self.config; + let peer = self.peer.load(std::sync::atomic::Ordering::Relaxed); + create_state_(idx, config, peer) + } + + pub fn create_unknown_state(&self, idx: ContainerIdx) -> State { + State::UnknownState(UnknownState::new(idx)) + } + + pub fn get_relative_position(&mut self, pos: &Cursor, use_event_index: bool) -> Option { + let idx = self.arena.register_container(&pos.container); + let state = self.store.get_container_mut(idx)?; + if let Some(id) = pos.id { + match state { + State::ListState(s) => s.get_index_of_id(id), + State::RichtextState(s) => s.get_text_index_of_id(id, use_event_index), + State::MovableListState(s) => s.get_index_of_id(id), + State::MapState(_) | State::TreeState(_) | State::UnknownState(_) => unreachable!(), + #[cfg(feature = "counter")] + State::CounterState(_) => unreachable!(), + } + } else { + if matches!(pos.side, crate::cursor::Side::Left) { + return Some(0); + } + + match state { + State::ListState(s) => Some(s.len()), + State::RichtextState(s) => Some(if use_event_index { + s.len_event() + } else { + s.len_unicode() + }), + State::MovableListState(s) => Some(s.len()), + State::MapState(_) | State::TreeState(_) | State::UnknownState(_) => unreachable!(), + #[cfg(feature = "counter")] + State::CounterState(_) => unreachable!(), + } + } + } + + pub fn get_value_by_path(&mut self, path: &[Index]) -> Option { + if path.is_empty() { + return None; + } + + enum CurContainer { + Container(ContainerIdx), + TreeNode { + tree: ContainerIdx, + node: Option, + }, + } + + let mut state_idx = { + let root_index = path[0].as_key()?; + CurContainer::Container(self.preferred_root_container_idx_by_key(root_index)?) + }; + + if path.len() == 1 { + if let CurContainer::Container(c) = state_idx { + let cid = self.arena.idx_to_id(c)?; + return Some(LoroValue::Container(cid)); + } + } + + let mut i = 1; + while i < path.len() - 1 { + let index = &path[i]; + match state_idx { + CurContainer::Container(idx) => { + let parent_id = self.arena.idx_to_id(idx); + let parent_state = self.store.get_container_mut(idx)?; + match parent_state { + State::ListState(l) => { + let Some(LoroValue::Container(c)) = l.get(*index.as_seq()?) else { + return None; + }; + state_idx = CurContainer::Container(self.arena.register_container(c)); + } + State::MovableListState(l) => { + let Some(LoroValue::Container(c)) = + l.get(*index.as_seq()?, IndexType::ForUser) + else { + return None; + }; + state_idx = CurContainer::Container(self.arena.register_container(c)); + } + State::MapState(m) => { + let key = index.as_key()?; + let value = m.get(key)?; + let c = match value { + LoroValue::Container(c) => c.clone(), + value => { + let parent_id = parent_id?; + let kind = loro_common::parse_mergeable_marker( + &parent_id, key, value, + )?; + ContainerID::new_mergeable(&parent_id, key, kind) + } + }; + state_idx = CurContainer::Container(self.arena.register_container(&c)); + } + State::RichtextState(_) => return None, + State::TreeState(_) => { + state_idx = CurContainer::TreeNode { + tree: idx, + node: None, + }; + continue; + } + #[cfg(feature = "counter")] + State::CounterState(_) => return None, + State::UnknownState(_) => unreachable!(), + } + } + CurContainer::TreeNode { tree, node } => match index { + Index::Key(internal_string) => { + let node = node?; + let idx = self + .arena + .register_container(&node.associated_meta_container()); + let map = self.store.get_container(idx)?; + let Some(LoroValue::Container(c)) = + map.as_map_state().unwrap().get(internal_string) + else { + return None; + }; + + state_idx = CurContainer::Container(self.arena.register_container(c)); + } + Index::Seq(i) => { + let tree_state = + self.store.get_container_mut(tree)?.as_tree_state().unwrap(); + let parent: TreeParentId = if let Some(node) = node { + node.into() + } else { + TreeParentId::Root + }; + let child = tree_state.get_children(&parent)?.nth(*i)?; + state_idx = CurContainer::TreeNode { + tree, + node: Some(child), + }; + } + Index::Node(tree_id) => { + let tree_state = + self.store.get_container_mut(tree)?.as_tree_state().unwrap(); + if tree_state.parent(tree_id).is_some() { + state_idx = CurContainer::TreeNode { + tree, + node: Some(*tree_id), + } + } else { + return None; + } + } + }, + } + i += 1; + } + + let parent_idx = match state_idx { + CurContainer::Container(container_idx) => container_idx, + CurContainer::TreeNode { tree, node } => { + if let Some(node) = node { + self.arena + .register_container(&node.associated_meta_container()) + } else { + tree + } + } + }; + + let index = path.last().unwrap(); + let parent_id = self.arena.idx_to_id(parent_idx); + let parent_state = self.store.get_or_create_mut(parent_idx); + let value: LoroValue = match parent_state { + State::ListState(l) => l.get(*index.as_seq()?).cloned()?, + State::MovableListState(l) => l.get(*index.as_seq()?, IndexType::ForUser).cloned()?, + State::MapState(m) => { + if let Some(key) = index.as_key() { + let value = m.get(key).cloned()?; + if let Some(parent_id) = &parent_id { + if let Some(kind) = + loro_common::parse_mergeable_marker(parent_id, key, &value) + { + let cid = ContainerID::new_mergeable(parent_id, key, kind); + LoroValue::Container(cid) + } else { + value + } + } else { + value + } + } else if let CurContainer::TreeNode { tree, node } = state_idx { + match index { + Index::Seq(index) => { + let tree_state = + self.store.get_container_mut(tree)?.as_tree_state().unwrap(); + let parent: TreeParentId = if let Some(node) = node { + node.into() + } else { + TreeParentId::Root + }; + let child = tree_state.get_children(&parent)?.nth(*index)?; + child.associated_meta_container().into() + } + Index::Node(id) => id.associated_meta_container().into(), + _ => return None, + } + } else { + return None; + } + } + State::RichtextState(s) => { + let s = s.to_string_mut(); + s.chars() + .nth(*index.as_seq()?) + .map(|c| c.to_string().into())? + } + State::TreeState(_) => { + let id = index.as_node()?; + let cid = id.associated_meta_container(); + cid.into() + } + #[cfg(feature = "counter")] + State::CounterState(_) => unreachable!(), + State::UnknownState(_) => unreachable!(), + }; + + Some(value) + } + + pub(crate) fn shallow_root_store(&self) -> Option<&Arc> { + self.store.shallow_root_store() + } +} + +fn create_state_(idx: ContainerIdx, config: &Configure, peer: u64) -> State { + match idx.get_type() { + ContainerType::Map => State::MapState(Box::new(MapState::new(idx))), + ContainerType::List => State::ListState(Box::new(ListState::new(idx))), + ContainerType::Text => State::RichtextState(Box::new(RichtextState::new( + idx, + config.text_style_config.clone(), + ))), + ContainerType::Tree => State::TreeState(Box::new(TreeState::new(idx, peer))), + ContainerType::MovableList => State::MovableListState(Box::new(MovableListState::new(idx))), + #[cfg(feature = "counter")] + ContainerType::Counter => { + State::CounterState(Box::new(counter_state::CounterState::new(idx))) + } + ContainerType::Unknown(_) => State::UnknownState(UnknownState::new(idx)), + } +} + +fn trigger_on_new_container( + state_diff: &Diff, + mut listener: impl FnMut(ContainerIdx), + arena: &SharedArena, +) { + match state_diff { + Diff::List(list) => { + for delta in list.iter() { + if let DeltaItem::Replace { + value, + attr, + delete: _, + } = delta + { + if attr.from_move { + continue; + } + + for v in value.iter() { + if let ValueOrHandler::Handler(h) = v { + let idx = h.container_idx(); + listener(idx); + } + } + } + } + } + Diff::Map(map) => { + for (_, v) in map.updated.iter() { + if let Some(ValueOrHandler::Handler(h)) = &v.value { + let idx = h.container_idx(); + listener(idx); + } + } + } + Diff::Tree(tree) => { + for item in tree.iter() { + if matches!(item.action, TreeExternalDiff::Create { .. }) { + let id = item.target.associated_meta_container(); + // Ensure registration instead of assuming it's already in arena + listener(arena.register_container(&id)); + } + } + } + _ => {} + }; +} + +#[derive(Default, Clone)] +struct EventRecorder { + recording_diff: bool, + // A batch of diffs will be converted to a event when + // they cannot be merged with the next diff. + diffs: Vec>, + events: Vec, + diff_start_version: Option, +} + +impl EventRecorder { + #[allow(unused)] + pub fn new() -> Self { + Self::default() + } +} + +#[test] +fn test_size() { + println!("Size of State = {}", std::mem::size_of::()); + println!("Size of MapState = {}", std::mem::size_of::()); + println!("Size of ListState = {}", std::mem::size_of::()); + println!( + "Size of TextState = {}", + std::mem::size_of::() + ); + println!("Size of TreeState = {}", std::mem::size_of::()); +} diff --git a/3rdparty/loro/crates/loro-internal/src/state/AGENTS.md b/3rdparty/loro/crates/loro-internal/src/state/AGENTS.md new file mode 100644 index 0000000000..5c653e67a1 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/state/AGENTS.md @@ -0,0 +1,40 @@ +# State Guidelines + +This module owns materialized document state, container stores, diff application, +checkout/replay behavior, deep/shallow values, and mergeable container +visibility. Read +[../../../../context/mergeable-containers.md](../../../../context/mergeable-containers.md) +before changing mergeable child behavior. + +## Local Entry Points + +- `../state.rs`: `DocState`, checkout/path/deep-value traversal, state replay, + lifecycle, and alive-container discovery. +- `container_store/`: persisted KV-backed container snapshots and + `ContainerWrapper` encoding. +- `map_state.rs`, `list_state.rs`, `richtext_state.rs`, `tree_state.rs`, + `movable_list_state.rs`, `counter_state.rs`: per-container state and snapshot + codecs. +- `mergeable.rs`: logical child edge resolution for mergeable containers. +- `dead_containers_cache.rs`: dead/alive tracking and marker-driven mergeable + reactivation. +- `unknown_state.rs` and `../diff_calc/unknown.rs`: forward compatibility for + unknown container types. + +## Mergeable Rules + +- `MapHandler::ensure_mergeable_*` writes a compact marker into the parent map + and returns a handler for a deterministic `ContainerID`. +- The parent map marker, not "child has ops", decides whether a mergeable child + is visible. +- Non-mergeable occupants must block `ensure_mergeable_*`; same-kind marker + writes are idempotent; different-kind marker writes are deliberate kind + changes. +- Snapshot and shallow snapshot retention must preserve hidden losing-kind + mergeable state. + +## Validation + +- `cargo test -p loro-internal --test mergeable_cid_encoding` +- `cargo test -p loro-internal --test mergeable_container` +- `cargo test -p loro-internal import_atomicity` if import or rollback is involved. diff --git a/3rdparty/loro/crates/loro-internal/src/state/CLAUDE.md b/3rdparty/loro/crates/loro-internal/src/state/CLAUDE.md new file mode 120000 index 0000000000..47dc3e3d86 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/state/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/3rdparty/loro/crates/loro-internal/src/state/analyzer.rs b/3rdparty/loro/crates/loro-internal/src/state/analyzer.rs new file mode 100644 index 0000000000..1ecef1cee7 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/state/analyzer.rs @@ -0,0 +1,93 @@ +use crate::{change::Timestamp, LoroDoc}; +use loro_common::ContainerID; +use rle::HasLength; +use rustc_hash::FxHashMap; + +#[derive(Debug, Clone)] +pub struct DocAnalysis { + pub containers: FxHashMap, +} + +#[derive(Debug, Clone)] +pub struct ContainerAnalysisInfo { + pub size: u32, + pub dropped: bool, + pub depth: u32, + pub ops_num: u32, + pub last_edit_time: Timestamp, +} + +impl DocAnalysis { + pub fn analyze(doc: &LoroDoc) -> Self { + let mut ops_nums = FxHashMap::default(); + let mut last_edit_time = FxHashMap::default(); + { + let oplog = doc.oplog().lock(); + oplog.change_store().visit_all_changes(&mut |c| { + for op in c.ops().iter() { + let idx = op.container; + let info = ops_nums.entry(idx).or_insert(0); + *info += op.atom_len(); + + let time = last_edit_time.entry(idx).or_insert(c.timestamp()); + if *time < c.timestamp() { + *time = c.timestamp(); + } + } + }); + } + + let mut containers = FxHashMap::default(); + let mut state = doc.app_state().lock(); + let alive_containers = state.get_all_alive_containers(); + for (idx, c) in state.iter_all_containers_mut() { + let ops_num = ops_nums.get(&idx).unwrap_or(&0); + let id = doc.arena().get_container_id(idx).unwrap(); + let dropped = !alive_containers.contains(&id); + containers.insert( + id, + ContainerAnalysisInfo { + depth: c.depth() as u32, + dropped, + size: c.encode().len() as u32, + ops_num: *ops_num as u32, + last_edit_time: *last_edit_time.get(&idx).unwrap_or(&0), + }, + ); + } + + Self { containers } + } + + #[allow(unused)] + pub fn len(&self) -> usize { + self.containers.len() + } + + #[allow(unused)] + #[must_use] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + pub fn dropped_len(&self) -> usize { + self.containers + .iter() + .filter(|(_, info)| info.dropped) + .count() + } + + pub fn tiny_container_len(&self) -> usize { + self.containers + .iter() + .filter(|(_, info)| info.size < 128) + .count() + } + + pub fn large_container_len(&self) -> usize { + self.containers + .iter() + .filter(|(_, info)| info.size >= 1024) + .count() + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/state/container_store.rs b/3rdparty/loro/crates/loro-internal/src/state/container_store.rs new file mode 100644 index 0000000000..73d4d8c81d --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/state/container_store.rs @@ -0,0 +1,453 @@ +use super::{ContainerCreationContext, State}; +use crate::arena::LoadAllFlag; +use crate::sync::{AtomicU64, Mutex}; +use crate::{ + arena::SharedArena, configure::Configure, container::idx::ContainerIdx, + utils::kv_wrapper::KvWrapper, version::Frontiers, +}; +use bytes::Bytes; +use inner_store::InnerStore; +use loro_common::{ContainerID, InternalString, LoroResult, LoroValue}; +use std::sync::Arc; + +pub(crate) use container_wrapper::ContainerWrapper; + +mod container_wrapper; +mod inner_store; + +/// Encoding Schema for Container Store +/// +/// KV-Store: +/// - Key: Encoded Container ID +/// - Value: Encoded Container State +/// +/// ─ ─ ─ ─ ─ ─ ─ For Each Container Type ─ ─ ─ ─ ─ ─ ─ ─ +/// +/// ┌────────────────┬──────────────────────────────────┐ +/// │ u16 Depth │ ParentID │ +/// └────────────────┴──────────────────────────────────┘ +/// ┌───────────────────────────────────────────────────┐ +/// │ ┌───────────────────────────────────────────────┐ │ +/// │ │ Into │ │ +/// │ └───────────────────────────────────────────────┘ │ +/// │ │ +/// │ Container Specific Encode │ +/// │ │ +/// │ │ +/// │ │ +/// │ │ +/// └───────────────────────────────────────────────────┘ +pub(crate) struct ContainerStore { + arena: SharedArena, + store: InnerStore, + shallow_root_store: Option>, + conf: Configure, + peer: Arc, +} + +pub(crate) const FRONTIERS_KEY: &[u8] = b"fr"; +impl std::fmt::Debug for ContainerStore { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ContainerStore") + .field("store", &self.store) + .finish() + } +} + +#[derive(Debug)] +pub(crate) struct GcStore { + pub shallow_root_frontiers: Frontiers, + pub encoded_state_bytes: Bytes, + pub store: Mutex, +} + +macro_rules! ctx { + ($self:expr) => { + ContainerCreationContext { + configure: &$self.conf, + peer: $self.peer.load(std::sync::atomic::Ordering::Relaxed), + } + }; +} + +impl ContainerStore { + pub fn new(arena: SharedArena, conf: Configure, peer: Arc) -> Self { + ContainerStore { + store: InnerStore::new(arena.clone(), conf.clone()), + arena, + conf, + shallow_root_store: None, + peer, + } + } + + pub fn can_import_snapshot(&self) -> bool { + if self.shallow_root_store.is_some() { + return false; + } + + self.store.can_import_snapshot() + } + + pub fn get_container_mut(&mut self, idx: ContainerIdx) -> Option<&mut State> { + self.store + .get_mut(idx) + .map(|x| x.get_state_mut(idx, ctx!(self))) + } + + #[allow(unused)] + pub fn get_container(&mut self, idx: ContainerIdx) -> Option<&State> { + self.store + .get_mut(idx) + .map(|x| x.get_state(idx, ctx!(self))) + } + + pub fn shallow_root_store(&self) -> Option<&Arc> { + self.shallow_root_store.as_ref() + } + + pub fn get_value(&mut self, idx: ContainerIdx) -> Option { + self.store + .with_container_for_read(idx, |c| c.get_value(idx, ctx!(self))) + } + + pub fn map_get(&mut self, idx: ContainerIdx, key: &str) -> Option { + self.store + .with_container_for_read(idx, |c| c.map_get(idx, ctx!(self), key))? + } + + pub fn map_len(&mut self, idx: ContainerIdx) -> usize { + self.store + .with_container_for_read(idx, |c| c.map_len(idx, ctx!(self))) + .unwrap_or(0) + } + + pub fn map_keys(&mut self, idx: ContainerIdx) -> Vec { + self.store + .with_container_for_read(idx, |c| c.map_keys(idx, ctx!(self))) + .unwrap_or_default() + } + + pub fn map_entries(&mut self, idx: ContainerIdx) -> Vec<(InternalString, LoroValue)> { + self.store + .with_container_for_read(idx, |c| c.map_entries(idx, ctx!(self))) + .unwrap_or_default() + } + + pub fn list_get(&mut self, idx: ContainerIdx, index: usize) -> Option { + self.store + .with_container_for_read(idx, |c| c.list_get(idx, ctx!(self), index))? + } + + pub fn list_len(&mut self, idx: ContainerIdx) -> usize { + self.store + .with_container_for_read(idx, |c| c.list_len(idx, ctx!(self))) + .unwrap_or(0) + } + + pub fn list_values(&mut self, idx: ContainerIdx) -> Vec { + self.store + .with_container_for_read(idx, |c| c.list_values(idx, ctx!(self))) + .unwrap_or_default() + } + + pub fn text_unicode_len(&mut self, idx: ContainerIdx) -> Option { + self.store + .with_container_for_read(idx, |c| c.text_unicode_len(idx, ctx!(self)))? + } + + pub fn text_utf16_len(&mut self, idx: ContainerIdx) -> Option { + self.store + .with_container_for_read(idx, |c| c.text_utf16_len(idx, ctx!(self)))? + } + + pub fn text_utf8_len(&mut self, idx: ContainerIdx) -> Option { + self.store + .with_container_for_read(idx, |c| c.text_utf8_len(idx, ctx!(self)))? + } + + pub fn has_decoded_state(&mut self, idx: ContainerIdx) -> bool { + self.store.has_decoded_state(idx) + } + + pub fn encode(&mut self) -> Bytes { + self.store.encode() + } + + pub(crate) fn flush(&mut self) { + self.store.flush() + } + + pub fn shallow_root_frontiers(&self) -> Option<&Frontiers> { + self.shallow_root_store + .as_ref() + .map(|x| &x.shallow_root_frontiers) + } + + pub(crate) fn encode_shallow_root_state(&self) -> Option { + let shallow_root = self.shallow_root_store.as_ref()?; + Some(shallow_root.encoded_state_bytes.clone()) + } + + pub(crate) fn shallow_root_state_for_export(&self) -> Option<(Bytes, KvWrapper)> { + let shallow_root = self.shallow_root_store.as_ref()?; + let shallow_root_kv = shallow_root.store.lock().get_kv_clone(); + Some((shallow_root.encoded_state_bytes.clone(), shallow_root_kv)) + } + + pub(crate) fn decode(&mut self, bytes: Bytes) -> LoroResult> { + self.store.decode(bytes) + } + + pub(crate) fn decode_gc( + &mut self, + shallow_bytes: Bytes, + start_frontiers: Frontiers, + config: Configure, + ) -> LoroResult> { + assert!(self.shallow_root_store.is_none()); + let mut inner = InnerStore::new(self.arena.clone(), config); + let encoded_state_bytes = shallow_bytes.clone(); + let f = inner.decode(shallow_bytes)?; + let encoded_state_bytes = if f.as_ref() == Some(&start_frontiers) { + encoded_state_bytes + } else { + let kv = inner.get_kv_clone(); + kv.insert(FRONTIERS_KEY, start_frontiers.encode().into()); + kv.export() + }; + self.shallow_root_store = Some(Arc::new(GcStore { + shallow_root_frontiers: start_frontiers, + encoded_state_bytes, + store: Mutex::new(inner), + })); + Ok(f) + } + + pub(crate) fn decode_state_by_two_bytes( + &mut self, + shallow_bytes: Bytes, + state_bytes: Bytes, + ) -> LoroResult<()> { + self.store + .decode_twice(shallow_bytes.clone(), state_bytes)?; + Ok(()) + } + + pub fn iter_and_decode_all(&mut self) -> impl Iterator { + self.store.iter_all_containers_mut().map(|(idx, v)| { + v.get_state_mut( + idx, + ContainerCreationContext { + configure: &self.conf, + peer: self.peer.load(std::sync::atomic::Ordering::Relaxed), + }, + ) + }) + } + + pub fn get_kv_clone(&self) -> KvWrapper { + self.store.get_kv_clone() + } + + pub fn contains_id(&mut self, id: &ContainerID) -> bool { + self.store.contains_id(id) + } + + pub fn iter_all_containers( + &mut self, + ) -> impl Iterator { + self.store.iter_all_containers_mut() + } + + pub fn iter_all_container_ids(&mut self) -> impl Iterator + '_ { + self.store.iter_all_container_ids() + } + + pub fn load_all(&mut self) -> LoadAllFlag { + self.store.load_all(); + LoadAllFlag + } + + pub fn load_root_containers(&mut self) -> LoadAllFlag { + self.store.load_roots(); + LoadAllFlag + } + + pub(super) fn get_or_create_mut(&mut self, idx: ContainerIdx) -> &mut State { + self.store + .get_or_insert_with(idx, || { + let state = super::create_state_( + idx, + &self.conf, + self.peer.load(std::sync::atomic::Ordering::Relaxed), + ); + ContainerWrapper::new(state, &self.arena) + }) + .get_state_mut(idx, ctx!(self)) + } + + pub(crate) fn ensure_container(&mut self, id: &loro_common::ContainerID) { + let idx = self.arena.register_container(id); + self.store.ensure_container(idx, || { + let state = super::create_state_( + idx, + &self.conf, + self.peer.load(std::sync::atomic::Ordering::Relaxed), + ); + ContainerWrapper::new(state, &self.arena) + }); + } + + pub(super) fn get_or_create_imm(&mut self, idx: ContainerIdx) -> &State { + self.store + .get_or_insert_with(idx, || { + let state = super::create_state_( + idx, + &self.conf, + self.peer.load(std::sync::atomic::Ordering::Relaxed), + ); + ContainerWrapper::new(state, &self.arena) + }) + .get_state(idx, ctx!(self)) + } + + pub(crate) fn fork( + &mut self, + arena: SharedArena, + peer: Arc, + config: Configure, + ) -> ContainerStore { + ContainerStore { + store: self.store.fork(arena.clone(), &config), + arena, + conf: config, + peer, + shallow_root_store: None, + } + } + + #[allow(unused)] + fn check_eq_after_parsing(&mut self, other: &mut ContainerStore) { + for (idx, container) in self.store.iter_all_containers_mut() { + let id = self.arena.get_container_id(idx).unwrap(); + let other_idx = other.arena.register_container(&id); + let other_container = other + .store + .get_mut(other_idx) + .expect("container not found on other store"); + let other_id = other.arena.get_container_id(other_idx).unwrap(); + assert_eq!( + id, other_id, + "container id mismatch {:?} {:?}", + id, other_id + ); + assert_eq!( + container.get_value(idx, ctx!(self)), + other_container.get_value(other_idx, ctx!(other)), + "value mismatch" + ); + + if container.encode() != other_container.encode() { + panic!( + "container mismatch Origin: {:#?}, New: {:#?}", + &container, &other_container + ); + } + + other_container + .decode_state(other_idx, ctx!(other)) + .unwrap(); + other_container.clear_bytes(); + if container.encode() != other_container.encode() { + panic!( + "container mismatch Origin: {:#?}, New: {:#?}", + &container, &other_container + ); + } + } + } +} + +#[cfg(test)] +mod test { + use super::*; + use crate::{ + cursor::PosType, state::TreeParentId, ContainerType, ListHandler, LoroDoc, MapHandler, + MovableListHandler, + }; + + fn decode_container_store(bytes: Bytes) -> ContainerStore { + let mut new_store = ContainerStore::new( + SharedArena::new(), + Configure::default(), + Arc::new(AtomicU64::new(233)), + ); + + new_store.decode(bytes).unwrap(); + new_store + } + + fn init_doc() -> LoroDoc { + let doc = LoroDoc::new(); + doc.start_auto_commit(); + let text = doc.get_text("text"); + text.insert(0, "hello", PosType::Unicode).unwrap(); + let map = doc.get_map("map"); + map.insert("key", "value").unwrap(); + + let list = doc.get_list("list"); + list.push("item1").unwrap(); + + let tree = doc.get_tree("tree"); + let root = tree.create(TreeParentId::Root).unwrap(); + tree.create_at(TreeParentId::Node(root), 0).unwrap(); + + let movable_list = doc.get_movable_list("movable_list"); + movable_list.insert(0, "movable_item").unwrap(); + + // Create child containers + let child_map = map + .insert_container("child_map", MapHandler::new_detached()) + .unwrap(); + child_map.insert("child_key", "child_value").unwrap(); + + let child_list = list + .insert_container(0, ListHandler::new_detached()) + .unwrap(); + child_list.push("child_item").unwrap(); + let child_movable_list = movable_list + .insert_container(0, MovableListHandler::new_detached()) + .unwrap(); + child_movable_list.insert(0, "child_movable_item").unwrap(); + doc + } + + fn export_container_store(doc: &LoroDoc) -> Bytes { + let mut state = doc.app_state().lock(); + state.ensure_all_alive_containers(); + state.store.encode() + } + + #[test] + fn test_container_store_exports_imports() { + let doc = init_doc(); + let bytes = export_container_store(&doc); + let mut new_store = decode_container_store(bytes); + let mut s = doc.app_state().lock(); + s.store.check_eq_after_parsing(&mut new_store); + } + + #[test] + fn first_lazy_read_caches_value() { + let doc = init_doc(); + let bytes = export_container_store(&doc); + let mut store = decode_container_store(bytes); + let map_id = ContainerID::new_root("map", ContainerType::Map); + let map_idx = store.arena.register_container(&map_id); + + assert!(!store.store.has_cached_value_for_test(map_idx)); + assert_eq!(store.map_len(map_idx), 2); + assert!(store.store.has_cached_value_for_test(map_idx)); + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/state/container_store/container_wrapper.rs b/3rdparty/loro/crates/loro-internal/src/state/container_store/container_wrapper.rs new file mode 100644 index 0000000000..0dc46c55b4 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/state/container_store/container_wrapper.rs @@ -0,0 +1,712 @@ +use std::collections::BTreeMap; + +use bytes::Bytes; +use loro_common::{ContainerID, ContainerType, InternalString, LoroError, LoroResult, LoroValue}; +use once_cell::sync::OnceCell; +use tracing::trace; + +#[cfg(feature = "counter")] +use crate::state::counter_state::CounterState; +use crate::{ + arena::SharedArena, + container::idx::ContainerIdx, + state::{ + unknown_state::UnknownState, ContainerCreationContext, ContainerState, FastStateSnapshot, + IndexType, ListState, MapState, MovableListState, RichtextState, State, TreeState, + }, + utils::utf16::{count_unicode_chars, count_utf16_len}, +}; + +#[derive(Debug)] +pub(crate) struct ContainerWrapper { + depth: usize, + kind: ContainerType, + parent: Option, + data: ContainerData, + flushed: bool, +} + +#[derive(Debug)] +enum ContainerData { + State(State), + Lazy(Box), +} + +#[derive(Debug)] +struct LazyContainerData { + /// Lazily decoded snapshot bytes and optional decoded value. + bytes: Option, + value: Option, + bytes_offset_for_value: Option, + bytes_offset_for_state: Option, +} + +#[derive(Debug)] +enum LazyDecodedValue { + Value(LoroValue), + Text { + value: LoroValue, + unicode_len: usize, + utf16_len: usize, + }, + Map { + ordered: BTreeMap, + value: OnceCell, + }, +} + +impl LazyDecodedValue { + fn text(value: LoroValue) -> Self { + let text = value + .as_string() + .expect("decoded text value should be a string"); + Self::Text { + unicode_len: count_unicode_chars(text.as_bytes()), + utf16_len: count_utf16_len(text.as_bytes()), + value, + } + } + + fn to_loro_value(&self) -> LoroValue { + match self { + Self::Value(value) | Self::Text { value, .. } => value.clone(), + Self::Map { ordered, value } => value + .get_or_init(|| { + LoroValue::Map( + ordered + .iter() + .map(|(key, value)| (key.clone(), value.clone())) + .collect::(), + ) + }) + .clone(), + } + } + + fn as_map(&self) -> Option<&BTreeMap> { + match self { + Self::Map { ordered, .. } => Some(ordered), + _ => None, + } + } + + fn unicode_len(&self) -> Option { + match self { + Self::Text { unicode_len, .. } => Some(*unicode_len), + _ => None, + } + } + + fn utf16_len(&self) -> Option { + match self { + Self::Text { utf16_len, .. } => Some(*utf16_len), + _ => None, + } + } + + fn utf8_len(&self) -> Option { + match self { + // The decoded text value is the string itself, so its byte length + // is a cheap `str::len` — no need to cache a separate field. + Self::Text { value, .. } => value.as_string().map(|s| s.len()), + _ => None, + } + } +} + +impl ContainerWrapper { + pub fn new(state: State, arena: &SharedArena) -> Self { + let idx = state.container_idx(); + let parent = arena + .get_parent(idx) + .and_then(|p| arena.get_container_id(p)); + let depth = arena.get_depth(idx).unwrap().get() as usize; + Self { + depth, + parent, + kind: idx.get_type(), + data: ContainerData::State(state), + flushed: false, + } + } + + pub fn depth(&self) -> usize { + self.depth + } + + pub(crate) fn kind(&self) -> ContainerType { + self.kind + } + + /// It will not decode the state if it is not decoded + #[allow(unused)] + pub fn try_get_state(&self) -> Option<&State> { + match &self.data { + ContainerData::State(state) => Some(state), + ContainerData::Lazy(_) => None, + } + } + + /// It will decode the state if it is not decoded + pub fn get_state(&mut self, idx: ContainerIdx, ctx: ContainerCreationContext) -> &State { + self.decode_state(idx, ctx).unwrap(); + match &self.data { + ContainerData::State(state) => state, + ContainerData::Lazy(_) => unreachable!("ContainerWrapper state should be decoded"), + } + } + + /// It will decode the state if it is not decoded + pub fn get_state_mut( + &mut self, + idx: ContainerIdx, + ctx: ContainerCreationContext, + ) -> &mut State { + self.decode_state(idx, ctx).unwrap(); + self.flushed = false; + match &mut self.data { + ContainerData::State(state) => state, + ContainerData::Lazy(_) => unreachable!("ContainerWrapper state should be decoded"), + } + } + + pub(crate) fn try_get_value( + &mut self, + idx: ContainerIdx, + ctx: ContainerCreationContext, + ) -> LoroResult { + match &mut self.data { + ContainerData::State(state) => { + trace!("state"); + Ok(state.get_value()) + } + ContainerData::Lazy(lazy) if lazy.value.is_some() => { + trace!("value"); + Ok(lazy.value.as_ref().unwrap().to_loro_value()) + } + ContainerData::Lazy(_) => { + trace!("decode value"); + self.decode_value(idx, ctx)?; + match &mut self.data { + ContainerData::State(state) => Ok(state.get_value()), + ContainerData::Lazy(lazy) => Ok(lazy.value.as_ref().unwrap().to_loro_value()), + } + } + } + } + + pub fn get_value(&mut self, idx: ContainerIdx, ctx: ContainerCreationContext) -> LoroValue { + self.try_get_value(idx, ctx).unwrap() + } + + pub fn map_get( + &mut self, + idx: ContainerIdx, + ctx: ContainerCreationContext, + key: &str, + ) -> Option { + match &mut self.data { + ContainerData::State(state) => state.as_map_state().unwrap().get(key).cloned(), + ContainerData::Lazy(_) => { + self.decode_value(idx, ctx).unwrap(); + match &self.data { + ContainerData::Lazy(lazy) => lazy.value.as_ref()?.as_map()?.get(key).cloned(), + ContainerData::State(state) => state.as_map_state().unwrap().get(key).cloned(), + } + } + } + } + + pub fn map_len(&mut self, idx: ContainerIdx, ctx: ContainerCreationContext) -> usize { + match &mut self.data { + ContainerData::State(state) => state.as_map_state().unwrap().len(), + ContainerData::Lazy(_) => { + self.decode_value(idx, ctx).unwrap(); + match &self.data { + ContainerData::Lazy(lazy) => lazy + .value + .as_ref() + .and_then(|value| value.as_map()) + .map_or(0, BTreeMap::len), + ContainerData::State(state) => state.as_map_state().unwrap().len(), + } + } + } + } + + pub fn map_keys( + &mut self, + idx: ContainerIdx, + ctx: ContainerCreationContext, + ) -> Vec { + match &mut self.data { + ContainerData::State(state) => state + .as_map_state() + .unwrap() + .iter() + .filter(|(_, value)| value.value.is_some()) + .map(|(key, _)| key.clone()) + .collect(), + ContainerData::Lazy(_) => { + self.decode_value(idx, ctx).unwrap(); + match &self.data { + ContainerData::Lazy(lazy) => lazy + .value + .as_ref() + .and_then(|value| value.as_map()) + .map(|map| map.keys().map(|key| key.as_str().into()).collect()) + .unwrap_or_default(), + ContainerData::State(state) => state + .as_map_state() + .unwrap() + .iter() + .filter(|(_, value)| value.value.is_some()) + .map(|(key, _)| key.clone()) + .collect(), + } + } + } + } + + pub fn map_entries( + &mut self, + idx: ContainerIdx, + ctx: ContainerCreationContext, + ) -> Vec<(InternalString, LoroValue)> { + match &mut self.data { + ContainerData::State(state) => state + .as_map_state() + .unwrap() + .iter() + .filter_map(|(key, value)| value.value.clone().map(|value| (key.clone(), value))) + .collect(), + ContainerData::Lazy(_) => { + self.decode_value(idx, ctx).unwrap(); + match &self.data { + ContainerData::Lazy(lazy) => lazy + .value + .as_ref() + .and_then(|value| value.as_map()) + .map(|map| { + map.iter() + .map(|(key, value)| (key.as_str().into(), value.clone())) + .collect() + }) + .unwrap_or_default(), + ContainerData::State(state) => state + .as_map_state() + .unwrap() + .iter() + .filter_map(|(key, value)| { + value.value.clone().map(|value| (key.clone(), value)) + }) + .collect(), + } + } + } + } + + pub fn text_unicode_len( + &mut self, + idx: ContainerIdx, + ctx: ContainerCreationContext, + ) -> Option { + match &mut self.data { + ContainerData::State(state) => { + Some(state.as_richtext_state_mut().unwrap().len_unicode()) + } + ContainerData::Lazy(_) => { + self.decode_value(idx, ctx).unwrap(); + match &mut self.data { + ContainerData::State(state) => { + Some(state.as_richtext_state_mut().unwrap().len_unicode()) + } + ContainerData::Lazy(lazy) => lazy.value.as_ref()?.unicode_len(), + } + } + } + } + + pub fn text_utf16_len( + &mut self, + idx: ContainerIdx, + ctx: ContainerCreationContext, + ) -> Option { + match &mut self.data { + ContainerData::State(state) => Some(state.as_richtext_state_mut().unwrap().len_utf16()), + ContainerData::Lazy(_) => { + self.decode_value(idx, ctx).unwrap(); + match &mut self.data { + ContainerData::State(state) => { + Some(state.as_richtext_state_mut().unwrap().len_utf16()) + } + ContainerData::Lazy(lazy) => lazy.value.as_ref()?.utf16_len(), + } + } + } + } + + pub fn text_utf8_len( + &mut self, + idx: ContainerIdx, + ctx: ContainerCreationContext, + ) -> Option { + match &mut self.data { + ContainerData::State(state) => Some(state.as_richtext_state_mut().unwrap().len_utf8()), + ContainerData::Lazy(_) => { + self.decode_value(idx, ctx).unwrap(); + match &mut self.data { + ContainerData::State(state) => { + Some(state.as_richtext_state_mut().unwrap().len_utf8()) + } + ContainerData::Lazy(lazy) => lazy.value.as_ref()?.utf8_len(), + } + } + } + } + + pub fn list_get( + &mut self, + idx: ContainerIdx, + ctx: ContainerCreationContext, + index: usize, + ) -> Option { + match &mut self.data { + ContainerData::State(state) => match self.kind { + ContainerType::List => state.as_list_state().unwrap().get(index).cloned(), + ContainerType::MovableList => state + .as_movable_list_state() + .unwrap() + .get(index, IndexType::ForUser) + .cloned(), + _ => None, + }, + ContainerData::Lazy(_) => match self.get_value(idx, ctx) { + LoroValue::List(list) => list.get(index).cloned(), + _ => None, + }, + } + } + + pub fn list_len(&mut self, idx: ContainerIdx, ctx: ContainerCreationContext) -> usize { + match &mut self.data { + ContainerData::State(state) => match self.kind { + ContainerType::List => state.as_list_state().unwrap().len(), + ContainerType::MovableList => state.as_movable_list_state().unwrap().len(), + _ => 0, + }, + ContainerData::Lazy(_) => match self.get_value(idx, ctx) { + LoroValue::List(list) => list.len(), + _ => 0, + }, + } + } + + pub fn list_values( + &mut self, + idx: ContainerIdx, + ctx: ContainerCreationContext, + ) -> Vec { + match &mut self.data { + ContainerData::State(state) => match self.kind { + ContainerType::List => state.as_list_state().unwrap().iter().cloned().collect(), + ContainerType::MovableList => state + .as_movable_list_state() + .unwrap() + .iter() + .cloned() + .collect(), + _ => Vec::new(), + }, + ContainerData::Lazy(_) => match self.get_value(idx, ctx) { + LoroValue::List(list) => list.iter().cloned().collect(), + _ => Vec::new(), + }, + } + } + + pub fn encode(&mut self) -> Bytes { + let ContainerData::State(state) = &mut self.data else { + let lazy = match &self.data { + ContainerData::Lazy(lazy) => lazy, + ContainerData::State(_) => unreachable!(), + }; + assert!(self.flushed, "lazy container should be flushed"); + return lazy.bytes.as_ref().unwrap().clone(); + }; + + // ContainerType + // Depth + // ParentID + // StateSnapshot + let mut output = Vec::new(); + output.push(self.kind.to_u8()); + leb128::write::unsigned(&mut output, self.depth as u64).unwrap(); + postcard::to_io(&self.parent, &mut output).unwrap(); + state.encode_snapshot_fast(&mut output); + output.into() + } + + #[allow(unused)] + pub fn decode_parent(b: &[u8]) -> Option { + let mut bytes = &b[1..]; + let _depth = leb128::read::unsigned(&mut bytes).unwrap(); + let (parent, _bytes) = postcard::take_from_bytes::>(bytes).unwrap(); + parent + } + + pub fn new_from_bytes(bytes: Bytes) -> Self { + Self::try_new_from_bytes(bytes).unwrap() + } + + pub fn try_new_from_bytes(bytes: Bytes) -> LoroResult { + if bytes.is_empty() { + return Err(LoroError::DecodeError( + "Decode container state failed".to_string().into_boxed_str(), + )); + } + + let kind = ContainerType::try_from_u8(bytes[0])?; + let mut reader = &bytes[1..]; + let depth = leb128::read::unsigned(&mut reader).map_err(|_| { + LoroError::DecodeError("Decode container state failed".to_string().into_boxed_str()) + })?; + let (parent, reader) = postcard::take_from_bytes(reader).map_err(|_| { + LoroError::DecodeError("Decode container state failed".to_string().into_boxed_str()) + })?; + let size = bytes.len() - reader.len(); + Ok(Self { + depth: depth as usize, + kind, + parent, + data: ContainerData::Lazy(Box::new(LazyContainerData { + value: None, + bytes: Some(bytes.clone()), + bytes_offset_for_value: Some(size), + bytes_offset_for_state: None, + })), + flushed: true, + }) + } + + fn decode_value(&mut self, idx: ContainerIdx, ctx: ContainerCreationContext) -> LoroResult<()> { + if matches!(self.data, ContainerData::State(_)) { + return Ok(()); + } + + if matches!(&self.data, ContainerData::Lazy(lazy) if lazy.value.is_some()) { + return Ok(()); + } + + let (v, state_offset, decoded_state) = self.decode_value_from_bytes(idx, ctx)?; + if let Some(state) = decoded_state { + self.data = ContainerData::State(state); + return Ok(()); + } + + let ContainerData::Lazy(lazy) = &mut self.data else { + unreachable!(); + }; + lazy.value = Some(v); + lazy.bytes_offset_for_state = Some(state_offset); + Ok(()) + } + + pub(super) fn has_cached_value(&self) -> bool { + match &self.data { + ContainerData::State(_) => true, + ContainerData::Lazy(lazy) => lazy.value.is_some(), + } + } + + #[cfg(test)] + pub(super) fn has_cached_value_for_test(&self) -> bool { + self.has_cached_value() + } + + fn value_bytes_and_offset(&mut self) -> Option<(Bytes, usize)> { + let ContainerData::Lazy(lazy) = &mut self.data else { + return None; + }; + + let bytes = lazy.bytes.as_ref()?.clone(); + if lazy.bytes_offset_for_value.is_none() { + let mut reader: &[u8] = &bytes; + reader = &reader[1..]; + let _depth = leb128::read::unsigned(&mut reader).unwrap(); + let (_parent, reader) = + postcard::take_from_bytes::>(reader).unwrap(); + // SAFETY: bytes is a slice of b + let size = bytes.len() - reader.len(); + lazy.bytes_offset_for_value = Some(size); + } + + Some((bytes, lazy.bytes_offset_for_value.unwrap())) + } + + fn decode_value_from_bytes( + &mut self, + idx: ContainerIdx, + ctx: ContainerCreationContext, + ) -> LoroResult<(LazyDecodedValue, usize, Option)> { + let Some((bytes, value_offset)) = self.value_bytes_and_offset() else { + return Ok((LazyDecodedValue::Value(self.kind.default_value()), 0, None)); + }; + let b = &bytes[value_offset..]; + + let mut decoded_state = None; + let (v, state_offset) = match self.kind { + ContainerType::Text => { + let (v, rest) = RichtextState::decode_value(b)?; + ( + LazyDecodedValue::text(v), + b.len() - rest.len() + value_offset, + ) + } + ContainerType::Map => { + let (v, rest) = MapState::decode_value_as_btree_map(b)?; + ( + LazyDecodedValue::Map { + ordered: v, + value: OnceCell::new(), + }, + b.len() - rest.len() + value_offset, + ) + } + ContainerType::List => { + let (v, rest) = ListState::decode_value(b)?; + ( + LazyDecodedValue::Value(v), + b.len() - rest.len() + value_offset, + ) + } + ContainerType::MovableList => { + let (v, rest) = MovableListState::decode_value(b)?; + ( + LazyDecodedValue::Value(v), + b.len() - rest.len() + value_offset, + ) + } + ContainerType::Tree => { + let mut state = TreeState::decode_snapshot_fast(idx, (LoroValue::Null, b), ctx)?; + let value = state.get_value(); + decoded_state = Some(State::TreeState(Box::new(state))); + (LazyDecodedValue::Value(value), value_offset) + } + #[cfg(feature = "counter")] + ContainerType::Counter => { + let (v, rest) = CounterState::decode_value(b)?; + ( + LazyDecodedValue::Value(v), + b.len() - rest.len() + value_offset, + ) + } + ContainerType::Unknown(_) => { + let (v, rest) = UnknownState::decode_value(b)?; + ( + LazyDecodedValue::Value(v), + b.len() - rest.len() + value_offset, + ) + } + }; + + Ok((v, state_offset, decoded_state)) + } + + pub(super) fn decode_state( + &mut self, + idx: ContainerIdx, + ctx: ContainerCreationContext, + ) -> LoroResult<()> { + if matches!(self.data, ContainerData::State(_)) { + return Ok(()); + } + + let need_value = match &self.data { + ContainerData::Lazy(lazy) => lazy.value.is_none(), + ContainerData::State(_) => false, + }; + if need_value { + self.decode_value(idx, ctx)?; + } + + if matches!(self.data, ContainerData::State(_)) { + return Ok(()); + } + + let ContainerData::Lazy(lazy) = &self.data else { + unreachable!(); + }; + let bytes = lazy.bytes.as_ref().unwrap(); + let offset = lazy.bytes_offset_for_state.unwrap(); + let b = &bytes[offset..]; + let v = lazy.value.as_ref().unwrap().to_loro_value(); + let state: State = match self.kind { + ContainerType::Text => RichtextState::decode_snapshot_fast(idx, (v, b), ctx)?.into(), + ContainerType::Map => MapState::decode_snapshot_fast(idx, (v, b), ctx)?.into(), + ContainerType::List => ListState::decode_snapshot_fast(idx, (v, b), ctx)?.into(), + ContainerType::MovableList => { + MovableListState::decode_snapshot_fast(idx, (v, b), ctx)?.into() + } + ContainerType::Tree => TreeState::decode_snapshot_fast(idx, (v, b), ctx)?.into(), + #[cfg(feature = "counter")] + ContainerType::Counter => CounterState::decode_snapshot_fast(idx, (v, b), ctx)?.into(), + ContainerType::Unknown(_) => { + UnknownState::decode_snapshot_fast(idx, (v, b), ctx)?.into() + } + }; + self.data = ContainerData::State(state); + Ok(()) + } + + #[allow(unused)] + pub(crate) fn is_state_empty(&self) -> bool { + match &self.data { + ContainerData::State(state) => state.is_state_empty(), + ContainerData::Lazy(lazy) => { + // FIXME: it's not very accurate... + lazy.bytes.as_ref().unwrap().len() > 10 + } + } + } + + pub(crate) fn is_deleted_root_value_cleared(&mut self) -> bool { + fn value_is_cleared(kind: ContainerType, value: &LoroValue) -> bool { + match kind { + ContainerType::Text => value.as_string().is_some_and(|value| value.is_empty()), + ContainerType::Map | ContainerType::List | ContainerType::MovableList => { + value.is_empty_collection() + } + ContainerType::Tree => value.as_list().is_some_and(|value| value.is_empty()), + #[cfg(feature = "counter")] + ContainerType::Counter => value.as_double().is_some_and(|value| *value == 0.0), + ContainerType::Unknown(_) => false, + } + } + + match &mut self.data { + ContainerData::State(state) => value_is_cleared(self.kind, &state.get_value()), + ContainerData::Lazy(lazy) => lazy + .value + .as_ref() + .is_some_and(|value| value_is_cleared(self.kind, &value.to_loro_value())), + } + } + + pub(crate) fn clear_bytes(&mut self) { + assert!(matches!(self.data, ContainerData::State(_))); + } + + pub(crate) fn is_flushed(&self) -> bool { + self.flushed + } + + pub(crate) fn set_flushed(&mut self, flushed: bool) { + self.flushed = flushed; + } + + #[allow(unused)] + pub(crate) fn parent(&self) -> Option<&ContainerID> { + self.parent.as_ref() + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/state/container_store/inner_store.rs b/3rdparty/loro/crates/loro-internal/src/state/container_store/inner_store.rs new file mode 100644 index 0000000000..a030be9668 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/state/container_store/inner_store.rs @@ -0,0 +1,522 @@ +use crate::{ + arena::SharedArena, configure::Configure, container::idx::ContainerIdx, + state::container_store::FRONTIERS_KEY, utils::kv_wrapper::KvWrapper, version::Frontiers, +}; +use bytes::Bytes; +use loro_common::ContainerID; + +use super::ContainerWrapper; + +/// The invariants about this struct: +/// +/// - `kv` is either the same or older than `store`. +/// - if `load_state` is `AllLoaded`, then `store` contains all the entries from `kv` +/// +/// Invariants: it should be agnostic to the users of this struct whether a container is stored in `kv` or `store` +pub(crate) struct InnerStore { + arena: SharedArena, + store: Vec>, + kv: KvWrapper, + load_state: LoadState, + config: Configure, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum LoadState { + Lazy, + RootsLoaded, + AllLoaded, +} + +impl std::fmt::Debug for InnerStore { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("InnerStore").finish() + } +} + +/// This impl block contains all the mutation code that may break the invariants of this struct +impl InnerStore { + #[inline] + fn slot(idx: ContainerIdx) -> usize { + idx.to_index() as usize + } + + #[inline] + fn get_entry_mut_in( + store: &mut [Option], + idx: ContainerIdx, + ) -> Option<&mut ContainerWrapper> { + let entry = store.get_mut(Self::slot(idx))?.as_mut()?; + debug_assert_eq!(entry.kind(), idx.get_type()); + Some(entry) + } + + #[inline] + fn get_entry_mut(&mut self, idx: ContainerIdx) -> Option<&mut ContainerWrapper> { + Self::get_entry_mut_in(&mut self.store, idx) + } + + #[inline] + fn contains_idx_in(store: &[Option], idx: ContainerIdx) -> bool { + store + .get(Self::slot(idx)) + .and_then(|entry| entry.as_ref()) + .is_some_and(|entry| entry.kind() == idx.get_type()) + } + + #[inline] + fn contains_idx(&self, idx: ContainerIdx) -> bool { + Self::contains_idx_in(&self.store, idx) + } + + fn insert_entry( + store: &mut Vec>, + idx: ContainerIdx, + container: ContainerWrapper, + ) -> Option { + let slot = Self::slot(idx); + if store.len() <= slot { + store.resize_with(slot + 1, || None); + } + + store[slot].replace(container) + } + + pub(super) fn get_or_insert_with( + &mut self, + idx: ContainerIdx, + f: impl FnOnce() -> ContainerWrapper, + ) -> &mut ContainerWrapper { + if self.get_entry_mut(idx).is_none() { + let id = self.arena.get_container_id(idx).unwrap(); + let key = id.to_bytes(); + let container = if self.load_state != LoadState::AllLoaded { + self.kv + .get(&key) + .map(ContainerWrapper::new_from_bytes) + .unwrap_or_else(f) + } else { + f() + }; + Self::insert_entry(&mut self.store, idx, container); + } + + self.get_entry_mut(idx).unwrap() + } + + pub(super) fn ensure_container( + &mut self, + idx: ContainerIdx, + f: impl FnOnce() -> ContainerWrapper, + ) { + if self.contains_idx(idx) { + return; + } + + if self.load_state != LoadState::AllLoaded { + let id = self.arena.get_container_id(idx).unwrap(); + let key = id.to_bytes(); + if let Some(v) = self.kv.get(&key) { + let c = ContainerWrapper::new_from_bytes(v); + Self::insert_entry(&mut self.store, idx, c); + return; + } + } + + let c = f(); + Self::insert_entry(&mut self.store, idx, c); + } + + pub(crate) fn get_mut(&mut self, idx: ContainerIdx) -> Option<&mut ContainerWrapper> { + if self.get_entry_mut(idx).is_none() && self.load_state != LoadState::AllLoaded { + let id = self.arena.get_container_id(idx).unwrap(); + let key = id.to_bytes(); + if let Some(v) = self.kv.get(&key) { + let c = ContainerWrapper::new_from_bytes(v); + Self::insert_entry(&mut self.store, idx, c); + } + } + + self.get_entry_mut(idx) + } + + pub(crate) fn with_container_for_read( + &mut self, + idx: ContainerIdx, + f: impl FnOnce(&mut ContainerWrapper) -> R, + ) -> Option { + if let Some(entry) = self.get_entry_mut(idx) { + return Some(f(entry)); + } + + if self.load_state != LoadState::AllLoaded { + let id = self.arena.get_container_id(idx).unwrap(); + let key = id.to_bytes(); + if let Some(v) = self.kv.get(&key) { + let mut container = ContainerWrapper::new_from_bytes(v); + let ans = f(&mut container); + if container.has_cached_value() { + Self::insert_entry(&mut self.store, idx, container); + } + return Some(ans); + } + } + + None + } + + pub(crate) fn has_decoded_state(&mut self, idx: ContainerIdx) -> bool { + self.get_entry_mut(idx) + .is_some_and(|entry| entry.try_get_state().is_some()) + } + + pub(crate) fn contains_id(&mut self, id: &ContainerID) -> bool { + if let Some(idx) = self.arena.id_to_idx(id) { + if self.contains_idx(idx) { + return true; + } + } + + if self.load_state != LoadState::AllLoaded { + let key = id.to_bytes(); + return self.kv.contains_key(&key); + } + + false + } + + pub(crate) fn iter_all_containers_mut( + &mut self, + ) -> impl Iterator { + self.load_all(); + self.store + .iter_mut() + .enumerate() + .filter_map(|(slot, entry)| { + entry.as_mut().map(|container| { + ( + ContainerIdx::from_index_and_type(slot as u32, container.kind()), + container, + ) + }) + }) + } + + pub(crate) fn iter_all_container_ids(&mut self) -> impl Iterator + '_ { + // PERF: we don't need to load all the containers here + self.load_all(); + self.store.iter().enumerate().filter_map(|(slot, entry)| { + entry.as_ref().map(|container| { + let idx = ContainerIdx::from_index_and_type(slot as u32, container.kind()); + self.arena.get_container_id(idx).unwrap() + }) + }) + } + + pub(crate) fn encode(&mut self) -> Bytes { + self.flush(); + self.kv.export() + } + + pub(crate) fn flush(&mut self) { + let deleted = self.config.deleted_root_containers.lock(); + let mut updates = Vec::new(); + let mut deleted_roots = Vec::new(); + + for (slot, entry) in self.store.iter_mut().enumerate() { + let Some(c) = entry.as_mut() else { + continue; + }; + let idx = ContainerIdx::from_index_and_type(slot as u32, c.kind()); + let cid = self.arena.get_container_id(idx).unwrap(); + if cid.is_root() && deleted.contains(&cid) && c.is_deleted_root_value_cleared() { + deleted_roots.push(cid.to_bytes()); + c.set_flushed(true); + continue; + } + + if c.is_flushed() { + continue; + } + + let cid: Bytes = cid.to_bytes().into(); + let value = c.encode(); + c.set_flushed(true); + updates.push((cid, value)); + } + + drop(deleted); + for cid in deleted_roots { + self.kv.remove(&cid); + } + self.kv.set_all(updates); + } + + pub(crate) fn get_kv_clone(&self) -> KvWrapper { + self.kv.clone() + } + + pub(crate) fn decode( + &mut self, + bytes: bytes::Bytes, + ) -> Result, loro_common::LoroError> { + assert!(self.kv.is_empty()); + let mut fr = None; + self.kv + .import(bytes) + .map_err(|e| loro_common::LoroError::DecodeError(e.into_boxed_str()))?; + if let Some(f) = self.kv.remove(FRONTIERS_KEY) { + fr = Some(Frontiers::decode(&f)?); + } + + let kv = self.kv.arc_clone(); + self.arena + .set_parent_resolver(Some(move |child_id: ContainerID| { + let k = child_id.to_bytes(); + let v = kv.get(&k)?; + let c = ContainerWrapper::new_from_bytes(v); + c.parent().cloned() + })); + + self.store.clear(); + self.load_state = LoadState::Lazy; + Ok(fr) + } + + pub(crate) fn decode_twice( + &mut self, + bytes_a: bytes::Bytes, + bytes_b: bytes::Bytes, + ) -> Result<(), loro_common::LoroError> { + assert!(self.kv.is_empty()); + // TODO: add assert that all containers in the store should be empty right now + self.kv + .import(bytes_a) + .map_err(|e| loro_common::LoroError::DecodeError(e.into_boxed_str()))?; + self.kv + .import(bytes_b) + .map_err(|e| loro_common::LoroError::DecodeError(e.into_boxed_str()))?; + self.kv.remove(FRONTIERS_KEY); + let entries = self.kv.scan_all_entries(); + let store = &mut self.store; + let arena = &self.arena; + arena.with_guards(|guards| { + for (k, v) in entries { + let cid = ContainerID::from_bytes(&k); + let c = ContainerWrapper::new_from_bytes(v); + let parent = c.parent(); + let idx = guards.register_container(&cid); + let p = parent.as_ref().map(|p| guards.register_container(p)); + guards.set_parent(idx, p); + if Self::insert_entry(store, idx, c).is_some() {} + } + }); + + self.load_state = LoadState::AllLoaded; + Ok(()) + } + + pub fn load_all(&mut self) { + if self.load_state == LoadState::AllLoaded { + return; + } + + let entries = self.kv.scan_all_entries(); + let store = &mut self.store; + let arena = &self.arena; + arena.with_guards(|guards| { + for (k, v) in entries { + let cid = ContainerID::from_bytes(&k); + let idx = guards.register_container(&cid); + if Self::contains_idx_in(store, idx) { + // the container is already loaded + // the content in `store` is guaranteed to be newer than the content in `kv` + continue; + } + + let container = ContainerWrapper::new_from_bytes(v); + Self::insert_entry(store, idx, container); + } + }); + + self.load_state = LoadState::AllLoaded; + } + + pub fn load_roots(&mut self) { + if self.load_state != LoadState::Lazy { + return; + } + + let arena = &self.arena; + let keys = self.kv.scan_all_keys(); + arena.with_guards(|guards| { + for k in keys { + let cid = ContainerID::from_bytes(&k); + if cid.is_root() { + guards.register_container(&cid); + } + } + }); + self.load_state = LoadState::RootsLoaded; + } + + pub(crate) fn can_import_snapshot(&self) -> bool { + if !self.kv.is_empty() { + return false; + } + + self.store + .iter() + .filter_map(|entry| entry.as_ref()) + .all(|c| c.is_state_empty()) + } + + #[cfg(test)] + pub(super) fn has_cached_value_for_test(&mut self, idx: ContainerIdx) -> bool { + self.get_entry_mut(idx) + .is_some_and(|entry| entry.has_cached_value_for_test()) + } +} + +impl InnerStore { + pub(crate) fn new(arena: SharedArena, config: Configure) -> Self { + Self { + arena, + store: Vec::new(), + kv: KvWrapper::new_mem(), + load_state: LoadState::AllLoaded, + config, + } + } + + pub(crate) fn fork(&mut self, arena: SharedArena, config: &Configure) -> InnerStore { + // PERF: we can try to reuse + let bytes = self.encode(); + let mut new_store = Self::new(arena, config.clone()); + new_store.decode(bytes).unwrap(); + new_store + } +} + +#[cfg(test)] +mod tests { + use super::*; + use loro_common::{ContainerType, ID}; + use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }; + + fn encoded_container_header(kind: ContainerType, parent: Option) -> Bytes { + let mut output = Vec::new(); + output.push(kind.to_u8()); + leb128::write::unsigned(&mut output, 2).unwrap(); + postcard::to_io(&parent, &mut output).unwrap(); + output.into() + } + + fn mergeable_child_entry() -> (ContainerID, Bytes) { + let parent_id = ContainerID::new_normal(ID::new(1, 0), ContainerType::Map); + let child_id = ContainerID::new_mergeable(&parent_id, "field", ContainerType::Text); + let value = encoded_container_header(ContainerType::Text, Some(parent_id)); + (child_id, value) + } + + fn install_parent_resolver_guard( + arena: &SharedArena, + loading: Arc, + ) -> Arc { + let resolver_called = Arc::new(AtomicBool::new(false)); + let called = resolver_called.clone(); + arena.set_parent_resolver(Some(move |_child_id: ContainerID| { + assert!( + !loading.load(Ordering::SeqCst), + "lazy parent resolver must not run while loading from KV" + ); + called.store(true, Ordering::SeqCst); + None + })); + resolver_called + } + + fn kv_bytes_with_entry(cid: ContainerID, value: Bytes) -> Bytes { + let kv = KvWrapper::new_mem(); + kv.set_all(vec![(Bytes::from(cid.to_bytes()), value)]); + kv.export() + } + + #[test] + fn load_all_does_not_resolve_parent_while_loading_from_kv() { + let arena = SharedArena::new(); + let mut store = InnerStore::new(arena.clone(), Configure::default()); + let loading = Arc::new(AtomicBool::new(false)); + let resolver_called = install_parent_resolver_guard(&arena, loading.clone()); + let (child_id, value) = mergeable_child_entry(); + let child_id_for_depth = child_id.clone(); + store.load_state = LoadState::Lazy; + store + .kv + .set_all(vec![(Bytes::from(child_id.to_bytes()), value)]); + + loading.store(true, Ordering::SeqCst); + store.load_all(); + loading.store(false, Ordering::SeqCst); + assert!(!resolver_called.load(Ordering::SeqCst)); + + let child_idx = arena.id_to_idx(&child_id_for_depth).unwrap(); + let _ = arena.get_depth(child_idx); + assert!( + resolver_called.load(Ordering::SeqCst), + "lazy parent resolver should still work after load_all returns" + ); + } + + #[test] + fn load_roots_does_not_resolve_parent_while_loading_from_kv() { + let arena = SharedArena::new(); + let mut store = InnerStore::new(arena.clone(), Configure::default()); + let loading = Arc::new(AtomicBool::new(false)); + let resolver_called = install_parent_resolver_guard(&arena, loading.clone()); + let (child_id, value) = mergeable_child_entry(); + let child_id_for_depth = child_id.clone(); + store.load_state = LoadState::Lazy; + store + .kv + .set_all(vec![(Bytes::from(child_id.to_bytes()), value)]); + + loading.store(true, Ordering::SeqCst); + store.load_roots(); + loading.store(false, Ordering::SeqCst); + assert!(!resolver_called.load(Ordering::SeqCst)); + + let child_idx = arena.id_to_idx(&child_id_for_depth).unwrap(); + let _ = arena.get_depth(child_idx); + assert!( + resolver_called.load(Ordering::SeqCst), + "lazy parent resolver should still work after load_roots returns" + ); + } + + #[test] + fn decode_twice_does_not_resolve_parent_while_loading_from_kv() { + let arena = SharedArena::new(); + let mut store = InnerStore::new(arena.clone(), Configure::default()); + let loading = Arc::new(AtomicBool::new(false)); + let resolver_called = install_parent_resolver_guard(&arena, loading.clone()); + let (child_id, value) = mergeable_child_entry(); + let child_id_for_depth = child_id.clone(); + let bytes = kv_bytes_with_entry(child_id, value); + + loading.store(true, Ordering::SeqCst); + store.decode_twice(bytes, Bytes::new()).unwrap(); + loading.store(false, Ordering::SeqCst); + assert!(!resolver_called.load(Ordering::SeqCst)); + + let child_idx = arena.id_to_idx(&child_id_for_depth).unwrap(); + let _ = arena.get_depth(child_idx); + assert!( + resolver_called.load(Ordering::SeqCst), + "lazy parent resolver should still work after decode_twice returns" + ); + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/state/counter_state.rs b/3rdparty/loro/crates/loro-internal/src/state/counter_state.rs new file mode 100644 index 0000000000..332b2f52d8 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/state/counter_state.rs @@ -0,0 +1,134 @@ +use std::sync::Weak; + +use loro_common::{ContainerID, LoroResult, LoroValue}; + +use crate::{ + configure::Configure, + container::idx::ContainerIdx, + event::{Diff, Index, InternalDiff}, + op::{Op, RawOp, RawOpContent}, + LoroDocInner, +}; + +use super::{ApplyLocalOpReturn, ContainerState, DiffApplyContext}; + +#[derive(Debug, Clone)] +pub struct CounterState { + idx: ContainerIdx, + value: f64, +} + +impl CounterState { + pub(crate) fn new(idx: ContainerIdx) -> Self { + Self { idx, value: 0. } + } +} + +impl ContainerState for CounterState { + fn container_idx(&self) -> ContainerIdx { + self.idx + } + + fn is_state_empty(&self) -> bool { + false + } + + fn apply_diff_and_convert(&mut self, diff: InternalDiff, _ctx: DiffApplyContext) -> Diff { + if let InternalDiff::Counter(diff) = diff { + self.value += diff; + Diff::Counter(diff) + } else { + unreachable!() + } + } + + fn apply_diff(&mut self, diff: InternalDiff, ctx: DiffApplyContext) -> LoroResult<()> { + let _ = self.apply_diff_and_convert(diff, ctx); + Ok(()) + } + + fn apply_local_op(&mut self, raw_op: &RawOp, _op: &Op) -> LoroResult { + if let RawOpContent::Counter(diff) = raw_op.content { + self.value += diff; + Ok(Default::default()) + } else { + unreachable!() + } + } + + #[doc = " Convert a state to a diff, such that an empty state will be transformed into the same as this state when it\'s applied."] + fn to_diff(&mut self, _doc: &Weak) -> Diff { + Diff::Counter(self.value) + } + + fn get_value(&mut self) -> LoroValue { + LoroValue::Double(self.value) + } + + #[doc = " Get the index of the child container"] + #[allow(unused)] + fn get_child_index(&self, id: &ContainerID) -> Option { + None + } + + #[allow(unused)] + fn get_child_containers(&self) -> Vec { + vec![] + } + + fn contains_child(&self, _id: &ContainerID) -> bool { + false + } + + fn fork(&self, _config: &Configure) -> Self { + self.clone() + } +} + +mod snapshot { + use crate::state::FastStateSnapshot; + + use super::*; + + impl FastStateSnapshot for CounterState { + fn encode_snapshot_fast(&mut self, mut w: W) { + let bytes = self.value.to_le_bytes(); + w.write_all(&bytes).unwrap(); + } + + fn decode_value(bytes: &[u8]) -> LoroResult<(LoroValue, &[u8])> { + // Builds without the counter feature can re-export an untouched counter as header only. + if bytes.is_empty() { + return Ok((LoroValue::Double(0.0), bytes)); + } + + if bytes.len() != 8 { + return Err(loro_common::LoroError::DecodeError( + "Decode counter value failed".to_string().into_boxed_str(), + )); + } + + let value = f64::from_le_bytes(bytes[..8].try_into().unwrap()); + Ok((LoroValue::Double(value), &bytes[8..])) + } + + fn decode_snapshot_fast( + idx: ContainerIdx, + v: (LoroValue, &[u8]), + _ctx: crate::state::ContainerCreationContext, + ) -> LoroResult + where + Self: Sized, + { + if !v.1.is_empty() { + return Err(loro_common::LoroError::DecodeError( + "Decode counter state failed".to_string().into_boxed_str(), + )); + } + + let mut counter = CounterState::new(idx); + counter.value = *v.0.as_double().unwrap(); + Ok(counter) + } + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/state/dead_containers_cache.rs b/3rdparty/loro/crates/loro-internal/src/state/dead_containers_cache.rs new file mode 100644 index 0000000000..3a15e3fa20 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/state/dead_containers_cache.rs @@ -0,0 +1,246 @@ +use super::DocState; +use crate::container::idx::ContainerIdx; +use rustc_hash::FxHashMap; +use smallvec::SmallVec; + +#[derive(Default, Debug, Clone)] +pub(super) struct DeadContainersCache { + cache: FxHashMap, +} + +impl DeadContainersCache { + pub fn clear(&mut self) { + self.cache.clear(); + } + + pub(crate) fn clear_alive(&mut self) { + self.cache.retain(|_, is_deleted| *is_deleted); + } +} + +impl DocState { + pub(crate) fn is_deleted(&mut self, idx: ContainerIdx) -> bool { + #[cfg(not(debug_assertions))] + { + // Cache stores only deleted containers. + if self.dead_containers_cache.cache.contains_key(&idx) { + return true; + } + } + + // Parent chains are shallow (depth 1 for a root container), so inline + // storage avoids a heap allocation on this per-op check. + let mut visited: SmallVec<[ContainerIdx; 4]> = SmallVec::new(); + visited.push(idx); + let mut idx = idx; + let mut depends_on_mergeable_edge = false; + let is_deleted = loop { + let id = self.arena.idx_to_id(idx).unwrap(); + if id.is_mergeable() { + depends_on_mergeable_edge = true; + } + if let Some(parent_idx) = self.arena.get_parent(idx) { + if !self.contains_logical_child(parent_idx, &id) { + break true; + } + + idx = parent_idx; + visited.push(idx); + } else { + // No parent in the arena: top-level Roots are always alive; anything else + // (including a mergeable Root whose parent edge was never wired) is treated + // as deleted. + break !id.is_root() || id.is_mergeable(); + } + }; + + #[cfg(debug_assertions)] + { + if !depends_on_mergeable_edge { + if let Some(cached_is_deleted) = self.dead_containers_cache.cache.get(&idx) { + assert_eq!(is_deleted, *cached_is_deleted); + } + } + } + + // A mergeable ancestor can be deleted and later reactivated by changing the parent map's + // marker. Do not cache deletion for any descendant whose liveness depends on that + // logical edge, including ordinary children nested inside a mergeable map. + if depends_on_mergeable_edge { + if !is_deleted { + for idx in visited { + self.dead_containers_cache.cache.remove(&idx); + } + } + return is_deleted; + } + + if is_deleted { + for idx in visited { + self.dead_containers_cache.cache.insert(idx, true); + } + } else { + for idx in visited { + self.dead_containers_cache.cache.remove(&idx); + } + } + + is_deleted + } + + #[cfg(test)] + pub(crate) fn dead_cache_entry(&self, idx: ContainerIdx) -> Option { + self.dead_containers_cache.cache.get(&idx).copied() + } +} + +#[cfg(test)] +#[cfg(feature = "counter")] +mod tests { + use loro_common::ContainerID; + + use crate::{cursor::PosType, HandlerTrait, LoroDoc, TextHandler}; + + /// A mergeable child can be deleted and then reactivated: `delete(key)` clears its + /// marker (child unreachable), and a later `ensure_mergeable_(key)` writes the + /// marker back (child reachable again). While the child is unreachable, querying its + /// liveness must not cache a `deleted` entry because that answer depends on the mutable + /// mergeable marker edge. + /// + /// The scenario: + /// 1. Create the mergeable counter and capture its container idx. + /// 2. Delete the key, then query `is_deleted()`. + /// 3. Re-get the counter to rewrite the marker and reactivate the child. + /// 4. Assert the cache never held a `deleted` entry for that idx. + /// + /// It asserts the cache contents directly because `is_deleted()` only trusts the cache via a + /// release-only early return; inspecting the cache makes stale-cache regressions fail in both + /// debug and release builds. + #[test] + fn reactivated_mergeable_child_has_no_stale_dead_cache_entry() { + let doc = LoroDoc::new_auto_commit(); + doc.set_peer_id(1).unwrap(); + let root = doc.get_map("state"); + let counter = root.ensure_mergeable_counter("revision").unwrap(); + counter.increment(1.0).unwrap(); + doc.commit_then_renew(); + + let cid: ContainerID = counter.id(); + let idx = doc.state.lock().arena.id_to_idx(&cid).unwrap(); + + root.delete("revision").unwrap(); + doc.commit_then_renew(); + assert!(counter.is_deleted()); + assert_eq!( + doc.state.lock().dead_cache_entry(idx), + None, + "mergeable-dependent deletion must not be cached" + ); + + root.ensure_mergeable_counter("revision").unwrap(); + doc.commit_then_renew(); + assert_eq!( + doc.state.lock().dead_cache_entry(idx), + None, + "reactivation must drop the stale deleted-cache entry" + ); + } + + /// The no-cache rule also applies to ordinary descendants under a mergeable ancestor. A + /// regular child container can look cache-safe by cid shape, but its liveness still depends on + /// the ancestor's marker edge. + #[test] + fn ordinary_child_under_reactivated_mergeable_map_has_no_stale_dead_cache_entry() { + let doc = LoroDoc::new_auto_commit(); + doc.set_peer_id(1).unwrap(); + let root = doc.get_map("state"); + let profile = root.ensure_mergeable_map("profile").unwrap(); + let text = profile + .insert_container("bio", TextHandler::new_detached()) + .unwrap(); + text.insert(0, "Ada", PosType::Unicode).unwrap(); + doc.commit_then_renew(); + + let text_id: ContainerID = text.id(); + let text_idx = doc.state.lock().arena.id_to_idx(&text_id).unwrap(); + + root.delete("profile").unwrap(); + doc.commit_then_renew(); + assert!(text.is_deleted()); + assert_eq!( + doc.state.lock().dead_cache_entry(text_idx), + None, + "ordinary descendants behind a mergeable edge must not be cached as deleted" + ); + + root.ensure_mergeable_map("profile").unwrap(); + doc.commit_then_renew(); + assert!(!text.is_deleted()); + assert_eq!( + doc.state.lock().dead_cache_entry(text_idx), + None, + "reactivated ordinary descendant must not leave a stale cache entry" + ); + } + + /// The same stale-cache hazard exists when reactivation arrives from a *peer* via import, + /// not just from a local `ensure_mergeable_*` call. The importing peer must not cache the + /// mergeable-dependent deleted result before the reactivation update arrives. + /// + /// The scenario, with two peers A (author) and B (importer): + /// 1. A creates the mergeable counter and deletes the key, then exports. + /// 2. B imports A's updates so the child exists but is unreachable, and queries + /// `is_deleted()`. + /// 3. A re-gets the counter (rewriting the marker, reactivating the child) and + /// exports just that new update. + /// 4. B imports the reactivation update. + /// 5. Assert B's cache never held a `deleted` entry for that idx. + /// + /// Like the local-reactivation test, this asserts cache contents directly rather than + /// through `is_deleted()`, because `is_deleted()` only trusts the cache via a release-only + /// early return; a public-API assertion would pass in debug even with the bug present. + #[test] + fn imported_mergeable_child_reactivation_clears_dead_cache() { + use crate::loro::ExportMode; + + let doc_a = LoroDoc::new_auto_commit(); + doc_a.set_peer_id(1).unwrap(); + let root_a = doc_a.get_map("state"); + let counter_a = root_a.ensure_mergeable_counter("revision").unwrap(); + counter_a.increment(1.0).unwrap(); + doc_a.commit_then_renew(); + root_a.delete("revision").unwrap(); + doc_a.commit_then_renew(); + + let cid: ContainerID = counter_a.id(); + + // B imports A's history: the child exists but is unreachable (marker cleared). + let doc_b = LoroDoc::new_auto_commit(); + doc_b.set_peer_id(2).unwrap(); + doc_b + .import(&doc_a.export(ExportMode::all_updates()).unwrap()) + .unwrap(); + + let idx = doc_b.state.lock().arena.id_to_idx(&cid).unwrap(); + assert!(doc_b.state.lock().is_deleted(idx)); + assert_eq!( + doc_b.state.lock().dead_cache_entry(idx), + None, + "imported mergeable-dependent deletion must not be cached" + ); + + // A reactivates the child locally and exports just the new update. + let vv_before = doc_a.oplog_vv(); + root_a.ensure_mergeable_counter("revision").unwrap(); + doc_a.commit_then_renew(); + let reactivation = doc_a.export(ExportMode::updates(&vv_before)).unwrap(); + + // B imports the reactivation. There must be no stale deleted-cache entry left to mask it. + doc_b.import(&reactivation).unwrap(); + assert_eq!( + doc_b.state.lock().dead_cache_entry(idx), + None, + "imported reactivation must drop the stale deleted-cache entry" + ); + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/state/list_state.rs b/3rdparty/loro/crates/loro-internal/src/state/list_state.rs new file mode 100644 index 0000000000..474d4b11da --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/state/list_state.rs @@ -0,0 +1,991 @@ +use std::{io::Write, ops::RangeBounds, sync::Weak}; + +use super::{ApplyLocalOpReturn, ContainerState, DiffApplyContext, FastStateSnapshot}; +use crate::{ + configure::Configure, + container::{idx::ContainerIdx, list::list_op::ListOp, ContainerID}, + event::{Diff, Index, InternalDiff, ListDiff}, + handler::ValueOrHandler, + op::{ListSlice, Op, RawOp, RawOpContent}, + LoroDocInner, LoroValue, +}; + +use generic_btree::{ + iter, + rle::{CanRemove, HasLength, Mergeable, Sliceable, TryInsert}, + BTree, BTreeTrait, Cursor, LeafIndex, LengthFinder, UseLengthFinder, +}; +use loro_common::{IdFull, LoroError, LoroResult, ID}; +use loro_delta::array_vec::ArrayVec; +use rustc_hash::FxHashMap; +use smallvec::SmallVec; + +const TINY_LIST_MAX: usize = 4; + +#[derive(Debug)] +pub struct ListState { + idx: ContainerIdx, + list: ListEntries, +} + +impl Clone for ListState { + fn clone(&self) -> Self { + Self { + idx: self.idx, + list: self.list.clone(), + } + } +} + +#[derive(Debug, Clone)] +pub(crate) struct Elem { + pub v: LoroValue, + pub id: IdFull, +} + +impl HasLength for Elem { + fn rle_len(&self) -> usize { + 1 + } +} + +impl Sliceable for Elem { + fn _slice(&self, range: std::ops::Range) -> Self { + assert_eq!(range.start, 0); + assert_eq!(range.end, 1); + self.clone() + } + + fn split(&mut self, _pos: usize) -> Self { + unreachable!() + } +} + +impl Mergeable for Elem { + fn can_merge(&self, _rhs: &Self) -> bool { + false + } + + fn merge_right(&mut self, _rhs: &Self) { + unreachable!() + } + + fn merge_left(&mut self, _left: &Self) { + unreachable!() + } +} + +impl TryInsert for Elem { + fn try_insert(&mut self, _pos: usize, _elem: Self) -> Result<(), Self> { + Err(_elem) + } +} + +impl CanRemove for Elem { + fn can_remove(&self) -> bool { + false + } +} + +struct ListImpl; +impl BTreeTrait for ListImpl { + type Elem = Elem; + type Cache = isize; + type CacheDiff = isize; + const USE_DIFF: bool = true; + + #[inline(always)] + fn calc_cache_internal( + cache: &mut Self::Cache, + caches: &[generic_btree::Child], + ) -> Self::CacheDiff { + let mut new_cache = 0; + for child in caches { + new_cache += child.cache; + } + + let diff = new_cache - *cache; + *cache = new_cache; + diff + } + + #[inline(always)] + fn apply_cache_diff(cache: &mut Self::Cache, diff: &Self::CacheDiff) { + *cache += diff; + } + + #[inline(always)] + fn merge_cache_diff(diff1: &mut Self::CacheDiff, diff2: &Self::CacheDiff) { + *diff1 += diff2 + } + + #[inline(always)] + fn get_elem_cache(_elem: &Self::Elem) -> Self::Cache { + 1 + } + + #[inline(always)] + fn new_cache_to_diff(cache: &Self::Cache) -> Self::CacheDiff { + *cache + } + + fn sub_cache(cache_lhs: &Self::Cache, cache_rhs: &Self::Cache) -> Self::CacheDiff { + cache_lhs - cache_rhs + } +} + +impl UseLengthFinder for ListImpl { + fn get_len(cache: &isize) -> usize { + *cache as usize + } +} + +#[derive(Debug, Clone)] +enum ListEntries { + Tiny { + entries: SmallVec<[Elem; 1]>, + child_container_to_index: FxHashMap, + }, + Tree { + tree: BTree, + child_container_to_index: FxHashMap, + }, +} + +impl ListEntries { + fn new_tiny() -> Self { + Self::Tiny { + entries: SmallVec::new(), + child_container_to_index: Default::default(), + } + } + + fn is_empty(&self) -> bool { + match self { + Self::Tiny { entries, .. } => entries.is_empty(), + Self::Tree { tree, .. } => tree.is_empty(), + } + } + + fn len(&self) -> usize { + match self { + Self::Tiny { entries, .. } => entries.len(), + Self::Tree { tree, .. } => *tree.root_cache() as usize, + } + } + + fn iter(&self) -> Box + '_> { + match self { + Self::Tiny { entries, .. } => Box::new(entries.iter()), + Self::Tree { tree, .. } => Box::new(tree.iter()), + } + } +} + +impl ListState { + pub fn new(idx: ContainerIdx) -> Self { + Self { + idx, + list: ListEntries::new_tiny(), + } + } + + fn rebuild_tiny_child_index(&mut self) { + let ListEntries::Tiny { + entries, + child_container_to_index, + } = &mut self.list + else { + return; + }; + + child_container_to_index.clear(); + for (index, elem) in entries.iter().enumerate() { + if let LoroValue::Container(id) = &elem.v { + child_container_to_index.insert(id.clone(), index); + } + } + } + + fn upgrade_to_tree(&mut self) { + let ListEntries::Tiny { entries, .. } = + std::mem::replace(&mut self.list, ListEntries::new_tiny()) + else { + return; + }; + + let mut tree = BTree::new(); + let mut child_container_to_index = FxHashMap::default(); + for elem in entries { + let container = elem.v.as_container().cloned(); + let leaf = tree.push(elem); + if let Some(container) = container { + child_container_to_index.insert(container, leaf.leaf); + } + } + + self.list = ListEntries::Tree { + tree, + child_container_to_index, + }; + } + + fn downgrade_tree_if_tiny(&mut self) { + let ListEntries::Tree { tree, .. } = &self.list else { + return; + }; + + if *tree.root_cache() as usize > TINY_LIST_MAX { + return; + } + + let entries = tree.iter().cloned().collect(); + self.list = ListEntries::Tiny { + entries, + child_container_to_index: Default::default(), + }; + self.rebuild_tiny_child_index(); + } + + pub fn contains_child_container(&self, id: &ContainerID) -> bool { + match &self.list { + ListEntries::Tiny { + entries, + child_container_to_index, + } => child_container_to_index.get(id).is_some_and(|index| { + entries + .get(*index) + .is_some_and(|elem| elem.v.as_container() == Some(id)) + }), + ListEntries::Tree { + tree, + child_container_to_index, + } => child_container_to_index + .get(id) + .is_some_and(|leaf| tree.get_elem(*leaf).is_some()), + } + } + + pub fn get_child_container_index(&self, id: &ContainerID) -> Option { + let (leaf, list) = match &self.list { + ListEntries::Tiny { + entries, + child_container_to_index, + } => { + let index = *child_container_to_index.get(id)?; + entries.get(index)?; + return Some(index); + } + ListEntries::Tree { + tree, + child_container_to_index, + } => (*child_container_to_index.get(id)?, tree), + }; + + list.get_elem(leaf)?; + let mut index = 0; + list.visit_previous_caches(Cursor { leaf, offset: 0 }, |cache| match cache { + generic_btree::PreviousCache::NodeCache(cache) => { + index += *cache; + } + generic_btree::PreviousCache::PrevSiblingElem(..) => { + index += 1; + } + generic_btree::PreviousCache::ThisElemAndOffset { .. } => {} + }); + + Some(index as usize) + } + + pub fn insert(&mut self, index: usize, value: LoroValue, id: IdFull) { + if index > self.len() { + panic!("Index {index} out of range. The length is {}", self.len()); + } + + if let ListEntries::Tiny { entries, .. } = &mut self.list { + if entries.len() < TINY_LIST_MAX { + entries.insert( + index, + Elem { + v: value.clone(), + id, + }, + ); + self.rebuild_tiny_child_index(); + return; + } + + self.upgrade_to_tree(); + } + + let ListEntries::Tree { + tree, + child_container_to_index, + } = &mut self.list + else { + unreachable!() + }; + let (leaf, data) = tree.insert::( + &index, + Elem { + v: value.clone(), + id, + }, + ); + + if value.is_container() { + child_container_to_index.insert(value.into_container().unwrap(), leaf.leaf); + } + + assert!(data.arr.is_empty()); + } + + pub fn push(&mut self, value: LoroValue, id: IdFull) { + if let ListEntries::Tiny { entries, .. } = &mut self.list { + if entries.len() < TINY_LIST_MAX { + entries.push(Elem { + v: value.clone(), + id, + }); + self.rebuild_tiny_child_index(); + return; + } + + self.upgrade_to_tree(); + } + + let ListEntries::Tree { + tree, + child_container_to_index, + } = &mut self.list + else { + unreachable!() + }; + let leaf = tree.push(Elem { + v: value.clone(), + id, + }); + + if value.is_container() { + child_container_to_index.insert(value.into_container().unwrap(), leaf.leaf); + } + } + + pub fn delete(&mut self, index: usize) -> LoroValue { + if let ListEntries::Tiny { entries, .. } = &mut self.list { + let elem = entries.remove(index); + self.rebuild_tiny_child_index(); + return elem.v; + } + + let ListEntries::Tree { + tree, + child_container_to_index, + } = &mut self.list + else { + unreachable!() + }; + let leaf = tree.query::(&index); + let leaf = tree.remove_leaf(leaf.unwrap().cursor).unwrap(); + if leaf.v.is_container() { + child_container_to_index.remove(leaf.v.as_container().unwrap()); + } + self.downgrade_tree_if_tiny(); + leaf.v + } + + pub fn delete_range( + &mut self, + range: impl RangeBounds, + mut notify_deletion: Option<&mut Vec>, + ) { + let start: usize = match range.start_bound() { + std::ops::Bound::Included(x) => *x, + std::ops::Bound::Excluded(x) => *x + 1, + std::ops::Bound::Unbounded => 0, + }; + let end: usize = match range.end_bound() { + std::ops::Bound::Included(x) => *x + 1, + std::ops::Bound::Excluded(x) => *x, + std::ops::Bound::Unbounded => self.len(), + }; + + if let ListEntries::Tiny { entries, .. } = &mut self.list { + for elem in entries.drain(start..end) { + if let LoroValue::Container(c) = elem.v { + if let Some(notify_deletion) = &mut notify_deletion { + notify_deletion.push(c); + } + } + } + self.rebuild_tiny_child_index(); + return; + } + + if end - start == 1 { + if let LoroValue::Container(c) = self.delete(start) { + if let Some(notify_deletion) = &mut notify_deletion { + notify_deletion.push(c); + } + } + return; + } + + let ListEntries::Tree { + tree, + child_container_to_index, + } = &mut self.list + else { + unreachable!() + }; + let q = start..end; + let start1 = tree.query::(&q.start); + let end1 = tree.query::(&q.end); + for v in iter::Drain::new(tree, start1, end1) { + if v.v.is_container() { + child_container_to_index.remove(v.v.as_container().unwrap()); + if let Some(notify_deletion) = &mut notify_deletion { + notify_deletion.push(v.v.into_container().unwrap()); + } + } + } + self.downgrade_tree_if_tiny(); + } + + // PERF: use &[LoroValue] + // PERF: batch + pub fn insert_batch(&mut self, index: usize, values: Vec, start_id: IdFull) { + let mut id = start_id; + for (i, value) in values.into_iter().enumerate() { + self.insert(index + i, value, id); + id = id.inc(1); + } + } + + pub fn iter(&self) -> impl Iterator { + self.iter_with_id().map(|x| &x.v) + } + + #[allow(unused)] + pub(crate) fn iter_with_id(&self) -> Box + '_> { + self.list.iter() + } + + pub fn len(&self) -> usize { + self.list.len() + } + + fn to_vec(&self) -> Vec { + let mut ans = Vec::with_capacity(self.len()); + for value in self.list.iter() { + ans.push(value.v.clone()); + } + ans + } + + pub fn get(&self, index: usize) -> Option<&LoroValue> { + match &self.list { + ListEntries::Tiny { entries, .. } => entries.get(index).map(|elem| &elem.v), + ListEntries::Tree { tree, .. } => { + let result = tree.query::(&index)?; + if result.found { + Some(&result.elem(tree).unwrap().v) + } else { + None + } + } + } + } + + pub fn get_id_at(&self, index: usize) -> Option { + match &self.list { + ListEntries::Tiny { entries, .. } => entries.get(index).map(|elem| elem.id), + ListEntries::Tree { tree, .. } => { + let result = tree.query::(&index)?; + if result.found { + Some(result.elem(tree).unwrap().id) + } else { + None + } + } + } + } + + #[allow(unused)] + pub(crate) fn check(&self) { + for value in self.iter() { + if let LoroValue::Container(c) = value { + self.get_child_index(c).unwrap(); + } + } + } + + pub fn get_index_of_id(&self, id: ID) -> Option { + for (i, elem) in self.iter_with_id().enumerate() { + if elem.id.id() == id { + return Some(i); + } + } + None + } +} + +impl ContainerState for ListState { + fn container_idx(&self) -> ContainerIdx { + self.idx + } + + fn is_state_empty(&self) -> bool { + self.list.is_empty() + } + + fn apply_diff_and_convert( + &mut self, + diff: InternalDiff, + DiffApplyContext { doc, .. }: DiffApplyContext, + ) -> Diff { + let InternalDiff::ListRaw(delta) = diff else { + unreachable!() + }; + let mut ans: ListDiff = ListDiff::default(); + let mut index = 0; + let doc = &doc.upgrade().unwrap(); + for span in delta.iter() { + match span { + crate::delta::DeltaItem::Retain { retain: len, .. } => { + index += len; + ans.push_retain(*len, Default::default()); + } + crate::delta::DeltaItem::Insert { insert: value, .. } => { + let mut arr = Vec::new(); + match &value.values { + either::Either::Left(range) => { + for i in range.to_range() { + let value = doc.arena.get_value(i).unwrap(); + arr.push(value); + } + } + either::Either::Right(v) => arr.push(v.clone()), + } + for arr in ArrayVec::from_many( + arr.iter() + .map(|v| ValueOrHandler::from_value(v.clone(), doc)), + ) { + ans.push_insert(arr, Default::default()); + } + let len = arr.len(); + self.insert_batch(index, arr, value.id); + index += len; + } + crate::delta::DeltaItem::Delete { delete: len, .. } => { + self.delete_range(index..index + len, None); + ans.push_delete(*len); + } + } + } + + Diff::List(ans) + } + + fn validate_diff(&self, diff: &InternalDiff) -> LoroResult<()> { + let InternalDiff::ListRaw(delta) = diff else { + unreachable!() + }; + + let mut cursor = 0usize; + let mut projected = self.len(); + for span in delta.iter() { + match span { + crate::delta::DeltaItem::Retain { retain: len, .. } => { + cursor += len; + if cursor > projected { + return Err(LoroError::internal(format!( + "list diff retains {cursor} items but state only has {projected}", + ))); + } + } + crate::delta::DeltaItem::Insert { insert: value, .. } => { + let len = match &value.values { + either::Either::Left(range) => range.to_range().len(), + either::Either::Right(_) => 1, + }; + if cursor > projected { + return Err(LoroError::internal(format!( + "list diff inserts at {cursor} but state only has {projected}", + ))); + } + cursor += len; + projected += len; + } + crate::delta::DeltaItem::Delete { delete: len, .. } => { + if cursor + len > projected { + return Err(LoroError::internal(format!( + "list diff deletes {len} at {cursor} but state only has {projected}", + ))); + } + projected -= len; + } + } + } + + Ok(()) + } + + fn apply_diff( + &mut self, + diff: InternalDiff, + DiffApplyContext { doc, .. }: DiffApplyContext, + ) -> LoroResult<()> { + let doc = &doc.upgrade().unwrap(); + match diff { + InternalDiff::ListRaw(delta) => { + let mut index = 0; + for span in delta.iter() { + match span { + crate::delta::DeltaItem::Retain { retain: len, .. } => { + index += len; + } + crate::delta::DeltaItem::Insert { insert: value, .. } => { + let mut arr = Vec::new(); + match &value.values { + either::Either::Left(range) => { + for i in range.to_range() { + let value = doc.arena.get_value(i).unwrap(); + arr.push(value); + } + } + either::Either::Right(v) => arr.push(v.clone()), + } + + let len = arr.len(); + self.insert_batch(index, arr, value.id); + index += len; + } + crate::delta::DeltaItem::Delete { delete: len, .. } => { + self.delete_range(index..index + len, None); + } + } + } + } + _ => unreachable!(), + } + Ok(()) + } + + fn apply_local_op(&mut self, op: &RawOp, _: &Op) -> LoroResult { + let mut ans: ApplyLocalOpReturn = Default::default(); + match &op.content { + RawOpContent::List(list) => match list { + ListOp::Insert { slice, pos } => match slice { + ListSlice::RawData(list) => match list { + std::borrow::Cow::Borrowed(list) => { + self.insert_batch(*pos, list.to_vec(), op.id_full()); + } + std::borrow::Cow::Owned(list) => { + self.insert_batch(*pos, list.clone(), op.id_full()); + } + }, + _ => unreachable!(), + }, + ListOp::Delete(del) => { + self.delete_range(del.span.to_urange(), Some(&mut ans.deleted_containers)); + } + ListOp::Move { .. } => { + unreachable!() + } + ListOp::StyleStart { .. } => unreachable!(), + ListOp::StyleEnd { .. } => unreachable!(), + ListOp::Set { .. } => { + unreachable!() + } + }, + _ => unreachable!(), + } + Ok(ans) + } + + #[doc = " Convert a state to a diff that when apply this diff on a empty state,"] + #[doc = " the state will be the same as this state."] + fn to_diff(&mut self, doc: &Weak) -> Diff { + let doc = &doc.upgrade().unwrap(); + Diff::List(ListDiff::from_many( + self.to_vec() + .into_iter() + .map(|v| ValueOrHandler::from_value(v, doc)), + )) + } + + fn get_value(&mut self) -> LoroValue { + let ans = self.to_vec(); + LoroValue::List(ans.into()) + } + + fn get_child_index(&self, id: &ContainerID) -> Option { + self.get_child_container_index(id).map(Index::Seq) + } + + fn contains_child(&self, id: &ContainerID) -> bool { + self.contains_child_container(id) + } + + fn get_child_containers(&self) -> Vec { + let mut ans = Vec::new(); + for elem in self.iter_with_id() { + if elem.v.is_container() { + ans.push(elem.v.as_container().unwrap().clone()); + } + } + ans + } + + fn fork(&self, _config: &Configure) -> Self { + self.clone() + } +} + +mod snapshot { + use loro_common::PeerID; + use serde_columnar::columnar; + + use crate::{ + encoding::value_register::ValueRegister, + state::{ + decode_lamport_from_delta, decode_peer_from_table, decode_peer_table, + state_decode_error, ContainerCreationContext, + }, + }; + + use super::*; + #[columnar(vec, ser, de, iterable)] + #[derive(Debug, Clone)] + struct EncodedListId { + #[columnar(strategy = "DeltaRle")] + peer_idx: usize, + #[columnar(strategy = "DeltaRle")] + counter: i32, + #[columnar(strategy = "DeltaRle")] + lamport_sub_counter: i32, + } + + #[columnar(ser, de)] + struct EncodedListIds { + #[columnar(class = "vec", iter = "EncodedListId")] + ids: Vec, + } + + impl FastStateSnapshot for ListState { + /// Encodes the ListState snapshot in a compact binary format: + /// 1. Encodes the list values using postcard serialization + /// 2. Encodes a table of unique peer IDs + /// 3. For each element, encodes its ID as: + /// - Index of the peer ID in the table (LEB128) + /// - Counter (LEB128) + /// - Lamport timestamp (LEB128) + fn encode_snapshot_fast(&mut self, mut w: W) { + let value = self.get_value().into_list().unwrap(); + postcard::to_io(&*value, &mut w).unwrap(); + let mut peers: ValueRegister = ValueRegister::new(); + let mut ids = Vec::with_capacity(self.len()); + for elem in self.iter_with_id() { + let id = elem.id; + let peer_idx = peers.register(&id.peer); + ids.push(EncodedListId { + peer_idx, + counter: id.counter, + lamport_sub_counter: (id.lamport as i32 - id.counter), + }); + } + + let peers = peers.unwrap_vec(); + leb128::write::unsigned(&mut w, peers.len() as u64).unwrap(); + for peer in peers { + w.write_all(&peer.to_le_bytes()).unwrap(); + } + + let id_bytes = serde_columnar::to_vec(&EncodedListIds { ids }).unwrap(); + w.write_all(&id_bytes).unwrap(); + } + fn decode_value(bytes: &[u8]) -> LoroResult<(LoroValue, &[u8])> { + let (value, bytes) = postcard::take_from_bytes(bytes).map_err(|_| { + loro_common::LoroError::DecodeError( + "Decode list value failed".to_string().into_boxed_str(), + ) + })?; + let value: Vec = value; + Ok((LoroValue::List(value.into()), bytes)) + } + + fn decode_snapshot_fast( + idx: ContainerIdx, + (v, mut bytes): (LoroValue, &[u8]), + _ctx: ContainerCreationContext, + ) -> LoroResult + where + Self: Sized, + { + let peers = decode_peer_table(&mut bytes, "Decode list state failed")?; + + let EncodedListIds { ids } = serde_columnar::from_bytes(bytes).map_err(|err| { + state_decode_error(format!("Decode list state failed: invalid id table: {err}")) + })?; + + let list = v.as_list().unwrap(); + if ids.len() != list.len() { + return Err(state_decode_error( + "Decode list state failed: id/value length mismatch", + )); + } + + let mut ans = Self::new(idx); + for (i, id) in ids.into_iter().enumerate() { + let peer = decode_peer_from_table(&peers, id.peer_idx, "Decode list state failed")?; + ans.insert( + i, + list[i].clone(), + IdFull::new( + peer, + id.counter, + decode_lamport_from_delta( + id.counter, + id.lamport_sub_counter, + "Decode list state failed", + )?, + ), + ); + } + + Ok(ans) + } + } + + #[cfg(test)] + mod tests { + use loro_common::LoroValue; + + use crate::{container::idx::ContainerIdx, state::ContainerCreationContext}; + + use super::*; + + #[test] + fn list_fast_snapshot_rejects_negative_counter() { + let mut bytes = Vec::new(); + postcard::to_io(&vec![LoroValue::I64(1)], &mut bytes).unwrap(); + leb128::write::unsigned(&mut bytes, 1).unwrap(); + bytes.extend_from_slice(&1_u64.to_le_bytes()); + bytes.extend_from_slice( + &serde_columnar::to_vec(&EncodedListIds { + ids: vec![EncodedListId { + peer_idx: 0, + counter: -1, + lamport_sub_counter: 1, + }], + }) + .unwrap(), + ); + + let idx = ContainerIdx::from_index_and_type(0, loro_common::ContainerType::List); + let (value, state_bytes) = ListState::decode_value(&bytes).unwrap(); + assert!(ListState::decode_snapshot_fast( + idx, + (value, state_bytes), + ContainerCreationContext { + configure: &Default::default(), + peer: 0, + }, + ) + .is_err()); + } + } +} + +#[cfg(test)] +mod test { + use itertools::Itertools; + use loro_common::{Counter, Lamport}; + + use crate::state::ContainerCreationContext; + + use super::*; + + #[test] + fn test() { + let mut list = ListState::new(ContainerIdx::from_index_and_type( + 0, + loro_common::ContainerType::List, + )); + fn id(name: &str) -> ContainerID { + ContainerID::new_root(name, crate::ContainerType::List) + } + list.insert(0, LoroValue::Container(id("abc")), IdFull::new(0, 0, 0)); + list.insert(0, LoroValue::Container(id("x")), IdFull::new(0, 0, 0)); + assert_eq!(list.get_child_container_index(&id("x")), Some(0)); + assert_eq!(list.get_child_container_index(&id("abc")), Some(1)); + list.insert(1, LoroValue::Bool(false), IdFull::new(0, 0, 0)); + assert_eq!(list.get_child_container_index(&id("x")), Some(0)); + assert_eq!(list.get_child_container_index(&id("abc")), Some(2)); + } + + #[test] + fn test_list_fast_snapshot() { + let mut list = ListState::new(ContainerIdx::from_index_and_type( + 0, + loro_common::ContainerType::List, + )); + let mut w = Vec::new(); + list.encode_snapshot_fast(&mut w); + println!("Empty: {}", w.len()); + + list.insert(0, LoroValue::I64(0), IdFull::new(0, 0, 0)); + list.insert(1, LoroValue::I64(2), IdFull::new(1, 1, 1)); + list.insert(2, LoroValue::I64(4), IdFull::new(1, 2, 2)); + let mut w = Vec::new(); + list.encode_snapshot_fast(&mut w); + assert!(w.len() <= 39, "w.len() = {}", w.len()); + let (v, left) = ListState::decode_value(&w).unwrap(); + let mut new_list = ListState::decode_snapshot_fast( + ContainerIdx::from_index_and_type(0, loro_common::ContainerType::List), + (v.clone(), left), + ContainerCreationContext { + configure: &Default::default(), + peer: 0, + }, + ) + .unwrap(); + new_list.check(); + assert_eq!( + new_list.get_value(), + vec![LoroValue::I64(0), LoroValue::I64(2), LoroValue::I64(4)].into() + ); + assert_eq!(new_list.get_value(), v); + let v = new_list.list.iter().collect_vec(); + assert_eq!(v[0].id.peer, 0); + assert_eq!(v[0].id.counter, 0 as Counter); + assert_eq!(v[0].id.lamport, 0 as Lamport); + + assert_eq!(v[1].id.peer, 1); + assert_eq!(v[1].id.counter, 1 as Counter); + assert_eq!(v[1].id.lamport, 1 as Lamport); + + assert_eq!(v[2].id.peer, 1); + assert_eq!(v[2].id.counter, 2 as Counter); + assert_eq!(v[2].id.lamport, 2 as Lamport); + } + + #[test] + fn list_fast_snapshot_rejects_corrupt_state_metadata() { + let idx = ContainerIdx::from_index_and_type(0, loro_common::ContainerType::List); + let ctx = ContainerCreationContext { + configure: &Default::default(), + peer: 0, + }; + + let value = LoroValue::from(vec![LoroValue::I64(1)]); + assert!(ListState::decode_snapshot_fast(idx, (value.clone(), &[1]), ctx).is_err()); + + let mut empty = ListState::new(idx); + let mut bytes = Vec::new(); + empty.encode_snapshot_fast(&mut bytes); + let (_, state_bytes) = ListState::decode_value(&bytes).unwrap(); + assert!(ListState::decode_snapshot_fast(idx, (value, state_bytes), ctx).is_err()); + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/state/map_state.rs b/3rdparty/loro/crates/loro-internal/src/state/map_state.rs new file mode 100644 index 0000000000..947eb6bcc8 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/state/map_state.rs @@ -0,0 +1,723 @@ +use std::{collections::BTreeMap, sync::Weak}; + +use loro_common::{ContainerID, IdLp, LoroResult, PeerID}; +use rustc_hash::FxHashMap; +use smallvec::SmallVec; + +use crate::{ + configure::Configure, + container::{idx::ContainerIdx, map::MapSet}, + delta::{MapValue, ResolvedMapDelta, ResolvedMapValue}, + diff_calc::DiffMode, + event::{Diff, Index, InternalDiff}, + handler::ValueOrHandler, + op::{Op, RawOp, RawOpContent}, + InternalString, LoroDocInner, LoroValue, +}; + +use super::{ApplyLocalOpReturn, ContainerState, DiffApplyContext}; + +const TINY_MAP_MAX: usize = 4; + +/// Apply the mergeable marker → `Container` boundary translation when a parent cid is known. +/// +/// `MapState` is used both with and without a registered parent cid (e.g. during decode), so the +/// translation no-ops when the parent is missing — markers stay as their stored `Binary` form. +fn translate_with_parent( + parent_id: Option<&ContainerID>, + key: &InternalString, + value: LoroValue, +) -> LoroValue { + match parent_id { + Some(parent) => { + loro_common::translate_mergeable_marker_value(parent, key.as_ref(), value) + } + None => value, + } +} + +#[derive(Debug, Clone)] +enum MapEntries { + SortedTiny(SmallVec<[(InternalString, MapValue); 1]>), + Tree(BTreeMap), +} + +impl Default for MapEntries { + fn default() -> Self { + Self::SortedTiny(SmallVec::new()) + } +} + +enum MapEntriesIter<'a> { + SortedTiny(std::slice::Iter<'a, (InternalString, MapValue)>), + Tree(std::collections::btree_map::Iter<'a, InternalString, MapValue>), +} + +impl<'a> Iterator for MapEntriesIter<'a> { + type Item = (&'a InternalString, &'a MapValue); + + fn next(&mut self) -> Option { + match self { + Self::SortedTiny(iter) => iter.next().map(|(key, value)| (key, value)), + Self::Tree(iter) => iter.next(), + } + } +} + +impl MapEntries { + fn debug_assert_sorted_tiny(entries: &[(InternalString, MapValue)]) { + debug_assert!(entries.windows(2).all(|pair| { + let left = &pair[0].0; + let right = &pair[1].0; + left < right + })); + } + + fn is_empty(&self) -> bool { + match self { + Self::SortedTiny(entries) => entries.is_empty(), + Self::Tree(map) => map.is_empty(), + } + } + + fn get(&self, key: &InternalString) -> Option<&MapValue> { + match self { + Self::SortedTiny(entries) => { + Self::debug_assert_sorted_tiny(entries); + entries + .binary_search_by(|(entry_key, _)| entry_key.cmp(key)) + .ok() + .map(|index| &entries[index].1) + } + Self::Tree(map) => map.get(key), + } + } + + fn insert(&mut self, key: InternalString, value: MapValue) -> Option { + match self { + Self::SortedTiny(entries) => { + Self::debug_assert_sorted_tiny(entries); + match entries.binary_search_by(|(entry_key, _)| entry_key.cmp(&key)) { + Ok(index) => return Some(std::mem::replace(&mut entries[index].1, value)), + Err(index) if entries.len() < TINY_MAP_MAX => { + entries.insert(index, (key, value)); + Self::debug_assert_sorted_tiny(entries); + return None; + } + Err(_) => {} + } + + let mut map = BTreeMap::new(); + for (key, value) in entries.drain(..) { + map.insert(key, value); + } + let result = map.insert(key, value); + *self = Self::Tree(map); + result + } + Self::Tree(map) => map.insert(key, value), + } + } + + fn remove(&mut self, key: &InternalString) -> Option { + match self { + Self::SortedTiny(entries) => { + Self::debug_assert_sorted_tiny(entries); + entries + .binary_search_by(|(entry_key, _)| entry_key.cmp(key)) + .ok() + .map(|index| entries.remove(index).1) + } + Self::Tree(map) => { + let result = map.remove(key); + if result.is_some() && map.len() <= TINY_MAP_MAX { + let entries = std::mem::take(map).into_iter().collect(); + *self = Self::SortedTiny(entries); + } + result + } + } + } + + fn iter(&self) -> MapEntriesIter<'_> { + match self { + Self::SortedTiny(entries) => { + Self::debug_assert_sorted_tiny(entries); + MapEntriesIter::SortedTiny(entries.iter()) + } + Self::Tree(map) => MapEntriesIter::Tree(map.iter()), + } + } +} + +#[derive(Debug, Clone)] +enum ChildContainers { + Tiny(SmallVec<[(ContainerID, InternalString); 1]>), + Map(FxHashMap), +} + +impl Default for ChildContainers { + fn default() -> Self { + Self::Tiny(SmallVec::new()) + } +} + +impl ChildContainers { + fn get(&self, id: &ContainerID) -> Option<&InternalString> { + match self { + Self::Tiny(entries) => entries + .iter() + .find_map(|(entry_id, key)| (entry_id == id).then_some(key)), + Self::Map(map) => map.get(id), + } + } + + fn contains_key(&self, id: &ContainerID) -> bool { + self.get(id).is_some() + } + + fn insert(&mut self, id: ContainerID, key: InternalString) -> Option { + match self { + Self::Tiny(entries) => { + if let Some((_, old_key)) = entries.iter_mut().find(|(entry_id, _)| entry_id == &id) + { + return Some(std::mem::replace(old_key, key)); + } + + if entries.len() < TINY_MAP_MAX { + entries.push((id, key)); + return None; + } + + let mut map = FxHashMap::default(); + for (id, key) in entries.drain(..) { + map.insert(id, key); + } + let result = map.insert(id, key); + *self = Self::Map(map); + result + } + Self::Map(map) => map.insert(id, key), + } + } + + fn remove(&mut self, id: &ContainerID) -> Option { + match self { + Self::Tiny(entries) => entries + .iter() + .position(|(entry_id, _)| entry_id == id) + .map(|index| entries.swap_remove(index).1), + Self::Map(map) => { + let result = map.remove(id); + if result.is_some() && map.len() <= TINY_MAP_MAX { + let entries = std::mem::take(map).into_iter().collect(); + *self = Self::Tiny(entries); + } + result + } + } + } +} + +#[derive(Debug, Clone)] +pub struct MapState { + idx: ContainerIdx, + map: MapEntries, + /// Parent-edge index for regular child containers (`MapValue::Container`). Mergeable child + /// edges are resolved by `DocState` from their deterministic cid and this map's marker + /// value, not stored here. + child_containers: ChildContainers, + size: usize, +} + +impl ContainerState for MapState { + fn container_idx(&self) -> ContainerIdx { + self.idx + } + + fn is_state_empty(&self) -> bool { + self.map.is_empty() + } + + fn apply_diff_and_convert( + &mut self, + diff: InternalDiff, + DiffApplyContext { doc, mode }: DiffApplyContext, + ) -> Diff { + let InternalDiff::Map(delta) = diff else { + unreachable!() + }; + let doc = &doc.upgrade().unwrap(); + // The parent cid is the binding side of a mergeable child marker. Cached here so we + // translate marker values into Container values once per delta apply instead of per entry. + let parent_id = doc.arena.idx_to_id(self.idx); + let force = matches!(mode, DiffMode::Checkout | DiffMode::Linear); + let mut resolved_delta = ResolvedMapDelta::new(); + for (key, value) in delta.updated.into_iter() { + let Some(value) = value else { + // uncreate op + assert_eq!(mode, DiffMode::Checkout); + self.remove(&key); + resolved_delta = resolved_delta.with_entry(key, ResolvedMapValue::new_unset()); + continue; + }; + + let mut changed = false; + if force { + self.insert(key.clone(), value.clone()); + changed = true; + } else { + match self.map.get(&key) { + Some(old_value) if old_value > &value => {} + _ => { + self.insert(key.clone(), value.clone()); + changed = true; + } + } + } + + if changed { + resolved_delta = resolved_delta.with_entry( + key.clone(), + ResolvedMapValue { + idlp: IdLp::new(value.peer, value.lamp), + value: value + .value + .map(|v| translate_with_parent(parent_id.as_ref(), &key, v)) + .map(|v| ValueOrHandler::from_value(v, doc)), + }, + ) + } + } + + Diff::Map(resolved_delta) + } + + fn apply_diff(&mut self, diff: InternalDiff, ctx: DiffApplyContext) -> LoroResult<()> { + let _ = self.apply_diff_and_convert(diff, ctx); + Ok(()) + } + + fn apply_local_op(&mut self, op: &RawOp, _: &Op) -> LoroResult { + let mut ans: ApplyLocalOpReturn = Default::default(); + match &op.content { + RawOpContent::Map(MapSet { key, value }) => { + let prev = self.insert( + key.clone(), + MapValue { + lamp: op.lamport, + peer: op.id.peer, + value: value.clone(), + }, + ); + + if let Some(MapValue { + value: Some(LoroValue::Container(c)), + .. + }) = prev + { + ans.deleted_containers.push(c); + } + } + _ => unreachable!(), + } + + Ok(ans) + } + + #[doc = " Convert a state to a diff that when apply this diff on a empty state,"] + #[doc = " the state will be the same as this state."] + fn to_diff(&mut self, doc: &Weak) -> Diff { + let doc_strong = doc.upgrade().unwrap(); + let parent_id = doc_strong.arena.idx_to_id(self.idx); + // Build resolved values directly instead of going through + // `ResolvedMapValue::from_map_value`: that path forced a full `MapValue` clone per entry + // just to overwrite `.value` with the translated form. Cloning only the inner + // `Option` is enough and saves an alloc per entry on every state-to-diff. + Diff::Map(ResolvedMapDelta { + updated: self + .map + .iter() + .map(|(k, v)| { + let value = v + .value + .clone() + .map(|val| translate_with_parent(parent_id.as_ref(), k, val)) + .map(|val| ValueOrHandler::from_value(val, &doc_strong)); + ( + k.clone(), + ResolvedMapValue { + idlp: IdLp::new(v.peer, v.lamp), + value, + }, + ) + }) + .collect::>(), + }) + } + + fn get_value(&mut self) -> LoroValue { + let ans = self.to_map(); + LoroValue::Map(ans.into()) + } + + fn get_child_index(&self, id: &ContainerID) -> Option { + self.child_containers.get(id).map(|x| Index::Key(x.clone())) + } + + fn contains_child(&self, id: &ContainerID) -> bool { + self.child_containers.contains_key(id) + } + + fn get_child_containers(&self) -> Vec { + let mut ans = Vec::new(); + for (_, value) in self.map.iter() { + if let Some(LoroValue::Container(x)) = &value.value { + ans.push(x.clone()); + } + } + ans + } + + fn fork(&self, _config: &Configure) -> Self { + self.clone() + } +} + +impl MapState { + pub fn new(idx: ContainerIdx) -> Self { + Self { + idx, + map: Default::default(), + child_containers: Default::default(), + size: 0, + } + } + + pub fn insert(&mut self, key: InternalString, value: MapValue) -> Option { + let value_yes = value.value.is_some(); + if let Some(LoroValue::Container(id)) = &value.value { + self.child_containers.insert(id.clone(), key.clone()); + } + + let result = self.map.insert(key.clone(), value); + if let Some(Some(LoroValue::Container(c))) = result.as_ref().map(|x| &x.value) { + self.child_containers.remove(c); + } + + match (&result, value_yes) { + (Some(x), true) if x.value.is_none() => { + self.size += 1; + } + (None, true) => { + self.size += 1; + } + (Some(x), false) if x.value.is_some() => { + self.size -= 1; + } + _ => {} + }; + + result + } + + pub fn remove(&mut self, key: &InternalString) { + let result = self.map.remove(key); + if let Some(x) = result { + if x.value.is_some() { + self.size -= 1; + } + if let Some(LoroValue::Container(id)) = x.value { + self.child_containers.remove(&id); + } + }; + } + + pub fn iter(&self) -> impl Iterator { + self.map.iter() + } + + fn to_map(&self) -> FxHashMap { + let mut ans = FxHashMap::with_capacity_and_hasher(self.len(), Default::default()); + for (key, value) in self.map.iter() { + if value.value.is_none() { + continue; + } + + ans.insert(key.to_string(), value.value.as_ref().cloned().unwrap()); + } + + ans + } + + pub fn get(&self, k: &str) -> Option<&LoroValue> { + match self.map.get(&k.into()) { + Some(value) => match &value.value { + Some(v) => Some(v), + None => None, + }, + None => None, + } + } + + pub fn len(&self) -> usize { + self.size + } + + pub fn get_last_edit_peer(&self, key: &str) -> Option { + self.map.get(&key.into()).map(|v| v.peer) + } +} + +mod snapshot { + + use std::collections::BTreeMap; + + use loro_common::{InternalString, LoroValue}; + use rustc_hash::{FxHashMap, FxHashSet}; + use serde_columnar::Itertools; + + use crate::{ + delta::MapValue, + encoding::value_register::ValueRegister, + state::{ + decode_peer_from_table, decode_peer_table, read_state_leb_u64, state_decode_error, + ContainerCreationContext, ContainerState, FastStateSnapshot, + }, + }; + + use super::MapState; + + impl FastStateSnapshot for MapState { + fn encode_snapshot_fast(&mut self, mut w: W) { + // 1. LoroValue + // 2. Vec keys_with_none_value + // 3. leb128 peer_num + peers (in u64) + // 3. Groups of (leb128 peer_idx, leb128 lamport), each has a respective map entry + // from either 1 or 2 when they all sorted by the key strings + let value = self.get_value().into_map().unwrap(); + postcard::to_io(&*value, &mut w).unwrap(); + + let keys_with_none_value = self + .map + .iter() + .filter_map(|(k, v)| if v.value.is_some() { None } else { Some(k) }) + .collect_vec(); + postcard::to_io(&keys_with_none_value, &mut w).unwrap(); + let mut peer_register = ValueRegister::new(); + for (_, v) in self.map.iter() { + peer_register.register(&v.peer); + } + + leb128::write::unsigned(&mut w, peer_register.vec().len() as u64).unwrap(); + for p in peer_register.vec() { + w.write_all(&p.to_le_bytes()).unwrap(); + } + let mut keys: Vec<&InternalString> = self.map.iter().map(|(key, _)| key).collect(); + keys.sort_unstable(); + for key in keys.into_iter() { + let value = self.map.get(key).unwrap(); + let peer_idx = peer_register.register(&value.peer); + leb128::write::unsigned(&mut w, peer_idx as u64).unwrap(); + leb128::write::unsigned(&mut w, value.lamp as u64).unwrap(); + } + } + + fn decode_value(bytes: &[u8]) -> loro_common::LoroResult<(loro_common::LoroValue, &[u8])> { + let (value, bytes) = postcard::take_from_bytes::>(bytes) + .map_err(|_| { + loro_common::LoroError::DecodeError( + "Decode map value failed".to_string().into_boxed_str(), + ) + })?; + Ok((LoroValue::Map(value.into()), bytes)) + } + + fn decode_snapshot_fast( + idx: crate::container::idx::ContainerIdx, + (value, bytes): (loro_common::LoroValue, &[u8]), + _ctx: ContainerCreationContext, + ) -> loro_common::LoroResult + where + Self: Sized, + { + let value = value.into_map().unwrap(); + // keys_with_none_value + let (keys_with_none_value, mut bytes) = + postcard::take_from_bytes::>(bytes).map_err(|_| { + loro_common::LoroError::DecodeError( + "Decode map keys_with_none_value failed" + .to_string() + .into_boxed_str(), + ) + })?; + let keys_with_none_value: FxHashSet<_> = keys_with_none_value.into_iter().collect(); + + // peers + let peers = decode_peer_table(&mut bytes, "Decode map state failed")?; + + // + let mut ans = MapState::new(idx); + let mut keys: Vec<_> = value.keys().map(|x| x.as_str().into()).collect(); + keys.extend(keys_with_none_value.iter().cloned()); + keys.sort_unstable(); + + for key in keys { + let peer_idx = + usize::try_from(read_state_leb_u64(&mut bytes, "Decode map state failed")?) + .map_err(|_| { + state_decode_error("Decode map state failed: peer index overflow") + })?; + let lamp = + u32::try_from(read_state_leb_u64(&mut bytes, "Decode map state failed")?) + .map_err(|_| { + state_decode_error("Decode map state failed: lamport overflow") + })?; + let peer = decode_peer_from_table(&peers, peer_idx, "Decode map state failed")?; + + if keys_with_none_value.contains(&key) { + ans.insert( + key, + MapValue { + value: None, + lamp, + peer, + }, + ); + } else { + let value = value.get(&*key).unwrap(); + ans.insert( + key, + MapValue { + value: Some(value.clone()), + lamp, + peer, + }, + ); + } + } + + if !bytes.is_empty() { + return Err(loro_common::LoroError::DecodeError( + "Decode map state failed".to_string().into_boxed_str(), + )); + } + + Ok(ans) + } + } + + impl MapState { + pub(crate) fn decode_value_as_btree_map( + bytes: &[u8], + ) -> loro_common::LoroResult<(BTreeMap, &[u8])> { + let (value, bytes) = postcard::take_from_bytes::>(bytes) + .map_err(|_| { + loro_common::LoroError::DecodeError( + "Decode map value failed".to_string().into_boxed_str(), + ) + })?; + Ok((value, bytes)) + } + } + + #[cfg(test)] + mod map_snapshot_test { + use loro_common::LoroValue; + + use crate::container::idx::ContainerIdx; + + use super::*; + + #[test] + fn map_fast_snapshot() { + let mut map = MapState::new(ContainerIdx::from_index_and_type( + 0, + loro_common::ContainerType::Map, + )); + map.insert( + "1".into(), + MapValue { + value: None, + lamp: 1, + peer: 1, + }, + ); + map.insert( + "2".into(), + MapValue { + value: Some(LoroValue::I64(0)), + lamp: 2, + peer: 2, + }, + ); + map.insert( + "3".into(), + MapValue { + value: Some(LoroValue::Double(1.0)), + lamp: 3, + peer: 3, + }, + ); + + let mut bytes = Vec::new(); + map.encode_snapshot_fast(&mut bytes); + assert!(bytes.len() <= 50); + + let (value, bytes) = MapState::decode_value(&bytes).unwrap(); + { + let m = value.clone().into_map().unwrap(); + assert_eq!(m.len(), 2); + assert_eq!(m.get("2").unwrap(), &LoroValue::I64(0)); + assert_eq!(m.get("3").unwrap(), &LoroValue::Double(1.0)); + } + + let new_map = MapState::decode_snapshot_fast( + ContainerIdx::from_index_and_type(0, loro_common::ContainerType::Map), + (value, bytes), + ContainerCreationContext { + configure: &Default::default(), + peer: 0, + }, + ) + .unwrap(); + let v = new_map.map.get(&"2".into()).unwrap(); + assert_eq!( + v, + &MapValue { + value: Some(LoroValue::I64(0)), + lamp: 2, + peer: 2, + } + ); + let v = new_map.map.get(&"1".into()).unwrap(); + assert_eq!( + v, + &MapValue { + value: None, + lamp: 1, + peer: 1, + } + ); + } + + #[test] + fn map_fast_snapshot_rejects_corrupt_state_metadata() { + let idx = ContainerIdx::from_index_and_type(0, loro_common::ContainerType::Map); + let ctx = ContainerCreationContext { + configure: &Default::default(), + peer: 0, + }; + let value = LoroValue::from(std::collections::HashMap::from([( + "key".to_string(), + LoroValue::I64(1), + )])); + + let mut empty = MapState::new(idx); + let mut bytes = Vec::new(); + empty.encode_snapshot_fast(&mut bytes); + let (_, state_bytes) = MapState::decode_value(&bytes).unwrap(); + assert!(MapState::decode_snapshot_fast(idx, (value, state_bytes), ctx).is_err()); + } + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/state/mergeable.rs b/3rdparty/loro/crates/loro-internal/src/state/mergeable.rs new file mode 100644 index 0000000000..f41080c269 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/state/mergeable.rs @@ -0,0 +1,93 @@ +//! Mergeable-container edge resolution on `DocState`. +//! +//! Mergeable child containers (created via `MapHandler::ensure_mergeable_*`) use deterministic +//! `ContainerID::Root` ids in a reserved namespace. The cid encodes `(parent, key, kind)`, while +//! the parent map's compact binary child ref is the single source of truth for whether that child +//! is currently active. +//! +//! The ref is intentionally stored as a specially constructed binary marker value. This keeps +//! user-editable strings from being reinterpreted as internal container topology, and lets the +//! resolver fail closed unless the ref's digest matches the exact `(parent, key, kind)` being +//! resolved. + +use loro_common::{ContainerID, ContainerType, InternalString, LoroMapValue}; + +use crate::{container::idx::ContainerIdx, event::Index}; + +use super::{ContainerState, DocState}; + +impl DocState { + /// Resolve a child edge using the document-level logical edge model. + /// + /// Ordinary child containers use the materialized child index stored in their parent state. + /// Mergeable children are resolved lazily from their deterministic cid plus the parent map's + /// current binary child ref, so snapshot/update import does not need any eager mergeable-edge + /// rebuild. + pub(super) fn get_logical_child_index( + &mut self, + parent_idx: ContainerIdx, + child_id: &ContainerID, + ) -> Option { + if let Some((parent_id, key, kind)) = child_id.parse_mergeable() { + return self.resolve_mergeable_child_index(parent_idx, &parent_id, &key, kind); + } + + self.store + .get_container_mut(parent_idx) + .and_then(|parent_state| parent_state.get_child_index(child_id)) + } + + pub(super) fn contains_logical_child( + &mut self, + parent_idx: ContainerIdx, + child_id: &ContainerID, + ) -> bool { + self.get_logical_child_index(parent_idx, child_id).is_some() + } + + fn resolve_mergeable_child_index( + &mut self, + parent_idx: ContainerIdx, + encoded_parent_id: &ContainerID, + key: &str, + kind: ContainerType, + ) -> Option { + if parent_idx.get_type() != ContainerType::Map { + return None; + } + + let actual_parent_id = self.arena.idx_to_id(parent_idx)?; + if &actual_parent_id != encoded_parent_id { + return None; + } + + let value = self.store.map_get(parent_idx, key)?; + if loro_common::parse_mergeable_marker(&actual_parent_id, key, &value) == Some(kind) { + Some(Index::Key(key.into())) + } else { + None + } + } + + /// Derive `(key, cid)` pairs for the active mergeable children of a map directly from its + /// current value table. + /// + /// This is used by deep-value and alive-container walks. It mirrors regular map semantics: + /// the current value at a key determines which child is visible. For mergeable children, that + /// value is a compact binary child ref rather than `LoroValue::Container`. + pub(super) fn mergeable_children_from_value( + &self, + parent_id: &ContainerID, + map_value: &LoroMapValue, + ) -> Vec<(InternalString, ContainerID)> { + let mut ans = Vec::new(); + for (key, value) in map_value.iter() { + if let Some(kind) = loro_common::parse_mergeable_marker(parent_id, key, value) { + let key_istr: InternalString = key.as_str().into(); + let cid = ContainerID::new_mergeable(parent_id, key, kind); + ans.push((key_istr, cid)); + } + } + ans + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/state/movable_list_state.rs b/3rdparty/loro/crates/loro-internal/src/state/movable_list_state.rs new file mode 100644 index 0000000000..ea8f89ba12 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/state/movable_list_state.rs @@ -0,0 +1,2032 @@ +use itertools::Itertools; +use loro_delta::{array_vec::ArrayVec, DeltaRope, DeltaRopeBuilder}; +use serde_columnar::columnar; +use std::sync::Weak; +use tracing::{instrument, warn}; + +use generic_btree::BTree; +use loro_common::{CompactIdLp, ContainerID, IdFull, IdLp, LoroResult, LoroValue, PeerID, ID}; +use rustc_hash::FxHashMap; + +use crate::{ + configure::Configure, + container::{idx::ContainerIdx, list::list_op::ListOp}, + delta::DeltaItem, + diff_calc::DiffMode, + event::{Diff, Index, InternalDiff, ListDeltaMeta}, + handler::ValueOrHandler, + op::{ListSlice, Op, RawOp}, + ListDiff, LoroDocInner, +}; + +use self::{ + inner::{InnerState, UpdateResultFromPosChange}, + list_item_tree::{MovableListTreeTrait, OpLenQuery, UserLenQuery}, +}; + +use super::{ApplyLocalOpReturn, ContainerState, DiffApplyContext}; + +#[derive(Debug, Clone)] +pub struct MovableListState { + idx: ContainerIdx, + inner: InnerState, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ListItem { + pointed_by: Option, + pub(crate) id: IdFull, +} + +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct Element { + pub(crate) value: LoroValue, + pub(crate) value_id: IdLp, + pub(crate) pos: IdLp, +} + +impl Element { + pub(crate) fn value(&self) -> &LoroValue { + &self.value + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum IndexType { + /// This includes the ones that are not being pointed at, which may not be visible to users + ForOp, + /// This only includes the ones that are being pointed at, which means visible to users + ForUser, +} + +mod list_item_tree { + use std::{ + iter::Sum, + ops::{Add, AddAssign, Sub}, + }; + + use generic_btree::{ + rle::{CanRemove, HasLength, Mergeable, Sliceable, TryInsert}, + BTreeTrait, + }; + + use crate::utils::query_by_len::{IndexQuery, QueryByLen}; + + use super::ListItem; + + impl HasLength for ListItem { + fn rle_len(&self) -> usize { + 1 + } + } + + impl Mergeable for ListItem { + fn can_merge(&self, _rhs: &Self) -> bool { + false + } + + fn merge_right(&mut self, _rhs: &Self) { + unreachable!() + } + + fn merge_left(&mut self, _left: &Self) { + unreachable!() + } + } + + impl Sliceable for ListItem { + fn _slice(&self, range: std::ops::Range) -> Self { + assert_eq!(range.len(), 1); + self.clone() + } + } + + impl CanRemove for ListItem { + fn can_remove(&self) -> bool { + false + } + } + + impl TryInsert for ListItem { + fn try_insert(&mut self, _pos: usize, elem: Self) -> Result<(), Self> + where + Self: Sized, + { + Err(elem) + } + } + + #[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] + pub(super) struct Cache { + /// This length info include the ones that were deleted. + /// It's used in op and diff calculation. + pub include_dead_len: i32, + /// This length info does not include the ones that were deleted. + /// So it's facing the users. + pub user_len: i32, + } + + impl Add for Cache { + type Output = Self; + + fn add(self, rhs: Self) -> Self::Output { + Self { + include_dead_len: self.include_dead_len + rhs.include_dead_len, + user_len: self.user_len + rhs.user_len, + } + } + } + + impl AddAssign for Cache { + fn add_assign(&mut self, rhs: Self) { + *self = *self + rhs; + } + } + + impl Sub for Cache { + type Output = Self; + + fn sub(self, rhs: Self) -> Self::Output { + Self { + include_dead_len: self.include_dead_len - rhs.include_dead_len, + user_len: self.user_len - rhs.user_len, + } + } + } + + impl Sum for Cache { + fn sum>(iter: I) -> Self { + iter.fold(Self::default(), |acc, x| acc + x) + } + } + + impl CanRemove for Cache { + fn can_remove(&self) -> bool { + self.include_dead_len == 0 + } + } + + pub(super) struct MovableListTreeTrait; + + impl BTreeTrait for MovableListTreeTrait { + type Elem = ListItem; + + type Cache = Cache; + + type CacheDiff = Cache; + + fn calc_cache_internal( + cache: &mut Self::Cache, + caches: &[generic_btree::Child], + ) -> Self::CacheDiff { + let new: Cache = caches.iter().map(|x| *x.cache()).sum(); + let diff = new - *cache; + *cache = new; + diff + } + + fn apply_cache_diff(cache: &mut Self::Cache, diff: &Self::CacheDiff) { + *cache += *diff; + } + + fn merge_cache_diff(diff1: &mut Self::CacheDiff, diff2: &Self::CacheDiff) { + *diff1 += *diff2; + } + + fn get_elem_cache(elem: &Self::Elem) -> Self::Cache { + if elem.pointed_by.is_some() { + Cache { + include_dead_len: 1, + user_len: 1, + } + } else { + Cache { + include_dead_len: 1, + user_len: 0, + } + } + } + + fn new_cache_to_diff(cache: &Self::Cache) -> Self::CacheDiff { + *cache + } + + fn sub_cache(cache_lhs: &Self::Cache, cache_rhs: &Self::Cache) -> Self::CacheDiff { + *cache_lhs - *cache_rhs + } + } + + pub(crate) struct UserLenQueryT; + pub(crate) type UserLenQuery = IndexQuery; + + impl QueryByLen for UserLenQueryT { + fn get_cache_len(cache: &::Cache) -> usize { + cache.user_len as usize + } + + fn get_elem_len(elem: &::Elem) -> usize { + if elem.pointed_by.is_some() { + 1 + } else { + 0 + } + } + + fn get_offset_and_found( + left: usize, + elem: &::Elem, + ) -> (usize, bool) { + if left == 0 && elem.pointed_by.is_some() { + (0, true) + } else { + (1, false) + } + } + + fn get_cache_entity_len(cache: &::Cache) -> usize { + cache.user_len as usize + } + } + + pub(crate) struct IncludeDeadLenQueryT; + pub(crate) type OpLenQuery = IndexQuery; + + impl QueryByLen for IncludeDeadLenQueryT { + fn get_cache_len(cache: &::Cache) -> usize { + cache.include_dead_len as usize + } + + fn get_elem_len(_elem: &::Elem) -> usize { + 1 + } + + fn get_offset_and_found( + left: usize, + _elem: &::Elem, + ) -> (usize, bool) { + if left == 0 { + return (0, true); + } + + (1, false) + } + + fn get_cache_entity_len(cache: &::Cache) -> usize { + cache.include_dead_len as usize + } + } +} + +/// The inner state of the list. +/// +/// The operations inside this mod need to ensure certain invariants. +/// +/// - list items' `pointed_by` must be consistent with the `elements`' `pos`. +/// - `id_to_list_leaf` must be consistent with the list. +/// - `child_container_to_elem` must be consistent with the element. +mod inner { + use generic_btree::{BTree, Cursor, LeafIndex, Query}; + use loro_common::{CompactIdLp, ContainerID, IdFull, IdLp, LoroValue, PeerID}; + use rustc_hash::{FxHashMap, FxHashSet}; + use tracing::error; + + use super::{ + list_item_tree::{MovableListTreeTrait, OpLenQuery, UserLenQuery}, + Element, IndexType, ListItem, + }; + + #[derive(Debug, Clone)] + pub(super) struct InnerState { + list: BTree, + id_to_list_leaf: FxHashMap, + elements: FxHashMap, + /// This mapping may be out of date when elem is removed/updated. + /// But it's guaranteed that if there is a ContainerID in the actual list, + /// it will be mapped correctly. + child_container_to_elem: FxHashMap, + } + + impl PartialEq for InnerState { + fn eq(&self, other: &Self) -> bool { + let v = self.id_to_list_leaf == other.id_to_list_leaf + && self.elements == other.elements + && self.child_container_to_elem == other.child_container_to_elem; + if !v { + false + } else { + self.list.iter().zip(other.list.iter()).all(|(a, b)| a == b) + } + } + } + + fn eq(a: T, b: T) -> Result<(), ()> { + if a == b { + Ok(()) + } else { + Err(()) + } + } + + impl InnerState { + pub fn new() -> Self { + Self { + list: BTree::new(), + id_to_list_leaf: FxHashMap::default(), + elements: FxHashMap::default(), + child_container_to_elem: FxHashMap::default(), + } + } + + #[inline] + pub fn child_container_to_elem(&self) -> &FxHashMap { + &self.child_container_to_elem + } + + #[inline] + pub fn elements(&self) -> &FxHashMap { + &self.elements + } + + #[inline] + pub fn list(&self) -> &BTree { + &self.list + } + + pub fn remove_elem_by_id(&mut self, elem_id: &CompactIdLp) { + self.elements.remove(elem_id); + } + + #[allow(dead_code)] + pub fn check_consistency(&self) { + if !cfg!(debug_assertions) { + return; + } + + let mut failed = false; + if self.check_list_item_consistency().is_err() { + error!("list item consistency check failed, self={:#?}", self); + failed = true; + } + + if self.check_child_container_to_elem_consistency().is_err() { + error!( + "child container to elem consistency check failed, self={:#?}", + self + ); + failed = true; + } + + if failed { + panic!("consistency check failed"); + } + } + + fn check_list_item_consistency(&self) -> Result<(), ()> { + let mut visited_ids = FxHashSet::default(); + for list_item in self.list.iter() { + { + if visited_ids.contains(&list_item.id.idlp()) { + error!("duplicate list item id"); + return Err(()); + } + visited_ids.insert(list_item.id.idlp()); + } + let leaf = self.id_to_list_leaf.get(&list_item.id.idlp()).unwrap(); + let list_item_found = self.list.get_elem(*leaf).unwrap(); + eq(list_item_found, list_item)?; + if let Some(pointed_by) = list_item_found.pointed_by { + let elem = self.elements.get(&pointed_by).unwrap(); + eq(elem.pos, list_item.id.idlp())?; + } + } + + for (elem_id, elem) in self.elements.iter() { + if let Some(item) = self.get_list_item_by_id(elem.pos) { + eq(item.pointed_by, Some(*elem_id))?; + } + } + + Ok(()) + } + + fn check_child_container_to_elem_consistency(&self) -> Result<(), ()> { + for (elem_id, elem) in self.elements.iter() { + if let LoroValue::Container(c) = &elem.value { + eq(self.child_container_to_elem.get(c), Some(elem_id))?; + } + } + + Ok(()) + } + + pub fn get_list_item_index(&self, id: IdLp, kind: IndexType) -> Option { + self.id_to_list_leaf + .get(&id) + .map(|leaf| self.get_index_of(*leaf, kind) as usize) + } + + pub fn get_index_of(&self, leaf: LeafIndex, kind: IndexType) -> i32 { + let mut ans = 0; + self.list + .visit_previous_caches(Cursor { leaf, offset: 0 }, |cache| match cache { + generic_btree::PreviousCache::NodeCache(c) => { + if matches!(kind, IndexType::ForUser) { + ans += c.user_len; + } else { + ans += c.include_dead_len; + } + } + generic_btree::PreviousCache::PrevSiblingElem(p) => { + if matches!(kind, IndexType::ForUser) { + ans += if p.pointed_by.is_some() { 1 } else { 0 }; + } else { + ans += 1; + } + } + generic_btree::PreviousCache::ThisElemAndOffset { .. } => {} + }); + ans + } + + #[inline] + pub fn get_list_item_at(&self, pos: usize, index_type: IndexType) -> Option<&ListItem> { + let index = match index_type { + IndexType::ForUser => self.list.query::(&pos)?, + IndexType::ForOp => self.list.query::(&pos)?, + }; + self.list.get_elem(index.leaf()) + } + + pub fn get_child_index(&self, id: &ContainerID, index_type: IndexType) -> Option { + let ans = self.child_container_to_elem.get(id).and_then(|eid| { + let this = &self; + let elem_id = eid.to_id().compact(); + let elem = this.elements.get(&elem_id)?; + if elem.value.as_container() != Some(id) { + // TODO: may be better to find a way clean up these invalid elements? + return None; + } + if let Some(leaf) = self.id_to_list_leaf.get(&elem.pos) { + let list_item = self.list.get_elem(*leaf)?; + assert_eq!(list_item.pointed_by, Some(elem_id)); + } else { + return None; + } + this.get_list_item_index(elem.pos, index_type) + }); + + ans + } + + pub fn contains_child_container(&self, id: &ContainerID) -> bool { + self.get_child_index(id, IndexType::ForUser).is_some() + } + + #[inline] + pub fn get_list_item_by_id(&self, id: IdLp) -> Option<&ListItem> { + self.id_to_list_leaf + .get(&id) + .map(|&leaf| self.list.get_elem(leaf).unwrap()) + } + + pub fn len_kind(&self, kind: IndexType) -> usize { + match kind { + IndexType::ForUser => self.list.root_cache().user_len as usize, + IndexType::ForOp => self.list.root_cache().include_dead_len as usize, + } + } + + /// Insert a new list item at the given position (op index). + /// Return the value if the insertion activated an element. + pub fn insert_list_item(&mut self, pos: usize, list_item_id: IdFull) { + let c = self + .list + .insert::( + &pos, + ListItem { + pointed_by: None, + id: list_item_id, + }, + ) + .0; + self.id_to_list_leaf.insert(list_item_id.idlp(), c.leaf); + } + + /// Drain the list items in the given range (op index). + /// + /// We also remove the elements that are pointed by the list items. + pub fn list_drain( + &mut self, + range: std::ops::Range, + mut on_elem_id: impl FnMut(CompactIdLp, &Element), + ) { + for item in Self::drain_by_query::(&mut self.list, range) { + self.id_to_list_leaf.remove(&item.id.idlp()); + if let Some(elem_id) = &item.pointed_by { + let elem = self.elements.get(elem_id).unwrap(); + on_elem_id(*elem_id, elem); + self.elements.remove(elem_id); + } + } + } + + fn drain_by_query>( + list: &mut BTree, + range: std::ops::Range, + ) -> generic_btree::iter::Drain<'_, MovableListTreeTrait> { + let start = list.query::(&range.start); + let end = list.query::(&range.end); + generic_btree::iter::Drain::new(list, start, end) + } + + /// Update the pos pointer of an element and the corresponding list items. + /// + /// The old list item will be removed if `remove_old` is true; Otherwise, + /// it will be updated with its `pointed_by` removed. + /// + #[must_use] + pub fn update_pos( + &mut self, + elem_id: CompactIdLp, + new_pos: IdLp, + remove_old: bool, + ) -> UpdateResultFromPosChange { + let mut old_item_id = None; + if let Some(element) = self.elements.get_mut(&elem_id) { + if element.pos != new_pos { + old_item_id = Some(element.pos); + element.pos = new_pos; + } else { + return UpdateResultFromPosChange { + activate_new_list_item: false, + new_list_item_leaf: None, + removed_old_list_item_leaf: None, + }; + } + } else { + self.elements.insert( + elem_id, + Element { + value: LoroValue::Null, + value_id: IdLp::NONE_ID, + pos: new_pos, + }, + ); + } + + let mut ans = UpdateResultFromPosChange { + activate_new_list_item: true, + new_list_item_leaf: None, + removed_old_list_item_leaf: None, + }; + if let Some(leaf) = self.id_to_list_leaf.get(&new_pos) { + ans.new_list_item_leaf = Some(*leaf); + self.list.update_leaf(*leaf, |list_item| { + debug_assert!( + list_item.pointed_by.is_none(), + "list_item was pointed by {:?} but need to be changed to {:?}", + list_item.pointed_by, + elem_id + ); + ans.activate_new_list_item = list_item.pointed_by.is_none(); + list_item.pointed_by = Some(elem_id); + (true, None, None) + }); + } else { + ans.activate_new_list_item = false; + } + + if let Some(old) = old_item_id { + if old.is_none() || old == new_pos { + return ans; + } + + if remove_old { + let leaf = self.id_to_list_leaf.remove(&old).unwrap(); + let elem = self.list.remove_leaf(Cursor { leaf, offset: 0 }).unwrap(); + assert_eq!(elem.pointed_by, Some(elem_id)); + } else if let Some(leaf) = self.id_to_list_leaf.get(&old) { + ans.removed_old_list_item_leaf = Some(*leaf); + let (still_valid, split) = self.list.update_leaf(*leaf, |item| { + item.pointed_by = None; + (true, None, None) + }); + assert!(still_valid); + assert!(split.arr.is_empty()); + } + } + + ans + } + + /// The update can reflected on an event exposed to users. This method calculates the + /// corresponding event position information. + pub fn convert_update_to_event_pos( + &self, + update: UpdateResultFromPosChange, + ) -> EventPosInfo { + if update.activate_new_list_item { + let new = update.new_list_item_leaf.unwrap(); + if let Some(del) = update.removed_old_list_item_leaf { + let mut insert_pos = self.get_index_of(new, IndexType::ForUser) as usize; + let del_pos = self.get_index_of(del, IndexType::ForUser) as usize; + if insert_pos > del_pos { + insert_pos += 1; + } + EventPosInfo { + activate_new_list_item: update.activate_new_list_item, + insert: Some(insert_pos), + delete: Some(del_pos), + } + } else { + EventPosInfo { + activate_new_list_item: update.activate_new_list_item, + insert: Some(self.get_index_of(new, IndexType::ForUser) as usize), + delete: None, + } + } + } else if let Some(del) = update.removed_old_list_item_leaf { + EventPosInfo { + activate_new_list_item: update.activate_new_list_item, + insert: None, + delete: Some(self.get_index_of(del, IndexType::ForUser) as usize), + } + } else { + EventPosInfo { + activate_new_list_item: update.activate_new_list_item, + insert: None, + delete: None, + } + } + } + + pub fn update_value( + &mut self, + elem_id: CompactIdLp, + new_value: LoroValue, + value_id: IdLp, + ) -> Option { + debug_assert!(elem_id.peer != PeerID::MAX); + debug_assert!(!value_id.is_none()); + if let LoroValue::Container(c) = &new_value { + self.child_container_to_elem.insert(c.clone(), elem_id); + } + + if let Some(element) = self.elements.get_mut(&elem_id) { + if let LoroValue::Container(c) = &element.value { + if element.value != new_value { + self.child_container_to_elem.remove(c); + } + } + let old_value = std::mem::replace(&mut element.value, new_value); + element.value_id = value_id; + Some(old_value) + } else { + self.elements.insert( + elem_id, + Element { + value: new_value, + value_id, + pos: IdLp::NONE_ID, + }, + ); + None + } + } + + /// push a new elem into the list + pub fn push_inner(&mut self, list_item_id: IdFull, elem: Option) { + let pointed_by = elem.as_ref().map(|x| x.elem_id); + if let Some(elem) = elem { + if let LoroValue::Container(c) = &elem.value { + self.child_container_to_elem.insert(c.clone(), elem.elem_id); + } + debug_assert!(!elem.last_set_id.is_none()); + self.elements.insert( + elem.elem_id, + Element { + value: elem.value, + value_id: elem.last_set_id, + pos: list_item_id.idlp(), + }, + ); + } + let cursor = self.list.push(ListItem { + pointed_by, + id: list_item_id, + }); + self.id_to_list_leaf + .insert(list_item_id.idlp(), cursor.leaf); + } + } + + pub(super) struct PushElemInfo { + pub elem_id: CompactIdLp, + pub value: LoroValue, + pub last_set_id: IdLp, + } + + pub(super) struct UpdateResultFromPosChange { + /// Whether the list item is activated by this change. + /// If so, the length of the list will be increased by 1 from the users' perspective. + pub activate_new_list_item: bool, + pub new_list_item_leaf: Option, + /// Remove from the users' perspective. But they are still in the state. + pub removed_old_list_item_leaf: Option, + } + + /// This struct assumes we apply insert first and then delete. + pub(super) struct EventPosInfo { + pub activate_new_list_item: bool, + pub insert: Option, + pub delete: Option, + } +} + +impl MovableListState { + pub fn new(idx: ContainerIdx) -> Self { + Self { + idx, + inner: InnerState::new(), + } + } + + #[inline] + fn list(&self) -> &BTree { + self.inner.list() + } + + #[inline] + pub(crate) fn elements(&self) -> &FxHashMap { + self.inner.elements() + } + + /// Return whether the list item is activated by this change. + /// + /// If so, the length of the list will be increased by 1 from the users' perspective. + /// + /// It's false when the list item is already activated or deleted. + #[must_use] + fn create_new_elem( + &mut self, + elem_id: CompactIdLp, + new_pos: IdLp, + new_value: LoroValue, + value_id: IdLp, + ) -> UpdateResultFromPosChange { + self.inner.update_value(elem_id, new_value, value_id); + self.inner.update_pos(elem_id, new_pos, false) + } + + /// Return the values that are activated by the insertions of the list items, and their elem ids. + /// + /// The activation means there were elements that already points to the list items + /// inserted by this op. + fn list_insert_batch(&mut self, index: usize, items: impl Iterator) { + for (i, item) in items.enumerate() { + self.inner.insert_list_item(index + i, item); + } + } + + /// Get the length defined by op, where the length includes the ones that are not being pointed at (moved, invisible to users). + #[allow(unused)] + fn op_len(&self) -> usize { + self.inner.list().root_cache().include_dead_len as usize + } + + fn mov( + &mut self, + from_index: usize, + to_index: usize, + elem_id: IdLp, + new_pos_id: IdFull, + kind: IndexType, + ) { + // PERF: can be optimized by inlining try update + if cfg!(debug_assertions) { + let item = self.inner.get_list_item_at(from_index, kind).unwrap(); + assert_eq!(item.pointed_by, Some(elem_id.compact())); + } + + self.inner.insert_list_item( + if to_index > from_index { + to_index + 1 + } else { + to_index + }, + new_pos_id, + ); + let _ = self + .inner + .update_pos(elem_id.compact(), new_pos_id.idlp(), true); + } + + pub(crate) fn get(&self, index: usize, kind: IndexType) -> Option<&LoroValue> { + if index >= self.len() { + return None; + } + + let item = self.inner.get_list_item_at(index, kind)?; + let elem = item.pointed_by?; + self.inner.elements().get(&elem).map(|x| &x.value) + } + + pub(crate) fn get_elem_at_given_pos( + &self, + index: usize, + kind: IndexType, + ) -> Option<(CompactIdLp, &Element)> { + self.inner.get_list_item_at(index, kind).and_then(|x| { + x.pointed_by + .map(|pointed_by| (pointed_by, self.inner.elements().get(&pointed_by).unwrap())) + }) + } + + pub(crate) fn get_list_id_at(&self, index: usize, kind: IndexType) -> Option { + self.inner.get_list_item_at(index, kind).map(|x| x.id.id()) + } + + pub(crate) fn get_elem_id_at(&self, index: usize, kind: IndexType) -> Option { + self.inner + .get_list_item_at(index, kind) + .and_then(|x| x.pointed_by) + } + + pub(crate) fn convert_index( + &self, + index: usize, + from: IndexType, + to: IndexType, + ) -> Option { + let len = self.inner.len_kind(from); + if index == len { + return Some(self.inner.len_kind(to)); + } + + if index > len { + return None; + } + + let c = match from { + IndexType::ForOp => self.inner.list().query::(&index).unwrap(), + IndexType::ForUser => self.inner.list().query::(&index).unwrap(), + }; + + Some(self.inner.get_index_of(c.cursor.leaf, to) as usize) + } + + pub(crate) fn get_list_item(&self, id: IdLp) -> Option<&ListItem> { + self.inner.get_list_item_by_id(id) + } + + /// Get the user index of elem + /// + /// If we cannot find the list item in the list, we will return None. + fn get_index_of_elem(&self, elem_id: CompactIdLp) -> Option { + let elem = self.inner.elements().get(&elem_id)?; + self.inner.get_list_item_index(elem.pos, IndexType::ForUser) + } + + /// Debug check the consistency between the list and the elements + /// + /// We need to ensure that: + /// - Every element's pos is in the list, and it has a `pointed_by` value that points + /// back to the element + /// - Every list item's `pointed_by` value points to an element in the elements, and + /// the element has the correct `pos` value + #[cfg(any(debug_assertions, test))] + #[allow(unused)] + pub(crate) fn check_consistency(&self) { + for (id, elem) in self.inner.elements().iter() { + let item = self + .get_list_item(id.to_id()) + .expect("Elem's pos should be in the list"); + assert_eq!(item.pointed_by.unwrap(), *id); + } + + for item in self.inner.list().iter() { + if let Some(elem_id) = item.pointed_by { + let elem = self.inner.elements().get(&elem_id).unwrap(); + assert_eq!(elem.pos, item.id.idlp()); + } + } + } + + pub fn iter(&self) -> impl Iterator { + // PERF: can be optimized + (0..self.len()).map(move |i| self.get(i, IndexType::ForUser).unwrap()) + } + + pub fn iter_with_last_move_id_and_elem_id( + &self, + ) -> impl Iterator { + self.inner.list().iter().filter_map(|list_item| { + if let Some(elem_id) = list_item.pointed_by.as_ref() { + let elem = self.inner.elements().get(elem_id).unwrap(); + Some((list_item.id, *elem_id, &elem.value)) + } else { + None + } + }) + } + + pub fn len(&self) -> usize { + self.inner.list().root_cache().user_len as usize + } + + fn to_vec(&self) -> Vec { + self.iter().cloned().collect_vec() + } + + pub(crate) fn get_index_of_id(&self, id: ID) -> Option { + let mut user_index = 0; + for item in self.list().iter() { + if item.id.peer == id.peer && item.id.counter == id.counter { + return Some(user_index); + } + + user_index += if item.pointed_by.is_some() { 1 } else { 0 }; + } + None + } + + fn get_value_inner(&self) -> Vec { + let list = self + .inner + .list() + .iter_with_filter(|x| (x.user_len > 0, 0)) + .filter_map(|(_, item)| { + item.pointed_by + .map(|eid| self.elements()[&eid].value.clone()) + }) + .collect(); + list + } + + pub(crate) fn get_list_item_id_at(&self, pos: usize) -> Option { + let item = self.inner.get_list_item_at(pos, IndexType::ForUser); + item.map(|x| x.id) + } + + #[allow(unused)] + fn check_get_child_index_correctly(&mut self) { + let value = self.get_value(); + let list = value.into_list().unwrap(); + for (i, item) in list.iter().enumerate() { + if let LoroValue::Container(c) = item { + let child_index = self.get_child_index(c).expect("cannot find child index"); + assert_eq!(child_index.into_seq().unwrap(), i); + } + } + } + + pub fn get_creator_at(&self, pos: usize) -> Option { + self.inner + .get_list_item_at(pos, IndexType::ForUser) + .and_then(|x| x.pointed_by.map(|x| x.peer)) + } + + pub fn get_last_mover_at(&self, pos: usize) -> Option { + self.inner + .get_list_item_at(pos, IndexType::ForUser) + .map(|x| x.id.peer) + } + + pub fn get_last_editor_at(&self, pos: usize) -> Option { + self.inner + .get_list_item_at(pos, IndexType::ForUser) + .and_then(|x| { + x.pointed_by + .and_then(|x| self.inner.elements().get(&x).map(|x| x.value_id.peer)) + }) + } +} + +impl ContainerState for MovableListState { + fn container_idx(&self) -> ContainerIdx { + self.idx + } + + fn is_state_empty(&self) -> bool { + self.list().is_empty() && self.elements().is_empty() + } + + // How we apply the diff is coupled with the [DiffMode] we used to calculate the diff. + // So be careful when you modify this function. + #[instrument(skip_all)] + fn apply_diff_and_convert( + &mut self, + diff: InternalDiff, + DiffApplyContext { doc, mode }: DiffApplyContext, + ) -> Diff { + let InternalDiff::MovableList(mut diff) = diff else { + unreachable!() + }; + + // let start_value = self.get_value(); + if cfg!(debug_assertions) { + self.inner.check_consistency(); + } + + let mut event: ListDiff = DeltaRope::new(); + let mut maybe_moved: FxHashMap = FxHashMap::default(); + let need_compare = matches!(mode, DiffMode::Import); + + { + // apply deletions and calculate `maybe_moved` + let mut index = 0; + for delta_item in diff.list.iter() { + match delta_item { + DeltaItem::Retain { + retain, + attributes: _, + } => { + index += retain; + } + DeltaItem::Insert { .. } => {} + DeltaItem::Delete { + delete, + attributes: _, + } => { + let mut user_index = self + .convert_index(index, IndexType::ForOp, IndexType::ForUser) + .unwrap(); + let user_index_end = self + .convert_index(index + delete, IndexType::ForOp, IndexType::ForUser) + .unwrap(); + event.compose( + &DeltaRopeBuilder::new() + .retain(user_index, Default::default()) + .delete(user_index_end - user_index) + .build(), + ); + self.inner + .list_drain(index..index + delete, |elem_id, elem| { + maybe_moved.insert(elem_id, (user_index, elem.value.clone())); + if !matches!(mode, DiffMode::Checkout) { + if let Some(new_elem) = diff.elements.get_mut(&elem_id) { + if new_elem.value_id.is_none() { + new_elem.value = elem.value.clone(); + new_elem.value_id = Some(elem.value_id); + new_elem.value_updated = false; + } + + if new_elem.pos.is_none() { + new_elem.pos = Some(elem.pos); + } + } + } + + user_index += 1; + }); + assert_eq!(user_index, user_index_end); + } + } + } + } + + { + // apply list item changes + let mut index = 0; + for delta_item in diff.list.into_iter() { + match delta_item { + DeltaItem::Retain { + retain, + attributes: _, + } => { + index += retain; + } + DeltaItem::Insert { + insert, + attributes: _, + } => { + let len = insert.len(); + self.list_insert_batch(index, insert.into_iter()); + index += len; + } + DeltaItem::Delete { .. } => {} + } + } + } + + { + let doc = &doc.upgrade().unwrap(); + // Apply element changes + // + // In this block, we need to handle the events generated from the following sources: + // + // - The change of elem's pos that activated the list item. This creates an insert event. + // - The change of elem's value. This create a deletion and an insert event. + // + // It doesn't need to worry about the deletion of the list item, because it's handled by the list diff. + + for (elem_id, delta_item) in diff.elements.into_iter() { + let crate::delta::ElementDelta { + pos, + value, + value_updated, + value_id, + } = delta_item; + // Element may be dropped after snapshot encoding, + // so we need to check which kind of update we need to do + + match self.inner.elements().get(&elem_id).cloned() { + Some(elem) => { + // Update value if needed + if let Some(value_id) = value_id { + if elem.value != value && (!need_compare || elem.value_id < value_id) { + maybe_moved.remove(&elem_id); + self.inner.update_value(elem_id, value.clone(), value_id); + let index = self.get_index_of_elem(elem_id); + if let Some(index) = index { + event.compose( + &DeltaRopeBuilder::new() + .retain(index, Default::default()) + .delete(1) + .insert( + ArrayVec::from([ValueOrHandler::from_value( + value, doc, + )]), + ListDeltaMeta { from_move: false }, + ) + .build(), + ) + } + } + } + + // Update pos if needed + if let Some(pos) = pos { + if elem.pos != pos && (!need_compare || elem.pos < pos) { + // don't need to update old list item, because it's handled by list diff already + let result = self.inner.update_pos(elem_id, pos, false); + let result = self.inner.convert_update_to_event_pos(result); + if let Some(new_index) = result.insert { + let new_value = + self.elements().get(&elem_id).unwrap().value.clone(); + let from_delete = if let Some((_elem_index, elem_old_value)) = + maybe_moved.remove(&elem_id) + { + elem_old_value == new_value + } else { + false + }; + let new_delta: ListDiff = DeltaRopeBuilder::new() + .retain(new_index, Default::default()) + .insert( + ArrayVec::from([ValueOrHandler::from_value( + new_value, doc, + )]), + ListDeltaMeta { + from_move: (result.delete.is_some() + && !value_updated) + || from_delete, + }, + ) + .build(); + event.compose(&new_delta); + } + if let Some(del_index) = result.delete { + event.compose( + &DeltaRopeBuilder::new() + .retain(del_index, Default::default()) + .delete(1) + .build(), + ); + } + if !result.activate_new_list_item { + // not matched list item found, remove directly + self.inner.remove_elem_by_id(&elem_id); + } + } + } + } + None => { + // Need to create new element + let result = self.create_new_elem( + elem_id, + pos.unwrap(), + value.clone(), + value_id.unwrap(), + ); + // Composing events + let result = self.inner.convert_update_to_event_pos(result); + // Create event for pos change and value change + if let Some(index) = result.insert { + let from_delete = if let Some((_elem_index, elem_old_value)) = + maybe_moved.remove(&elem_id) + { + elem_old_value == value + } else { + false + }; + event.compose( + &DeltaRopeBuilder::new() + .retain(index, Default::default()) + .insert( + ArrayVec::from([ValueOrHandler::from_value(value, doc)]), + ListDeltaMeta { + from_move: (result.delete.is_some() && !value_updated) + || from_delete, + }, + ) + .build(), + ) + } + if let Some(index) = result.delete { + event.compose( + &DeltaRopeBuilder::new() + .retain(index, Default::default()) + .delete(1) + .build(), + ); + } + if !result.activate_new_list_item { + // not matched list item found, remove directly + self.inner.remove_elem_by_id(&elem_id); + } + } + } + } + } + + { + // Remove redundant elements + + // We now know that the elements inside `maybe_moved` are actually deleted. + // So we can safely remove them. + let redundant = maybe_moved; + for (elem_id, _) in redundant.iter() { + self.inner.remove_elem_by_id(elem_id); + } + } + + // if cfg!(debug_assertions) { + // self.inner.check_consistency(); + // let mut end_value = start_value.clone(); + // end_value.apply_diff_shallow(&[Diff::List(event.clone())]); + // let cur_value = self.get_value(); + // assert_eq!( + // end_value, cur_value, + // "start_value={:#?} event={:#?} new_state={:#?} but the end_value={:#?}", + // start_value, event, cur_value, end_value + // ); + // self.check_get_child_index_correctly(); + // } + + Diff::List(event) + } + + // How we apply the diff is coupled with the [DiffMode] we used to calculate the diff. + // So be careful when you modify this function. + fn apply_diff(&mut self, diff: InternalDiff, ctx: DiffApplyContext) -> LoroResult<()> { + let _ = self.apply_diff_and_convert(diff, ctx); + Ok(()) + } + + #[instrument(skip_all)] + fn apply_local_op(&mut self, op: &RawOp, _: &Op) -> LoroResult { + let mut ans: ApplyLocalOpReturn = Default::default(); + match op.content.as_list().unwrap() { + ListOp::Insert { slice, pos } => match slice { + ListSlice::RawData(list) => { + for (i, x) in list.as_ref().iter().enumerate() { + let elem_id = op.idlp().inc(i as i32).try_into().unwrap(); + let pos_id = op.id_full().inc(i as i32); + self.inner.insert_list_item(*pos + i, pos_id); + let _ = self.create_new_elem( + elem_id, + pos_id.idlp(), + x.clone(), + elem_id.to_id(), + ); + } + } + _ => unreachable!(), + }, + ListOp::Delete(span) => { + self.inner + .list_drain(span.start() as usize..span.end() as usize, |_, elem| { + if let Some(c) = elem.value.as_container() { + ans.deleted_containers.push(c.clone()); + } + }); + } + ListOp::Move { from, to, elem_id } => { + self.mov( + *from as usize, + *to as usize, + *elem_id, + op.id_full(), + IndexType::ForOp, + ); + } + ListOp::Set { elem_id, value } => { + let old_value = + self.inner + .update_value(elem_id.compact(), value.clone(), op.idlp()); + if let Some(LoroValue::Container(c)) = old_value { + ans.deleted_containers.push(c); + } + } + ListOp::StyleStart { .. } | ListOp::StyleEnd => unreachable!(), + } + + Ok(ans) + } + + fn to_diff(&mut self, doc: &Weak) -> Diff { + let doc = &doc.upgrade().unwrap(); + Diff::List( + DeltaRopeBuilder::new() + .insert_many( + self.to_vec() + .into_iter() + .map(|v| ValueOrHandler::from_value(v, doc)), + Default::default(), + ) + .build(), + ) + } + + fn get_value(&mut self) -> LoroValue { + let list = self.get_value_inner(); + LoroValue::List(list.into()) + } + + /// Get the index of the child container + fn get_child_index(&self, id: &ContainerID) -> Option { + self.inner + .get_child_index(id, IndexType::ForUser) + .map(Index::Seq) + } + + fn contains_child(&self, id: &ContainerID) -> bool { + self.inner.contains_child_container(id) + } + + #[allow(unused)] + fn get_child_containers(&self) -> Vec { + self.inner + .child_container_to_elem() + .iter() + .filter_map(|(c, elem_id)| { + let elem = self.elements().get(elem_id)?; + if elem.value.as_container() != Some(c) { + None + } else { + Some(c.clone()) + } + }) + .collect_vec() + } + + fn fork(&self, _config: &Configure) -> Self { + self.clone() + } +} + +#[columnar(vec, ser, de, iterable)] +#[derive(Debug, Clone)] +struct EncodedId { + #[columnar(strategy = "DeltaRle")] + peer_idx: usize, + #[columnar(strategy = "DeltaRle")] + lamport: u32, +} + +#[columnar(vec, ser, de, iterable)] +#[derive(Debug, Clone, Copy)] +struct EncodedItemForFastSnapshot { + #[columnar(strategy = "DeltaRle")] + invisible_list_item: usize, + #[columnar(strategy = "BoolRle")] + pos_id_eq_elem_id: bool, + #[columnar(strategy = "BoolRle")] + elem_id_eq_last_set_id: bool, +} + +#[columnar(vec, ser, de, iterable)] +#[derive(Debug, Clone)] +struct EncodedIdFull { + #[columnar(strategy = "DeltaRle")] + peer_idx: usize, + #[columnar(strategy = "DeltaRle")] + counter: i32, + #[columnar(strategy = "DeltaRle")] + lamport_sub_counter: i32, +} + +#[columnar(ser, de)] +struct EncodedFastSnapshot { + #[columnar(class = "vec", iter = "EncodedItemForFastSnapshot")] + items: Vec, + #[columnar(class = "vec", iter = "EncodedIdFull")] + list_item_ids: Vec, + #[columnar(class = "vec", iter = "EncodedId")] + elem_ids: Vec, + #[columnar(class = "vec", iter = "EncodedId")] + last_set_ids: Vec, +} + +mod snapshot { + use loro_common::{IdFull, IdLp, LoroValue, PeerID}; + + use crate::{ + encoding::value_register::ValueRegister, + state::{ + decode_lamport_from_delta, decode_peer_from_table, decode_peer_table, + state_decode_error, ContainerCreationContext, ContainerState, FastStateSnapshot, + }, + }; + + use super::{ + inner::PushElemInfo, EncodedFastSnapshot, EncodedId, EncodedIdFull, + EncodedItemForFastSnapshot, MovableListState, + }; + + impl FastStateSnapshot for MovableListState { + /// Encodes the MovableListState into a compact binary format for fast snapshot storage and retrieval. + /// + /// The encoding format consists of: + /// 1. The full value of the MovableListState, encoded using postcard serialization. + /// 2. A series of EncodedItemForFastSnapshot structs representing each visible list item: + /// - invisible_list_item: Count of invisible items before this item (RLE encoded) + /// - pos_id_eq_elem_id: Boolean indicating if position ID equals element ID (RLE encoded) + /// - elem_id_eq_last_set_id: Boolean indicating if element ID equals last set ID (RLE encoded) + /// 3. A series of EncodedIdFull structs for list item IDs: + /// - peer_idx: Index of the peer ID in a value register (delta-RLE encoded) + /// - counter: Operation counter (delta-RLE encoded) + /// - lamport: Lamport timestamp (delta-RLE encoded) + /// 4. EncodedId structs for element IDs (when different from position ID) + /// 5. EncodedId structs for last set IDs (when different from element ID) + /// 6. A list of unique peer IDs used in the encoding + fn encode_snapshot_fast(&mut self, mut w: W) { + let value = self.get_value().into_list().unwrap(); + postcard::to_io(&*value, &mut w).unwrap(); + let mut peers: ValueRegister = ValueRegister::new(); + let len = self.len(); + let mut items = Vec::with_capacity(len); + // starts with a sentinel value. The num of `invisible_list_item` may be updated later + items.push(EncodedItemForFastSnapshot { + pos_id_eq_elem_id: true, + invisible_list_item: 0, + elem_id_eq_last_set_id: true, + }); + + let mut list_item_ids = Vec::with_capacity(self.len()); + let mut elem_ids = Vec::new(); + let mut last_set_ids = Vec::new(); + for item in self.list().iter() { + if let Some(elem_id) = item.pointed_by { + let elem = self.elements().get(&elem_id).unwrap(); + let elem_eq_list_item = elem_id.to_id() == item.id.idlp(); + let elem_id_eq_last_set_id = elem.value_id.compact() == elem_id; + items.push(EncodedItemForFastSnapshot { + invisible_list_item: 0, + pos_id_eq_elem_id: elem_eq_list_item, + elem_id_eq_last_set_id, + }); + + list_item_ids.push(super::EncodedIdFull { + peer_idx: peers.register(&item.id.peer), + counter: item.id.counter, + lamport_sub_counter: (item.id.lamport as i32 - item.id.counter), + }); + if !elem_eq_list_item { + elem_ids.push(super::EncodedId { + peer_idx: peers.register(&elem_id.peer), + lamport: elem_id.lamport.get(), + }) + } + if !elem_id_eq_last_set_id { + last_set_ids.push(super::EncodedId { + peer_idx: peers.register(&elem.value_id.peer), + lamport: elem.value_id.lamport, + }) + } + } else { + items.last_mut().unwrap().invisible_list_item += 1; + list_item_ids.push(super::EncodedIdFull { + peer_idx: peers.register(&item.id.peer), + counter: item.id.counter, + lamport_sub_counter: (item.id.lamport as i32 - item.id.counter), + }); + } + } + + let peers = peers.unwrap_vec(); + leb128::write::unsigned(&mut w, peers.len() as u64).unwrap(); + for peer in peers { + w.write_all(&peer.to_le_bytes()).unwrap(); + } + + let v = serde_columnar::to_vec(&EncodedFastSnapshot { + items, + list_item_ids, + elem_ids, + last_set_ids, + }) + .unwrap(); + w.write_all(&v).unwrap(); + } + + fn decode_value(bytes: &[u8]) -> loro_common::LoroResult<(loro_common::LoroValue, &[u8])> { + let (list_value, bytes) = + postcard::take_from_bytes::>(bytes).map_err(|_| { + loro_common::LoroError::DecodeError( + "Decode list value failed".to_string().into_boxed_str(), + ) + })?; + Ok((list_value.into(), bytes)) + } + + fn decode_snapshot_fast( + idx: crate::container::idx::ContainerIdx, + (list_value, mut bytes): (loro_common::LoroValue, &[u8]), + _ctx: ContainerCreationContext, + ) -> loro_common::LoroResult + where + Self: Sized, + { + let peers = decode_peer_table(&mut bytes, "Decode movable list state failed")?; + + let mut ans = MovableListState::new(idx); + + let iters = + serde_columnar::iter_from_bytes::(bytes).map_err(|err| { + state_decode_error(format!( + "Decode movable list state failed: invalid metadata: {err}" + )) + })?; + let mut elem_iter = iters.elem_ids; + let item_iter = iters.items; + let mut list_item_id_iter = iters.list_item_ids; + let mut last_set_id_iter = iters.last_set_ids; + let mut is_first = true; + + let list_value = list_value.into_list().map_err(|_| { + state_decode_error("Decode movable list state failed: value is not a list") + })?; + let mut list_value_iter = list_value.iter(); + for item in item_iter { + let EncodedItemForFastSnapshot { + invisible_list_item, + pos_id_eq_elem_id, + elem_id_eq_last_set_id, + } = item.map_err(|err| { + state_decode_error(format!( + "Decode movable list state failed: invalid item: {err}" + )) + })?; + + if !is_first { + let EncodedIdFull { + peer_idx, + counter, + lamport_sub_counter, + } = list_item_id_iter + .next() + .ok_or_else(|| { + state_decode_error( + "Decode movable list state failed: missing list item id", + ) + })? + .map_err(|err| { + state_decode_error(format!( + "Decode movable list state failed: invalid list item id: {err}" + )) + })?; + let peer = decode_peer_from_table( + &peers, + peer_idx, + "Decode movable list state failed", + )?; + let id_full = IdFull::new( + peer, + counter, + decode_lamport_from_delta( + counter, + lamport_sub_counter, + "Decode movable list state failed", + )?, + ); + let elem_id = if pos_id_eq_elem_id { + id_full.idlp() + } else { + let EncodedId { peer_idx, lamport } = elem_iter + .next() + .ok_or_else(|| { + state_decode_error( + "Decode movable list state failed: missing element id", + ) + })? + .map_err(|err| { + state_decode_error(format!( + "Decode movable list state failed: invalid element id: {err}" + )) + })?; + IdLp::new( + decode_peer_from_table( + &peers, + peer_idx, + "Decode movable list state failed", + )?, + lamport, + ) + }; + + let last_set_id = if elem_id_eq_last_set_id { + elem_id + } else { + let EncodedId { peer_idx, lamport } = last_set_id_iter + .next() + .ok_or_else(|| { + state_decode_error( + "Decode movable list state failed: missing last set id", + ) + })? + .map_err(|err| { + state_decode_error(format!( + "Decode movable list state failed: invalid last set id: {err}" + )) + })?; + IdLp::new( + decode_peer_from_table( + &peers, + peer_idx, + "Decode movable list state failed", + )?, + lamport, + ) + }; + + let value = list_value_iter.next().ok_or_else(|| { + state_decode_error("Decode movable list state failed: missing list value") + })?; + ans.inner.push_inner( + id_full, + Some(PushElemInfo { + elem_id: elem_id.compact(), + value: value.clone(), + last_set_id, + }), + ) + } + + is_first = false; + for _ in 0..invisible_list_item { + let EncodedIdFull { + peer_idx, + counter, + lamport_sub_counter, + } = list_item_id_iter + .next() + .ok_or_else(|| { + state_decode_error( + "Decode movable list state failed: missing invisible list item id", + ) + })? + .map_err(|err| { + state_decode_error(format!( + "Decode movable list state failed: invalid invisible list item id: {err}" + )) + })?; + let peer = decode_peer_from_table( + &peers, + peer_idx, + "Decode movable list state failed", + )?; + let id_full = IdFull::new( + peer, + counter, + decode_lamport_from_delta( + counter, + lamport_sub_counter, + "Decode movable list state failed", + )?, + ); + ans.inner.push_inner(id_full, None); + } + } + + if is_first { + return Err(state_decode_error( + "Decode movable list state failed: missing sentinel item", + )); + } + if elem_iter + .next() + .transpose() + .map_err(|err| { + state_decode_error(format!( + "Decode movable list state failed: invalid extra element id: {err}" + )) + })? + .is_some() + { + return Err(state_decode_error( + "Decode movable list state failed: unused element id", + )); + } + if list_item_id_iter + .next() + .transpose() + .map_err(|err| { + state_decode_error(format!( + "Decode movable list state failed: invalid extra list item id: {err}" + )) + })? + .is_some() + { + return Err(state_decode_error( + "Decode movable list state failed: unused list item id", + )); + } + if last_set_id_iter + .next() + .transpose() + .map_err(|err| { + state_decode_error(format!( + "Decode movable list state failed: invalid extra last set id: {err}" + )) + })? + .is_some() + { + return Err(state_decode_error( + "Decode movable list state failed: unused last set id", + )); + } + if list_value_iter.next().is_some() { + return Err(state_decode_error( + "Decode movable list state failed: unused list value", + )); + } + + Ok(ans) + } + } + + #[cfg(test)] + mod test { + + use loro_common::{CompactIdLp, ContainerID, LoroValue, ID}; + + use crate::container::idx::ContainerIdx; + + use super::*; + + #[test] + fn test_movable_list_snapshot() { + let mut list = MovableListState::new(ContainerIdx::from_index_and_type( + 0, + loro_common::ContainerType::MovableList, + )); + + list.inner.insert_list_item(0, IdFull::new(9, 9, 9)); + list.inner.insert_list_item(1, IdFull::new(0, 0, 0)); + let _ = list.create_new_elem( + CompactIdLp::new(10, 10), + IdLp { + lamport: 0, + peer: 0, + }, + LoroValue::Container(ContainerID::new_normal( + ID::new(10, 10), + loro_common::ContainerType::Text, + )), + IdLp { + lamport: 1, + peer: 2, + }, + ); + list.inner.insert_list_item(2, IdFull::new(1, 1, 1)); + list.inner.insert_list_item(3, IdFull::new(2, 2, 2)); + list.inner.insert_list_item(4, IdFull::new(3, 3, 3)); + let _ = list.create_new_elem( + CompactIdLp::new(3, 8), + IdLp { + lamport: 3, + peer: 3, + }, + LoroValue::String("abc".into()), + IdLp { + lamport: 4, + peer: 5, + }, + ); + + let mut bytes = Vec::new(); + list.encode_snapshot_fast(&mut bytes); + assert!(bytes.len() <= 117, "{}", bytes.len()); + + let (v, bytes) = MovableListState::decode_value(&bytes).unwrap(); + assert_eq!( + v, + vec![ + LoroValue::Container(ContainerID::new_normal( + ID::new(10, 10), + loro_common::ContainerType::Text, + )), + LoroValue::String("abc".into()), + ] + .into() + ); + let mut list2 = MovableListState::decode_snapshot_fast( + ContainerIdx::from_index_and_type(0, loro_common::ContainerType::MovableList), + (v.clone(), bytes), + ContainerCreationContext { + configure: &Default::default(), + peer: 0, + }, + ) + .unwrap(); + assert_eq!(&list2.get_value(), &v); + assert_eq!(&list.inner, &list2.inner); + } + + #[test] + fn test_movable_list_snapshot_size() { + let mut list = MovableListState::new(ContainerIdx::from_index_and_type( + 0, + loro_common::ContainerType::MovableList, + )); + + list.inner.insert_list_item(0, IdFull::new(0, 0, 0)); + let _ = list.create_new_elem( + CompactIdLp::new(0, 0), + IdLp { + lamport: 0, + peer: 0, + }, + LoroValue::I64(0), + IdLp { + lamport: 0, + peer: 0, + }, + ); + + list.inner.insert_list_item(1, IdFull::new(0, 1, 1)); + let _ = list.create_new_elem( + CompactIdLp::new(0, 1), + IdLp { + peer: 0, + lamport: 1, + }, + LoroValue::I64(0), + IdLp { + peer: 0, + lamport: 1, + }, + ); + + let mut bytes = Vec::new(); + list.encode_snapshot_fast(&mut bytes); + assert!(bytes.len() <= 42, "{}", bytes.len()); + + list.inner.insert_list_item(2, IdFull::new(0, 1, 2)); + let _ = list.create_new_elem( + CompactIdLp::new(0, 2), + IdLp { + peer: 0, + lamport: 2, + }, + LoroValue::I64(0), + IdLp { + peer: 0, + lamport: 2, + }, + ); + let mut bytes = Vec::new(); + list.encode_snapshot_fast(&mut bytes); + assert!(bytes.len() <= 47, "{}", bytes.len()); + } + + #[test] + fn movable_list_fast_snapshot_rejects_corrupt_state_metadata() { + let idx = ContainerIdx::from_index_and_type(0, loro_common::ContainerType::MovableList); + let configure = Default::default(); + let ctx = ContainerCreationContext { + configure: &configure, + peer: 0, + }; + + assert!(MovableListState::decode_snapshot_fast( + idx, + (Vec::::new().into(), &[1]), + ctx, + ) + .is_err()); + + let mut list = MovableListState::new(idx); + let mut bytes = Vec::new(); + list.encode_snapshot_fast(&mut bytes); + let (_, state_bytes) = MovableListState::decode_value(&bytes).unwrap(); + assert!(MovableListState::decode_snapshot_fast( + idx, + (vec![LoroValue::I64(1)].into(), state_bytes), + ctx, + ) + .is_err()); + } + } +} + +#[cfg(test)] +mod test { + use crate::{loro::ExportMode, HandlerTrait, LoroDoc, ToJson}; + use serde_json::json; + + #[test] + fn basic_handler_ops() { + let doc = LoroDoc::new_auto_commit(); + let list = doc.get_movable_list("list"); + list.insert(0, 0).unwrap(); + list.insert(1, 1).unwrap(); + list.insert(2, 2).unwrap(); + assert_eq!(list.get_value().to_json_value(), json!([0, 1, 2])); + list.mov(0, 1).unwrap(); + assert_eq!(list.get_value().to_json_value(), json!([1, 0, 2])); + list.mov(2, 0).unwrap(); + assert_eq!(list.get_value().to_json_value(), json!([2, 1, 0])); + list.delete(0, 2).unwrap(); + assert_eq!(list.get_value().to_json_value(), json!([0])); + list.insert(0, 9).unwrap(); + assert_eq!(list.get_value().to_json_value(), json!([9, 0])); + list.delete(0, 2).unwrap(); + assert_eq!(list.get_value().to_json_value(), json!([])); + } + + #[test] + fn basic_sync() { + let doc = LoroDoc::new_auto_commit(); + let list = doc.get_movable_list("list"); + list.insert(0, 1).unwrap(); + list.insert(1, 0).unwrap(); + list.mov(0, 1).unwrap(); + { + let doc_b = LoroDoc::new_auto_commit(); + doc_b + .import(&doc.export(ExportMode::all_updates()).unwrap()) + .unwrap(); + assert_eq!( + doc_b.get_deep_value().to_json_value(), + json!({ + "list": [0, 1] + }) + ); + } + list.mov(1, 0).unwrap(); + assert_eq!( + doc.get_deep_value().to_json_value(), + json!({ + "list": [1, 0] + }) + ); + { + let doc_b = LoroDoc::new_auto_commit(); + doc_b + .import(&doc.export(ExportMode::all_updates()).unwrap()) + .unwrap(); + assert_eq!( + doc_b.get_deep_value().to_json_value(), + json!({ + "list": [1, 0] + }) + ); + } + list.mov(0, 1).unwrap(); + list.insert(2, 3).unwrap(); + list.set(2, 2).unwrap(); + assert_eq!( + doc.get_deep_value().to_json_value(), + json!({ + "list": [0, 1, 2] + }) + ); + { + let doc_b = LoroDoc::new_auto_commit(); + doc_b + .import(&doc.export(ExportMode::all_updates()).unwrap()) + .unwrap(); + assert_eq!( + doc_b.get_deep_value().to_json_value(), + json!({ + "list": [0, 1, 2] + }) + ); + } + { + let doc_b = LoroDoc::new_auto_commit(); + doc_b + .import(&doc.export(ExportMode::Snapshot).unwrap()) + .unwrap(); + assert_eq!( + doc_b.get_deep_value().to_json_value(), + json!({ + "list": [0, 1, 2] + }) + ); + } + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/state/richtext_state.rs b/3rdparty/loro/crates/loro-internal/src/state/richtext_state.rs new file mode 100644 index 0000000000..fac441bde4 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/state/richtext_state.rs @@ -0,0 +1,1508 @@ +use generic_btree::{rle::HasLength, rle::Sliceable as _, Cursor}; +use loro_common::{ContainerID, InternalString, LoroError, LoroResult, LoroValue, ID}; +use loro_delta::{DeltaRope, DeltaRopeBuilder}; +use rustc_hash::{FxHashMap, FxHashSet}; +use smallvec::SmallVec; +use std::ops::Range; +use std::sync::{Arc, Weak}; + +use crate::{ + container::{ + idx::ContainerIdx, + list::list_op, + richtext::{ + config::StyleConfigMap, + richtext_state::{ + DrainInfo, EntityRanges, IterRangeItem, PosType, RichtextStateChunk, + }, + AnchorType, RichtextState as InnerState, StyleKey, StyleOp, Styles, + }, + }, + delta::{StyleMeta, StyleMetaItem}, + event::{Diff, Index, InternalDiff, TextDiff}, + handler::TextDelta, + op::{Op, RawOp}, + sync::RwLock, + utils::{lazy::LazyLoad, string_slice::StringSlice}, + LoroDocInner, +}; + +use super::{ApplyLocalOpReturn, ContainerState, DiffApplyContext}; + +#[derive(Debug)] +pub struct RichtextState { + idx: ContainerIdx, + config: Arc>, + state: LazyLoad, + /// This is used to indicate whether the richtext state is changed, so the downstream has an easy way to cache + /// NOTE: We need to ensure the invariance that the version id is always increased when the richtext state is changed + version_id: usize, +} + +struct Pos { + entity_index: usize, + event_index: usize, +} + +impl RichtextState { + #[inline] + pub fn new(idx: ContainerIdx, config: Arc>) -> Self { + Self { + idx, + config, + state: LazyLoad::Src(Default::default()), + version_id: 0, + } + } + + #[inline] + fn update_version(&mut self) { + self.version_id = self.version_id.wrapping_add(1); + } + + /// Get the version id of the richtext + /// + /// This can be used to detect whether the richtext is changed + #[inline] + pub fn get_version_id(&self) -> usize { + self.version_id + } + + /// Get the text content of the richtext + /// + /// This uses `mut` because we may need to build the state from snapshot + #[inline] + pub fn to_string_mut(&mut self) -> String { + self.state.get_mut().to_string() + } + + #[allow(unused)] + #[inline(always)] + pub(crate) fn is_empty(&self) -> bool { + match &self.state { + LazyLoad::Src(s) => s.elements.is_empty(), + LazyLoad::Dst(d) => d.is_empty(), + } + } + + pub(crate) fn diagnose(&self) { + match &self.state { + LazyLoad::Src(_) => {} + LazyLoad::Dst(d) => d.diagnose(), + } + } + + #[allow(unused)] + pub(crate) fn get_text_slice_by_event_index( + &mut self, + pos: usize, + len: usize, + ) -> LoroResult { + self.state.get_mut().get_text_slice_by_event_index(pos, len) + } + + pub(crate) fn slice_delta( + &mut self, + start_index: usize, + end_index: usize, + pos_type: PosType, + ) -> LoroResult> { + self.state + .get_mut() + .slice_delta(start_index, end_index, pos_type) + } + + #[allow(unused)] + pub(crate) fn get_char_by_event_index(&mut self, pos: usize) -> Result { + self.state.get_mut().get_char_by_event_index(pos) + } + + pub(crate) fn iter(&mut self, mut callback: impl FnMut(&str) -> bool) { + for span in self.state.get_mut().iter_chunk() { + match span { + RichtextStateChunk::Text(text_chunk) => { + if !callback(text_chunk.as_str()) { + return; + } + } + RichtextStateChunk::Style { .. } => {} + } + } + } + + pub(crate) fn iter_raw(&self, callback: &mut dyn FnMut(&RichtextStateChunk)) { + let iter: &mut dyn Iterator; + let mut a; + let mut b; + match &self.state { + LazyLoad::Src(s) => { + a = Some(s.elements.iter()); + iter = &mut *a.as_mut().unwrap(); + } + LazyLoad::Dst(s) => { + b = Some(s.iter_chunk()); + iter = &mut *b.as_mut().unwrap(); + } + } + + for c in iter { + callback(c); + } + } + + fn get_style_start( + &mut self, + style_starts: &mut FxHashMap, Pos>, + style: &Arc, + ) -> Pos { + self.update_version(); + match style_starts.remove(style) { + Some(x) => x, + None => { + // this should happen rarely, so it should be fine to scan + let mut pos = Pos { + entity_index: 0, + event_index: 0, + }; + + for c in self.state.get_mut().iter_chunk() { + match c { + RichtextStateChunk::Style { + style: s, + anchor_type: AnchorType::Start, + } if style == s => { + break; + } + RichtextStateChunk::Text(t) => { + pos.entity_index += t.unicode_len() as usize; + pos.event_index += t.event_len() as usize; + } + RichtextStateChunk::Style { .. } => { + pos.entity_index += 1; + } + } + } + pos + } + } + } + + pub fn get_index_of_id(&self, id: ID) -> Option { + let iter: &mut dyn Iterator; + let mut a; + let mut b; + match &self.state { + LazyLoad::Src(s) => { + a = Some(s.elements.iter()); + iter = &mut *a.as_mut().unwrap(); + } + LazyLoad::Dst(s) => { + b = Some(s.iter_chunk()); + iter = &mut *b.as_mut().unwrap(); + } + } + + let mut index = 0; + for elem in iter { + let span = elem.get_id_span(); + if span.contains(id) { + return Some(index + (id.counter - span.counter.start) as usize); + } + + index += elem.rle_len(); + } + + None + } + + pub fn get_text_index_of_id(&self, id: ID, use_event_index: bool) -> Option { + let iter: &mut dyn Iterator; + let mut a; + let mut b; + match &self.state { + LazyLoad::Src(s) => { + a = Some(s.elements.iter()); + iter = &mut *a.as_mut().unwrap(); + } + LazyLoad::Dst(s) => { + b = Some(s.iter_chunk()); + iter = &mut *b.as_mut().unwrap(); + } + } + + let mut index = 0; + for elem in iter { + let span = elem.get_id_span(); + if span.contains(id) { + match elem { + RichtextStateChunk::Text(t) => { + if use_event_index { + let event_offset = t.convert_unicode_offset_to_event_offset( + (id.counter - span.counter.start) as usize, + ); + return Some(index + event_offset); + } else { + return Some(index + (id.counter - span.counter.start) as usize); + } + } + RichtextStateChunk::Style { .. } => { + return Some(index); + } + } + } + + index += match elem { + RichtextStateChunk::Text(t) => { + if use_event_index { + t.event_len() as usize + } else { + t.unicode_len() as usize + } + } + RichtextStateChunk::Style { .. } => 0, + }; + } + + None + } + + pub(crate) fn get_delta(&mut self) -> Vec { + let mut delta = Vec::new(); + for span in self.state.get_mut().iter() { + let next_attr = span.attributes.to_option_map(); + match delta.last_mut() { + Some(TextDelta::Insert { insert, attributes }) if &next_attr == attributes => { + insert.push_str(span.text.as_str()); + continue; + } + _ => {} + } + delta.push(TextDelta::Insert { + insert: span.text.as_str().to_string(), + attributes: next_attr, + }) + } + delta + } +} + +#[cfg(test)] +mod tests { + use crate::{container::richtext::StyleKey, cursor::PosType, handler::HandlerTrait, LoroDoc}; + + #[test] + fn has_style_key_in_entity_range_basic() { + let loro = LoroDoc::new_auto_commit(); + let text = loro.get_text("text"); + text.insert(0, "abcdef", PosType::Unicode).unwrap(); + text.mark(1, 3, "bold", true.into(), PosType::Unicode) + .unwrap(); + + let bold_key = StyleKey::Key("bold".into()); + let has_style = text + .with_state(|state| { + let st = state.as_richtext_state_mut().unwrap(); + let (entity_range, _) = + st.get_entity_range_and_styles_at_range(1..3, PosType::Unicode); + Ok(st.has_style_key_in_entity_range(entity_range, &bold_key)) + }) + .unwrap(); + assert!(has_style); + + let missing = text + .with_state(|state| { + let st = state.as_richtext_state_mut().unwrap(); + let (entity_range, _) = + st.get_entity_range_and_styles_at_range(4..5, PosType::Unicode); + Ok(st.has_style_key_in_entity_range(entity_range, &bold_key)) + }) + .unwrap(); + assert!(!missing); + } + + #[test] + fn has_style_key_in_entity_range_spans_elements() { + let loro = LoroDoc::new_auto_commit(); + let text = loro.get_text("text"); + text.insert(0, "abcdefgh", PosType::Unicode).unwrap(); + text.mark(0, 2, "bold", true.into(), PosType::Unicode) + .unwrap(); + text.mark(3, 5, "bold", true.into(), PosType::Unicode) + .unwrap(); + + let bold_key = StyleKey::Key("bold".into()); + + let has_style_across_segments = text + .with_state(|state| { + let st = state.as_richtext_state_mut().unwrap(); + let (entity_range, _) = + st.get_entity_range_and_styles_at_range(0..5, PosType::Unicode); + Ok(st.has_style_key_in_entity_range(entity_range, &bold_key)) + }) + .unwrap(); + assert!(has_style_across_segments); + + let gap_has_style = text + .with_state(|state| { + let st = state.as_richtext_state_mut().unwrap(); + let (entity_range, _) = + st.get_entity_range_and_styles_at_range(6..7, PosType::Unicode); + Ok(st.has_style_key_in_entity_range(entity_range, &bold_key)) + }) + .unwrap(); + assert!(!gap_has_style); + } +} + +impl Clone for RichtextState { + fn clone(&self) -> Self { + Self { + idx: self.idx, + config: self.config.clone(), + state: self.state.clone(), + version_id: self.version_id, + } + } +} + +impl ContainerState for RichtextState { + fn container_idx(&self) -> ContainerIdx { + self.idx + } + + fn is_state_empty(&self) -> bool { + match &self.state { + LazyLoad::Src(s) => s.is_empty(), + LazyLoad::Dst(s) => s.is_empty(), + } + } + + // TODO: refactor + fn apply_diff_and_convert(&mut self, diff: InternalDiff, _ctx: DiffApplyContext) -> Diff { + self.update_version(); + let InternalDiff::RichtextRaw(richtext) = diff else { + unreachable!() + }; + + // tracing::info!("Self state = {:#?}", &self); + // PERF: compose delta + let mut ans: TextDiff = TextDiff::new(); + let mut style_delta: TextDiff = TextDiff::new(); + let mut style_starts: FxHashMap, Pos> = FxHashMap::default(); + let mut entity_index = 0; + let mut event_index = 0; + let mut new_style_deltas: Vec = Vec::new(); + for span in richtext.iter() { + match span { + loro_delta::DeltaItem::Retain { len, .. } => { + entity_index += len; + } + loro_delta::DeltaItem::Replace { value, delete, .. } => { + if *delete > 0 { + // Deletions + let mut deleted_style_keys: FxHashSet = + FxHashSet::default(); + let DrainInfo { + start_event_index: start, + end_event_index: end, + affected_style_range, + } = self.state.get_mut().drain_by_entity_index( + entity_index, + *delete, + Some(&mut |c| match c { + RichtextStateChunk::Style { + style, + anchor_type: AnchorType::Start, + } => { + deleted_style_keys.insert(style.key.clone()); + } + RichtextStateChunk::Style { + style, + anchor_type: AnchorType::End, + } => { + deleted_style_keys.insert(style.key.clone()); + } + _ => {} + }), + ); + + if start > event_index { + ans.push_retain(start - event_index, Default::default()); + event_index = start; + } + + if let Some((entity_range, event_range)) = affected_style_range { + let mut delta: TextDiff = DeltaRopeBuilder::new() + .retain(event_range.start, Default::default()) + .build(); + let mut entity_len_sum = 0; + let expected_sum = entity_range.len(); + + for IterRangeItem { + event_len, + chunk, + styles, + entity_len, + .. + } in self.state.get_mut().iter_range(entity_range) + { + entity_len_sum += entity_len; + match chunk { + RichtextStateChunk::Text(_) => { + let mut style_meta: StyleMeta = styles.into(); + for key in deleted_style_keys.iter() { + if !style_meta.contains_key(key) { + style_meta.insert( + key.clone(), + StyleMetaItem { + lamport: 0, + peer: 0, + value: LoroValue::Null, + }, + ) + } + } + delta.push_retain( + event_len, + style_meta.to_option_map().unwrap_or_default().into(), + ); + } + RichtextStateChunk::Style { .. } => {} + } + } + + debug_assert_eq!(entity_len_sum, expected_sum); + delta.chop(); + style_delta.compose(&delta); + } + + ans.push_delete(end - start); + } + + if value.rle_len() > 0 { + // Insertions + match value { + RichtextStateChunk::Text(s) => { + let (pos, styles) = + self.state.get_mut().insert_elem_at_entity_index( + entity_index, + RichtextStateChunk::Text(s.clone()), + ); + // PERF: this can be optimized + let insert_styles = Into::::into(styles.clone()) + .to_option_map() + .unwrap_or_default(); + + if pos > event_index { + ans.push_retain(pos - event_index, Default::default()); + } + event_index = pos + s.event_len() as usize; + ans.push_insert( + StringSlice::from(s.bytes().clone()), + insert_styles.into(), + ); + } + RichtextStateChunk::Style { anchor_type, style } => { + let (new_event_index, _) = + self.state.get_mut().insert_elem_at_entity_index( + entity_index, + RichtextStateChunk::Style { + style: style.clone(), + anchor_type: *anchor_type, + }, + ); + + if new_event_index > event_index { + ans.push_retain( + new_event_index - event_index, + Default::default(), + ); + // inserting style anchor will not affect event_index's positions + event_index = new_event_index; + } + + match anchor_type { + AnchorType::Start => { + style_starts.insert( + style.clone(), + Pos { + entity_index, + event_index: new_event_index, + }, + ); + } + AnchorType::End => { + // get the pair of style anchor. now we can annotate the range + let Pos { + entity_index: start_entity_index, + event_index: start_event_index, + } = self.get_style_start(&mut style_starts, style); + let mut delta: TextDiff = DeltaRopeBuilder::new() + .retain(start_event_index, Default::default()) + .build(); + // we need to + 1 because we also need to annotate the end anchor + let event = + self.state.get_mut().annotate_style_range_with_event( + start_entity_index..entity_index + 1, + style.clone(), + ); + for (s, l) in event { + delta.push_retain( + l, + s.to_option_map().unwrap_or_default().into(), + ); + } + + delta.chop(); + new_style_deltas.push(delta); + } + } + } + } + + entity_index += value.rle_len(); + } + } + } + } + + for s in new_style_deltas { + style_delta.compose(&s); + } + // self.check_consistency_between_content_and_style_ranges(); + ans.compose(&style_delta); + Diff::Text(ans) + } + + fn apply_diff(&mut self, diff: InternalDiff, _ctx: DiffApplyContext) -> LoroResult<()> { + self.update_version(); + let InternalDiff::RichtextRaw(richtext) = diff else { + unreachable!() + }; + + if let LazyLoad::Src(loader) = &mut self.state { + if loader.try_apply_append_delta(&richtext) { + return Ok(()); + } + } + + // Fast path for plain-text deltas (no style anchors / style ranges). + // + // Rebuilding avoids repeated BTree queries and mutations when the delta is very "choppy" + // (many small edit spans), but it allocates and clones chunks, so it can be slower for + // small deltas. Use a cheap cost model to enable it only when it's likely beneficial. + let should_fast_apply = { + #[inline] + fn ilog2_ceil(x: usize) -> usize { + debug_assert!(x > 0); + (usize::BITS - (x - 1).leading_zeros()) as usize + } + + let state = self.state.get_mut(); + if state.has_styles() { + false + } else { + // `edit_actions` approximates how many BTree mutations the incremental path will do: + // each Replace with delete>0 becomes a drain, and each Replace with value>0 becomes an insert. + let mut edit_actions: usize = 0; + let mut is_plain_text_delta = true; + for span in richtext.iter() { + match span { + loro_delta::DeltaItem::Retain { .. } => {} + loro_delta::DeltaItem::Replace { value, delete, .. } => { + if *delete > 0 { + edit_actions += 1; + } + if value.rle_len() > 0 { + if !matches!(value, RichtextStateChunk::Text(_)) { + is_plain_text_delta = false; + break; + } + edit_actions += 1; + } + } + } + } + + if !is_plain_text_delta || edit_actions == 0 { + false + } else { + let content_nodes = state.content_node_len().max(1); + let log_n = ilog2_ceil(content_nodes + 1).max(1); + let incremental_score = edit_actions.saturating_mul(log_n); + let rebuild_score = content_nodes.saturating_add(edit_actions); + + let old_len = richtext.old_len().max(1); + let avg_action_span = old_len / edit_actions; + // A very rough proxy for "choppiness": many edit actions with small average span. + // The thresholds are intentionally conservative to avoid rebuilding for small or + // localized deltas. + let is_choppy = edit_actions >= 256 && avg_action_span <= 32; + + is_choppy && incremental_score >= rebuild_score.saturating_mul(4) + } + } + }; + + if should_fast_apply { + let new_state = { + let state = self.state.get_mut(); + let mut chunks: Vec = Vec::new(); + + let mut src_iter = state.iter_chunk(); + let mut cur = src_iter.next(); + let mut cur_offset: usize = 0; + + for span in richtext.iter() { + match span { + loro_delta::DeltaItem::Retain { len, .. } => { + let mut left = *len; + while left > 0 { + let chunk = cur.expect("Delta retain exceeds source length"); + let chunk_len = chunk.rle_len(); + if chunk_len == 0 { + cur = src_iter.next(); + cur_offset = 0; + continue; + } + let available = chunk_len - cur_offset; + let take = left.min(available); + if take == chunk_len && cur_offset == 0 { + chunks.push(chunk.clone()); + } else { + chunks.push(chunk.slice(cur_offset..cur_offset + take)); + } + + left -= take; + cur_offset += take; + if cur_offset == chunk_len { + cur = src_iter.next(); + cur_offset = 0; + } + } + } + loro_delta::DeltaItem::Replace { value, delete, .. } => { + let mut left = *delete; + while left > 0 { + let chunk = cur.expect("Delta delete exceeds source length"); + let chunk_len = chunk.rle_len(); + if chunk_len == 0 { + cur = src_iter.next(); + cur_offset = 0; + continue; + } + let available = chunk_len - cur_offset; + let take = left.min(available); + left -= take; + cur_offset += take; + if cur_offset == chunk_len { + cur = src_iter.next(); + cur_offset = 0; + } + } + + if value.rle_len() > 0 { + chunks.push(value.clone()); + } + } + } + } + + if let Some(chunk) = cur { + let chunk_len = chunk.rle_len(); + if cur_offset < chunk_len { + if cur_offset == 0 { + chunks.push(chunk.clone()); + } else { + chunks.push(chunk.slice(cur_offset..chunk_len)); + } + } + } + for chunk in src_iter { + chunks.push(chunk.clone()); + } + + InnerState::from_chunks(chunks.into_iter()) + }; + + *self.state.get_mut() = new_state; + return Ok(()); + } + + let mut style_starts: FxHashMap, usize> = FxHashMap::default(); + let mut entity_index = 0; + for span in richtext.iter() { + match span { + loro_delta::DeltaItem::Retain { len, .. } => { + entity_index += len; + } + loro_delta::DeltaItem::Replace { value, delete, .. } => { + if *delete > 0 { + // Deletions + self.state + .get_mut() + .drain_by_entity_index(entity_index, *delete, None); + } + if value.rle_len() > 0 { + // Insertions + match value { + RichtextStateChunk::Text(s) => { + self.state.get_mut().insert_elem_at_entity_index( + entity_index, + RichtextStateChunk::Text(s.clone()), + ); + } + RichtextStateChunk::Style { style, anchor_type } => { + self.state.get_mut().insert_elem_at_entity_index( + entity_index, + RichtextStateChunk::Style { + style: style.clone(), + anchor_type: *anchor_type, + }, + ); + + if *anchor_type == AnchorType::Start { + style_starts.insert(style.clone(), entity_index); + } else { + let start_pos = match style_starts.get(style) { + Some(x) => *x, + None => { + // This should be rare, so it should be fine to scan + let mut start_entity_index = 0; + for c in self.state.get_mut().iter_chunk() { + match c { + RichtextStateChunk::Style { + style: s, + anchor_type: AnchorType::Start, + } if style == s => { + break; + } + RichtextStateChunk::Text(t) => { + start_entity_index += + t.unicode_len() as usize; + } + RichtextStateChunk::Style { .. } => { + start_entity_index += 1; + } + } + } + start_entity_index + } + }; + // we need to + 1 because we also need to annotate the end anchor + self.state.get_mut().annotate_style_range( + start_pos..entity_index + 1, + style.clone(), + ); + } + } + } + entity_index += value.rle_len(); + } + } + } + } + + // self.check_consistency_between_content_and_style_ranges() + Ok(()) + } + + fn apply_local_op(&mut self, r_op: &RawOp, op: &Op) -> LoroResult { + self.update_version(); + match &op.content { + crate::op::InnerContent::List(l) => match l { + list_op::InnerListOp::Insert { slice: _, pos: _ } => { + unreachable!() + } + list_op::InnerListOp::InsertText { + slice, + unicode_len: _, + unicode_start: _, + pos, + } => { + self.state.get_mut().insert_at_entity_index( + *pos as usize, + slice.clone(), + r_op.id_full(), + ); + } + list_op::InnerListOp::Delete(del) => { + self.state.get_mut().drain_by_entity_index( + del.start() as usize, + rle::HasLength::atom_len(&del), + None, + ); + } + list_op::InnerListOp::StyleStart { + start, + end, + key, + value, + info, + } => { + // Behavior here is a little different from apply_diff. + // + // When apply_diff, we only do the mark when we have included both + // StyleStart and StyleEnd. + // + // When applying local op, we can do the mark when we have StyleStart. + // We can assume StyleStart and StyleEnd are always appear in a pair + // for apply_local_op. (Because for local behavior, when we mark, + // we always create a pair of style ops.) + self.state.get_mut().mark_with_entity_index( + *start as usize..*end as usize, + Arc::new(StyleOp { + lamport: r_op.lamport, + peer: r_op.id.peer, + cnt: r_op.id.counter, + key: key.clone(), + value: value.clone(), + info: *info, + }), + ); + } + list_op::InnerListOp::Set { .. } => {} + list_op::InnerListOp::StyleEnd => {} + list_op::InnerListOp::Move { .. } => unreachable!(), + }, + _ => unreachable!(), + } + + // self.check_consistency_between_content_and_style_ranges(); + Ok(Default::default()) + } + + fn to_diff(&mut self, _doc: &Weak) -> Diff { + let mut delta = TextDiff::new(); + for span in self.state.get_mut().iter() { + delta.push_insert( + span.text, + span.attributes.to_option_map().unwrap_or_default().into(), + ); + } + + Diff::Text(delta) + } + + // value is a list + fn get_value(&mut self) -> LoroValue { + match &self.state { + LazyLoad::Src(loader) => loader.to_plain_string().into(), + LazyLoad::Dst(_) => self.state.get_mut().to_string().into(), + } + } + + #[doc = r" Get the index of the child container"] + #[allow(unused)] + fn get_child_index(&self, id: &ContainerID) -> Option { + None + } + + #[allow(unused)] + fn get_child_containers(&self) -> Vec { + Vec::new() + } + + fn contains_child(&self, _id: &ContainerID) -> bool { + false + } + + fn fork(&self, config: &crate::configure::Configure) -> Self { + Self { + idx: self.idx, + config: config.text_style_config.clone(), + state: self.state.clone(), + version_id: 0, + } + } +} + +impl RichtextState { + #[inline(always)] + pub fn len_utf8(&mut self) -> usize { + self.state.get_mut().len_utf8() + } + + #[inline(always)] + pub fn len(&mut self, pos_type: PosType) -> usize { + self.state.get_mut().len(pos_type) + } + + #[inline(always)] + pub fn len_utf16(&mut self) -> usize { + self.state.get_mut().len_utf16() + } + + #[inline(always)] + pub fn len_entity(&self) -> usize { + match &self.state { + LazyLoad::Src(s) => s.entity_index, + LazyLoad::Dst(d) => d.len_entity(), + } + } + + pub fn len_event(&mut self) -> usize { + if cfg!(feature = "wasm") { + self.len_utf16() + } else { + self.len_unicode() + } + } + + #[inline] + #[allow(dead_code)] + pub(crate) fn has_styles(&mut self) -> bool { + self.state.get_mut().has_styles() + } + + pub(crate) fn has_style_key_in_entity_range( + &mut self, + range: Range, + key: &StyleKey, + ) -> bool { + self.state.get_mut().range_has_style_key(range, key) + } + + /// Check if the content and style ranges are consistent. + /// + /// Panic if inconsistent. + #[allow(unused)] + pub(crate) fn check_consistency_between_content_and_style_ranges(&mut self) { + if !cfg!(debug_assertions) { + return; + } + + self.state + .get_mut() + .check_consistency_between_content_and_style_ranges(); + } + + #[inline] + pub fn len_unicode(&mut self) -> usize { + self.state.get_mut().len_unicode() + } + + #[inline] + pub(crate) fn get_entity_index_for_text_insert( + &mut self, + index: usize, + pos_type: PosType, + ) -> Result<(usize, Option), LoroError> { + self.state + .get_mut() + .get_entity_index_for_text_insert(index, pos_type) + } + + #[inline] + pub(crate) fn get_event_index_by_cursor(&mut self, cursor: Cursor) -> usize { + self.state.get_mut().cursor_to_event_index_cached(cursor) + } + + pub(crate) fn get_entity_range_and_styles_at_range( + &mut self, + range: Range, + pos_type: PosType, + ) -> (Range, Option<&Styles>) { + self.state + .get_mut() + .get_entity_range_and_text_styles_at_range(range, pos_type) + } + + #[inline] + pub(crate) fn get_styles_at_entity_index(&mut self, entity_index: usize) -> StyleMeta { + self.state + .get_mut() + .get_styles_at_entity_index_for_insert(entity_index) + } + + #[inline] + pub(crate) fn get_text_entity_ranges_in_event_index_range( + &mut self, + pos: usize, + len: usize, + ) -> LoroResult { + self.state + .get_mut() + .get_text_entity_ranges(pos, len, PosType::Event) + } + + #[inline] + pub fn get_richtext_value(&mut self) -> LoroValue { + self.state.get_mut().get_richtext_value() + } + + #[inline] + pub(crate) fn get_stable_position( + &mut self, + event_index: usize, + get_by_event_index: bool, + ) -> Option { + self.state.get_mut().get_stable_position_at_event_index( + event_index, + if get_by_event_index { + PosType::Event + } else { + PosType::Unicode + }, + ) + } + + pub(crate) fn entity_index_to_event_index(&mut self, entity_index: usize) -> usize { + self.state + .get_mut() + .entity_index_to_event_index(entity_index) + } + + pub(crate) fn index_to_event_index(&mut self, index: usize, pos_type: PosType) -> usize { + self.state.get_mut().index_to_event_index(index, pos_type) + } + + pub(crate) fn event_index_to_unicode_index(&mut self, event_index: usize) -> usize { + self.state + .get_mut() + .event_index_to_unicode_index(event_index) + } + + pub(crate) fn event_index_to_index(&mut self, event_index: usize, pos_type: PosType) -> usize { + self.state + .get_mut() + .event_index_to_index(event_index, pos_type) + } +} + +#[derive(Debug, Default, Clone)] +pub(crate) struct RichtextStateLoader { + start_anchor_pos: FxHashMap, + elements: SmallVec<[RichtextStateChunk; 1]>, + style_ranges: Vec<(Arc, Range)>, + entity_index: usize, +} + +impl From for InnerState { + fn from(value: RichtextStateLoader) -> Self { + value.into_state() + } +} + +impl RichtextStateLoader { + pub fn push(&mut self, elem: RichtextStateChunk) { + if let RichtextStateChunk::Style { style, anchor_type } = &elem { + if *anchor_type == AnchorType::Start { + self.start_anchor_pos + .insert(ID::new(style.peer, style.cnt), self.entity_index); + } else { + let start_pos = self + .start_anchor_pos + .remove(&ID::new(style.peer, style.cnt)) + .expect("Style start not found"); + + // we need to + 1 because we also need to annotate the end anchor + self.style_ranges + .push((style.clone(), start_pos..self.entity_index + 1)); + } + } + + self.entity_index += elem.rle_len(); + self.elements.push(elem); + } + + pub fn into_state(self) -> InnerState { + let mut state = InnerState::from_chunks(self.elements.into_iter()); + for (style, range) in self.style_ranges { + state.annotate_style_range(range, style); + } + + if cfg!(debug_assertions) { + state.check(); + } + + state + } + + fn is_empty(&self) -> bool { + self.elements.is_empty() + } + + fn to_plain_string(&self) -> String { + let len = self + .elements + .iter() + .map(|elem| match elem { + RichtextStateChunk::Text(text) => text.bytes().len(), + RichtextStateChunk::Style { .. } => 0, + }) + .sum(); + let mut text = String::with_capacity(len); + for elem in &self.elements { + if let RichtextStateChunk::Text(chunk) = elem { + text.push_str(chunk.as_str()); + } + } + text + } + + fn try_apply_append_delta(&mut self, delta: &DeltaRope) -> bool { + // Style anchors/ranges need InnerState's range maintenance. + if !self.start_anchor_pos.is_empty() || !self.style_ranges.is_empty() { + return false; + } + + let old_len = self.entity_index; + let mut current_len = self.entity_index; + let mut entity_index = 0; + let mut appended = Vec::new(); + for span in delta.iter() { + match span { + loro_delta::DeltaItem::Retain { len, .. } => { + entity_index += len; + if entity_index > old_len { + return false; + } + } + loro_delta::DeltaItem::Replace { value, delete, .. } => { + if *delete > 0 { + return false; + } + + let insert_len = value.rle_len(); + if insert_len == 0 { + continue; + } + + if !matches!(value, RichtextStateChunk::Text(_)) { + return false; + } + + if entity_index != current_len { + return false; + } + + appended.push(value.clone()); + entity_index += insert_len; + current_len += insert_len; + } + } + } + + if appended.is_empty() { + return false; + } + + for elem in appended { + self.push(elem); + } + true + } +} + +mod snapshot { + use loro_common::{IdFull, InternalString, LoroValue, PeerID}; + use rustc_hash::FxHashMap; + use serde_columnar::columnar; + use std::sync::Arc; + + use crate::{ + container::richtext::{ + self, richtext_state::RichtextStateChunk, str_slice::StrSlice, StyleOp, + TextStyleInfoFlag, + }, + encoding::value_register::ValueRegister, + state::{ + decode_lamport_from_delta, decode_peer_from_table, decode_peer_table, + state_decode_error, ContainerCreationContext, ContainerState, FastStateSnapshot, + }, + utils::lazy::LazyLoad, + }; + + use super::{RichtextState, RichtextStateLoader}; + + #[columnar(vec, ser, de, iterable)] + #[derive(Debug, Clone)] + struct EncodedTextSpan { + #[columnar(strategy = "DeltaRle")] + peer_idx: usize, + #[columnar(strategy = "DeltaRle")] + counter: i32, + #[columnar(strategy = "DeltaRle")] + lamport_sub_counter: i32, + /// positive for text + /// 0 for mark start + /// -1 for mark end + #[columnar(strategy = "DeltaRle")] + len: i32, + } + + #[columnar(vec, ser, de, iterable)] + #[derive(Debug, Clone)] + struct EncodedMark { + key_idx: usize, + value: LoroValue, + info: u8, + } + + #[columnar(ser, de)] + struct EncodedText { + #[columnar(class = "vec", iter = "EncodedTextSpan")] + spans: Vec, + keys: Vec, + marks: Vec, + } + + impl FastStateSnapshot for RichtextState { + /// Encodes the RichtextState into a compact binary format for fast snapshot storage and retrieval. + /// + /// The encoding format consists of: + /// 1. The full text content as a string, encoded using postcard serialization. + /// 2. A series of EncodedTextSpan structs representing text chunks and style markers: + /// - peer_idx: Index of the peer ID in a value register (delta-RLE encoded) + /// - counter: Operation counter (delta-RLE encoded) + /// - lamport_sub_counter: Lamport timestamp - counter (delta-RLE encoded) + /// - len: Length of text chunk or marker type (-1 for end, 0 for start, positive for text) + /// 3. A list of unique style keys as InternalString. + /// 4. A series of EncodedMark structs for style information: + /// - key_idx: Index of the style key in the keys list + /// - value: The style value + /// - info: Additional style information as a byte + fn encode_snapshot_fast(&mut self, mut w: W) { + let value = self.get_value().into_string().unwrap(); + postcard::to_io(&*value, &mut w).unwrap(); + let mut spans = Vec::new(); + let mut marks = Vec::new(); + + let mut peers: ValueRegister = ValueRegister::new(); + let iter: &mut dyn Iterator; + let mut a; + let mut b; + match &self.state { + LazyLoad::Src(s) => { + a = Some(s.elements.iter()); + iter = &mut *a.as_mut().unwrap(); + } + LazyLoad::Dst(s) => { + b = Some(s.iter_chunk()); + iter = &mut *b.as_mut().unwrap(); + } + } + + let mut keys: ValueRegister = ValueRegister::new(); + + for chunk in iter { + match chunk { + RichtextStateChunk::Text(t) => { + let id = t.id_full(); + assert!(t.unicode_len() > 0); + spans.push(EncodedTextSpan { + peer_idx: peers.register(&id.peer), + counter: id.counter, + lamport_sub_counter: id.lamport as i32 - id.counter, + len: t.unicode_len(), + }) + } + RichtextStateChunk::Style { style, anchor_type } => match anchor_type { + richtext::AnchorType::Start => { + let id = style.id_full(); + spans.push(EncodedTextSpan { + peer_idx: peers.register(&id.peer), + counter: id.counter, + lamport_sub_counter: id.lamport as i32 - id.counter, + len: 0, + }); + marks.push(EncodedMark { + key_idx: keys.register(&style.key), + value: style.value.clone(), + info: style.info.to_byte(), + }) + } + richtext::AnchorType::End => { + let id = style.id_full(); + spans.push(EncodedTextSpan { + peer_idx: peers.register(&id.peer), + counter: id.counter + 1, + lamport_sub_counter: id.lamport as i32 - id.counter, + len: -1, + }) + } + }, + } + } + + let peers = peers.unwrap_vec(); + leb128::write::unsigned(&mut w, peers.len() as u64).unwrap(); + for peer in peers { + w.write_all(&peer.to_le_bytes()).unwrap(); + } + + let bytes = serde_columnar::to_vec(&EncodedText { + spans, + keys: keys.unwrap_vec(), + marks, + }) + .unwrap(); + w.write_all(&bytes).unwrap(); + } + + fn decode_value(bytes: &[u8]) -> loro_common::LoroResult<(loro_common::LoroValue, &[u8])> { + let (value, bytes) = postcard::take_from_bytes(bytes).map_err(|_| { + loro_common::LoroError::DecodeError( + "Decode list value failed".to_string().into_boxed_str(), + ) + })?; + let s: String = value; + Ok((LoroValue::String(s.into()), bytes)) + } + + fn decode_snapshot_fast( + idx: crate::container::idx::ContainerIdx, + (string, mut bytes): (loro_common::LoroValue, &[u8]), + ctx: ContainerCreationContext, + ) -> loro_common::LoroResult + where + Self: Sized, + { + let mut text = RichtextState::new(idx, ctx.configure.text_style_config.clone()); + let mut loader = RichtextStateLoader::default(); + let peers = decode_peer_table(&mut bytes, "Decode richtext state failed")?; + + let string = string.into_string().map_err(|_| { + state_decode_error("Decode richtext state failed: value is not a string") + })?; + let mut s = StrSlice::new_from_str(&string); + let iters = serde_columnar::from_bytes::(bytes).map_err(|err| { + state_decode_error(format!( + "Decode richtext state failed: invalid spans: {err}" + )) + })?; + let keys = iters.keys; + let span_iter = iters.spans.into_iter(); + let mut mark_iter = iters.marks.into_iter(); + let mut id_to_style = FxHashMap::default(); + for span in span_iter { + let EncodedTextSpan { + peer_idx, + counter, + lamport_sub_counter, + len, + } = span; + let peer = + decode_peer_from_table(&peers, peer_idx, "Decode richtext state failed")?; + let lamport = decode_lamport_from_delta( + counter, + lamport_sub_counter, + "Decode richtext state failed", + )?; + let id_full = IdFull::new(peer, counter, lamport); + let chunk = match len { + 0 => { + // Style Start + let EncodedMark { + key_idx, + value, + info, + } = mark_iter.next().ok_or_else(|| { + state_decode_error("Decode richtext state failed: missing style mark") + })?; + let key = keys.get(key_idx).ok_or_else(|| { + state_decode_error( + "Decode richtext state failed: style key index out of range", + ) + })?; + let style_op = Arc::new(StyleOp { + lamport, + peer: id_full.peer, + cnt: id_full.counter, + key: key.clone(), + value, + info: TextStyleInfoFlag::from_byte(info), + }); + id_to_style.insert(id_full.id(), style_op.clone()); + RichtextStateChunk::new_style(style_op, richtext::AnchorType::Start) + } + -1 => { + // Style End + let style = id_to_style.remove(&id_full.id().inc(-1)).ok_or_else(|| { + state_decode_error("Decode richtext state failed: unmatched style end") + })?; + RichtextStateChunk::new_style(style, richtext::AnchorType::End) + } + len => { + if len < -1 { + return Err(state_decode_error( + "Decode richtext state failed: invalid text span length", + )); + } + + // Text + if s.as_str().chars().count() < len as usize { + return Err(state_decode_error( + "Decode richtext state failed: text span exceeds value length", + )); + } + + let (new, rest) = s.split_at_unicode_pos(len as usize); + s = rest; + RichtextStateChunk::new_text(new.bytes().clone(), id_full) + } + }; + + loader.push(chunk); + } + if !s.as_str().is_empty() { + return Err(state_decode_error( + "Decode richtext state failed: text value not fully covered", + )); + } + if mark_iter.next().is_some() { + return Err(state_decode_error( + "Decode richtext state failed: unused style mark", + )); + } + if !id_to_style.is_empty() { + return Err(state_decode_error( + "Decode richtext state failed: unclosed style mark", + )); + } + text.state = LazyLoad::Src(loader); + // NOTE: We need to ensure the invariance that the version id is always increased when the richtext state is changed + // This is used to avoid the version_id to be the same as the previous zero version + text.version_id = 1; + Ok(text) + } + } + + #[cfg(test)] + mod tests { + use loro_common::{ContainerType, LoroValue}; + + use crate::container::idx::ContainerIdx; + + use super::*; + + #[test] + fn richtext_fast_snapshot_rejects_corrupt_state_metadata() { + let idx = ContainerIdx::from_index_and_type(0, ContainerType::Text); + let configure = Default::default(); + let ctx = ContainerCreationContext { + configure: &configure, + peer: 0, + }; + + assert!(RichtextState::decode_snapshot_fast( + idx, + (LoroValue::String("a".into()), &[1]), + ctx, + ) + .is_err()); + + let mut text = RichtextState::new(idx, configure.text_style_config.clone()); + let mut bytes = Vec::new(); + text.encode_snapshot_fast(&mut bytes); + let (_, state_bytes) = RichtextState::decode_value(&bytes).unwrap(); + assert!(RichtextState::decode_snapshot_fast( + idx, + (LoroValue::String("a".into()), state_bytes), + ctx, + ) + .is_err()); + } + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/state/tree_state.rs b/3rdparty/loro/crates/loro-internal/src/state/tree_state.rs new file mode 100644 index 0000000000..e89964a2cf --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/state/tree_state.rs @@ -0,0 +1,1831 @@ +use either::Either; +use enum_as_inner::EnumAsInner; +use fractional_index::FractionalIndex; +use itertools::Itertools; +use loro_common::{ + ContainerID, IdFull, IdLp, LoroError, LoroResult, LoroTreeError, LoroValue, PeerID, TreeID, + DELETED_TREE_ROOT, ID, +}; +use rand::SeedableRng; +use rustc_hash::{FxHashMap, FxHashSet}; +use serde::Serialize; +use std::collections::VecDeque; +use std::fmt::Debug; +use std::ops::{Deref, DerefMut}; +use std::sync::Weak; + +use super::{ApplyLocalOpReturn, ContainerState, DiffApplyContext}; +use crate::configure::Configure; +use crate::container::idx::ContainerIdx; +use crate::delta::{TreeDiff, TreeDiffItem, TreeExternalDiff}; +use crate::diff_calc::DiffMode; +use crate::event::InternalDiff; +use crate::op::Op; +use crate::{ + container::tree::tree_op::TreeOp, + delta::TreeInternalDiff, + event::{Diff, Index}, + op::RawOp, +}; +use crate::{DocState, LoroDocInner}; + +#[derive(Clone, Debug, EnumAsInner)] +pub enum TreeFractionalIndexConfigInner { + GenerateFractionalIndex { + jitter: u8, + rng: Box, + }, + MoveDisabled, +} + +/// The state of movable tree. +/// +/// using flat representation +#[derive(Debug, Clone)] +pub struct TreeState { + idx: ContainerIdx, + trees: FxHashMap, + children: TreeChildrenCache, + fractional_index_config: TreeFractionalIndexConfigInner, + peer_id: PeerID, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct TreeStateNode { + pub parent: TreeParentId, + pub position: Option, + pub last_move_op: IdFull, +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) struct NodePosition { + pub(crate) position: FractionalIndex, + // different nodes created by a peer may have the same position + // when we merge updates that cause cycles. + // for example [::fuzz::test::test_tree::same_peer_have_same_position()] + pub(crate) idlp: IdLp, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, EnumAsInner, Serialize)] +pub enum TreeParentId { + Node(TreeID), + Root, + Deleted, + // We use `Unexist` as the old parent of a new node created + // so we can infer the retreat internal diff is `Uncreate` + Unexist, +} + +impl From> for TreeParentId { + fn from(id: Option) -> Self { + match id { + Some(id) => { + if TreeID::is_deleted_root(&id) { + TreeParentId::Deleted + } else { + TreeParentId::Node(id) + } + } + None => TreeParentId::Root, + } + } +} + +impl From for TreeParentId { + fn from(id: TreeID) -> Self { + if TreeID::is_deleted_root(&id) { + TreeParentId::Deleted + } else { + TreeParentId::Node(id) + } + } +} + +impl From<&TreeID> for TreeParentId { + fn from(id: &TreeID) -> Self { + if TreeID::is_deleted_root(id) { + TreeParentId::Deleted + } else { + TreeParentId::Node(*id) + } + } +} + +impl TreeParentId { + pub fn tree_id(&self) -> Option { + match self { + TreeParentId::Node(id) => Some(*id), + TreeParentId::Root => None, + TreeParentId::Deleted => Some(DELETED_TREE_ROOT), + TreeParentId::Unexist => unreachable!(), + } + } +} + +#[derive(Debug, Clone)] +enum NodeChildren { + Vec(Vec<(NodePosition, TreeID)>), + BTree(btree::ChildTree), +} + +impl Default for NodeChildren { + fn default() -> Self { + NodeChildren::Vec(vec![]) + } +} + +impl NodeChildren { + const MAX_SIZE_FOR_ARRAY: usize = 16; + fn get_index_by_child_id(&self, target: &TreeID) -> Option { + match self { + NodeChildren::Vec(v) => v.iter().position(|(_, id)| id == target), + NodeChildren::BTree(btree) => btree.id_to_index(target), + } + } + + fn get_last_insert_index_by_position( + &self, + node_position: &NodePosition, + ) -> Result { + match self { + NodeChildren::Vec(v) => v.binary_search_by_key(&node_position, |x| &x.0), + NodeChildren::BTree(btree) => btree.get_index_by_node_position(node_position), + } + } + + fn get_node_position_at(&self, pos: usize) -> Option<&NodePosition> { + match self { + NodeChildren::Vec(v) => v.get(pos).map(|x| &x.0), + NodeChildren::BTree(btree) => btree.get_elem_at(pos).map(|x| x.pos.as_ref()), + } + } + + fn get_elem_at(&self, pos: usize) -> Option<(&NodePosition, &TreeID)> { + match self { + NodeChildren::Vec(v) => v.get(pos).map(|x| (&x.0, &x.1)), + NodeChildren::BTree(btree) => btree.get_elem_at(pos).map(|x| (x.pos.as_ref(), &x.id)), + } + } + + fn generate_fi_at(&self, pos: usize, target: &TreeID) -> FractionalIndexGenResult { + let mut reset_ids = vec![]; + let mut left = None; + let mut next_right = None; + { + let mut right = None; + let children_num = self.len(); + if children_num == 0 { + return FractionalIndexGenResult::Ok(FractionalIndex::default()); + } + + if pos > 0 { + left = self.get_node_position_at(pos - 1); + } + if pos < children_num { + right = self.get_elem_at(pos); + } + + let left_fi = left.map(|x| &x.position); + // if left and right have the same fractional indexes, we need to scan further to + // find all the ids that need to be reset + if let Some(left_fi) = left_fi { + if Some(left_fi) == right.map(|x| &x.0.position) { + // TODO: the min length between left and right + reset_ids.push(*right.unwrap().1); + for i in (pos + 1)..children_num { + let this_position = + self.get_node_position_at(i).map(|x| &x.position).unwrap(); + if this_position == left_fi { + reset_ids.push(*self.get_elem_at(i).unwrap().1); + } else { + next_right = Some(this_position.clone()); + break; + } + } + } + } + + if reset_ids.is_empty() { + return FractionalIndexGenResult::Ok( + FractionalIndex::new(left.map(|x| &x.position), right.map(|x| &x.0.position)) + .unwrap(), + ); + } + } + let positions = FractionalIndex::generate_n_evenly( + left.map(|x| &x.position), + next_right.as_ref(), + reset_ids.len() + 1, + ) + .unwrap(); + FractionalIndexGenResult::Rearrange( + Some(*target) + .into_iter() + .chain(reset_ids) + .zip(positions) + .collect(), + ) + } + + fn get_id_at(&self, pos: usize) -> Option { + match self { + NodeChildren::Vec(v) => v.get(pos).map(|x| x.1), + NodeChildren::BTree(btree) => btree.get_elem_at(pos).map(|x| x.id), + } + } + + fn delete_child(&mut self, target: &TreeID) { + match self { + NodeChildren::Vec(v) => { + v.retain(|(_, id)| id != target); + } + NodeChildren::BTree(v) => { + v.delete_child(target); + } + } + } + + fn upgrade(&mut self) { + match self { + NodeChildren::Vec(v) => { + let mut btree = btree::ChildTree::new(); + for (pos, id) in v.drain(..) { + btree.insert_child(pos, id); + } + + *self = NodeChildren::BTree(btree); + } + NodeChildren::BTree(_) => unreachable!(), + } + } + + fn insert_child(&mut self, pos: NodePosition, id: TreeID) { + match self { + NodeChildren::Vec(v) => { + if v.len() >= Self::MAX_SIZE_FOR_ARRAY { + self.upgrade(); + return self.insert_child(pos, id); + } + + let r = v.binary_search_by(|(target, _)| target.cmp(&pos)); + match r { + Ok(_) => unreachable!(), + Err(i) => { + v.insert(i, (pos, id)); + } + } + } + NodeChildren::BTree(v) => { + v.insert_child(pos, id); + } + } + } + + fn push_child_in_order(&mut self, pos: NodePosition, id: TreeID) { + match self { + NodeChildren::Vec(v) => { + if v.len() >= Self::MAX_SIZE_FOR_ARRAY { + self.upgrade(); + return self.push_child_in_order(pos, id); + } + + if let Some(last) = v.last() { + assert!(last.0 < pos); + } + v.push((pos, id)); + } + NodeChildren::BTree(v) => { + v.push_child_in_order(pos, id); + } + } + } + + fn len(&self) -> usize { + match self { + NodeChildren::Vec(v) => v.len(), + NodeChildren::BTree(v) => v.len(), + } + } + + fn has_child(&self, node_position: &NodePosition) -> bool { + match self { + NodeChildren::Vec(v) => v + .binary_search_by(|(target, _)| target.cmp(node_position)) + .is_ok(), + NodeChildren::BTree(v) => v.has_child(node_position), + } + } + + fn iter(&self) -> impl Iterator { + match self { + NodeChildren::Vec(v) => Either::Left(v.iter().map(|x| (&x.0, &x.1))), + NodeChildren::BTree(t) => Either::Right(t.iter()), + } + } +} + +#[derive(Clone, Default)] +struct TreeChildrenCache(FxHashMap); + +mod btree { + use std::{cmp::Ordering, ops::Range, sync::Arc}; + + use generic_btree::{ + rle::{CanRemove, HasLength, Mergeable, Sliceable, TryInsert}, + BTree, BTreeTrait, Cursor, FindResult, LeafIndex, LengthFinder, Query, UseLengthFinder, + }; + use loro_common::TreeID; + use rustc_hash::FxHashMap; + + use super::NodePosition; + + struct ChildTreeTrait; + #[derive(Debug, Clone)] + pub(super) struct ChildTree { + tree: BTree, + id_to_leaf_index: FxHashMap, + } + + impl ChildTree { + pub(super) fn new() -> Self { + Self { + tree: BTree::new(), + id_to_leaf_index: FxHashMap::default(), + } + } + + pub(super) fn insert_child(&mut self, pos: NodePosition, id: TreeID) { + let (c, _) = self.tree.insert::( + &pos, + Elem { + pos: Arc::new(pos.clone()), + id, + }, + ); + + self.id_to_leaf_index.insert(id, c.leaf); + } + + pub(super) fn push_child_in_order(&mut self, pos: NodePosition, id: TreeID) { + let c = self.tree.push(Elem { + pos: Arc::new(pos.clone()), + id, + }); + self.id_to_leaf_index.insert(id, c.leaf); + } + + pub(super) fn delete_child(&mut self, id: &TreeID) { + if let Some(leaf) = self.id_to_leaf_index.remove(id) { + self.tree.remove_leaf(Cursor { leaf, offset: 0 }); + } + } + + pub(super) fn has_child(&self, pos: &NodePosition) -> bool { + match self.tree.query::(pos) { + Some(r) => r.found, + None => false, + } + } + + pub(super) fn iter(&self) -> impl Iterator { + self.tree.iter().map(|x| (&*x.pos, &x.id)) + } + + pub(super) fn len(&self) -> usize { + self.tree.root_cache().len + } + + pub(super) fn get_elem_at(&self, pos: usize) -> Option<&Elem> { + let result = self.tree.query::(&pos)?; + if !result.found { + return None; + } + self.tree.get_elem(result.leaf()) + } + + pub(super) fn id_to_index(&self, id: &TreeID) -> Option { + let leaf_index = self.id_to_leaf_index.get(id)?; + let mut ans = 0; + self.tree.visit_previous_caches( + Cursor { + leaf: *leaf_index, + offset: 0, + }, + |prev| match prev { + generic_btree::PreviousCache::NodeCache(c) => { + ans += c.len; + } + generic_btree::PreviousCache::PrevSiblingElem(_) => { + ans += 1; + } + generic_btree::PreviousCache::ThisElemAndOffset { .. } => {} + }, + ); + + Some(ans) + } + + pub(super) fn get_index_by_node_position( + &self, + node_position: &NodePosition, + ) -> Result { + let Some(res) = self.tree.query::(node_position) else { + return Ok(0); + }; + let mut ans = 0; + self.tree + .visit_previous_caches(res.cursor, |prev| match prev { + generic_btree::PreviousCache::NodeCache(c) => { + ans += c.len; + } + generic_btree::PreviousCache::PrevSiblingElem(_) => { + ans += 1; + } + generic_btree::PreviousCache::ThisElemAndOffset { elem: _, offset } => { + ans += offset; + } + }); + if res.found { + Ok(ans) + } else { + Err(ans) + } + } + } + + #[derive(Clone, Debug)] + pub(super) struct Elem { + pub(super) pos: Arc, + pub(super) id: TreeID, + } + + impl Mergeable for Elem { + fn can_merge(&self, _rhs: &Self) -> bool { + false + } + + fn merge_right(&mut self, _rhs: &Self) { + unreachable!() + } + + fn merge_left(&mut self, _left: &Self) { + unreachable!() + } + } + + impl HasLength for Elem { + fn rle_len(&self) -> usize { + 1 + } + } + + impl Sliceable for Elem { + fn _slice(&self, range: std::ops::Range) -> Self { + assert!(range.len() == 1); + self.clone() + } + } + + impl CanRemove for Elem { + fn can_remove(&self) -> bool { + false + } + } + + impl TryInsert for Elem { + fn try_insert(&mut self, _pos: usize, elem: Self) -> Result<(), Self> + where + Self: Sized, + { + Err(elem) + } + } + + #[derive(Clone, Debug, Default, PartialEq, Eq)] + struct Cache { + range: Option>>, + len: usize, + } + + impl BTreeTrait for ChildTreeTrait { + type Elem = Elem; + type Cache = Cache; + type CacheDiff = (); + const USE_DIFF: bool = false; + + fn calc_cache_internal( + cache: &mut Self::Cache, + caches: &[generic_btree::Child], + ) -> Self::CacheDiff { + if caches.is_empty() { + *cache = Default::default(); + return; + } + + *cache = Cache { + range: Some( + caches[0].cache.range.as_ref().unwrap().start.clone() + ..caches + .last() + .unwrap() + .cache + .range + .as_ref() + .unwrap() + .end + .clone(), + ), + len: caches.iter().map(|x| x.cache.len).sum(), + }; + } + + fn apply_cache_diff(_cache: &mut Self::Cache, _diff: &Self::CacheDiff) { + unreachable!() + } + + fn merge_cache_diff(_diff1: &mut Self::CacheDiff, _diff2: &Self::CacheDiff) {} + + fn get_elem_cache(elem: &Self::Elem) -> Self::Cache { + Cache { + range: Some(elem.pos.clone()..elem.pos.clone()), + len: 1, + } + } + + fn new_cache_to_diff(_cache: &Self::Cache) -> Self::CacheDiff {} + + fn sub_cache(_cache_lhs: &Self::Cache, _cache_rhs: &Self::Cache) -> Self::CacheDiff {} + } + + struct KeyQuery; + + impl Query for KeyQuery { + type QueryArg = NodePosition; + + #[inline(always)] + fn init(_target: &Self::QueryArg) -> Self { + KeyQuery + } + + #[inline] + fn find_node( + &mut self, + target: &Self::QueryArg, + caches: &[generic_btree::Child], + ) -> FindResult { + let result = caches.binary_search_by(|x| { + let range = x.cache.range.as_ref().unwrap(); + if target < &range.start { + core::cmp::Ordering::Greater + } else if target > &range.end { + core::cmp::Ordering::Less + } else { + core::cmp::Ordering::Equal + } + }); + + match result { + Ok(i) => FindResult::new_found(i, 0), + Err(i) => FindResult::new_missing( + i.min(caches.len() - 1), + if i == caches.len() { 1 } else { 0 }, + ), + } + } + + #[inline(always)] + fn confirm_elem( + &mut self, + q: &Self::QueryArg, + elem: &::Elem, + ) -> (usize, bool) { + match q.cmp(&elem.pos) { + Ordering::Less => (0, false), + Ordering::Equal => (0, true), + Ordering::Greater => (1, false), + } + } + } + + impl UseLengthFinder for ChildTreeTrait { + fn get_len(cache: &::Cache) -> usize { + cache.len + } + } +} + +impl Debug for TreeChildrenCache { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + writeln!(f, "TreeChildrenCache {{")?; + for (parent, children) in self.0.iter() { + writeln!(f, " {:?}:{{", parent)?; + for (position, id) in children.iter() { + writeln!(f, " {:?} -> {:?}", position, id)?; + } + writeln!(f, " }}")?; + } + writeln!(f, "}}") + } +} + +impl Deref for TreeChildrenCache { + type Target = FxHashMap; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for TreeChildrenCache { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl NodePosition { + fn new(position: FractionalIndex, idlp: IdLp) -> Self { + Self { position, idlp } + } +} + +pub(crate) enum FiIfNotConfigured { + UseJitterZero, + #[allow(unused)] + Throw, + #[allow(unused)] + Zero, +} + +impl TreeState { + pub fn new(idx: ContainerIdx, peer_id: PeerID) -> Self { + Self { + idx, + trees: FxHashMap::default(), + children: Default::default(), + fractional_index_config: TreeFractionalIndexConfigInner::GenerateFractionalIndex { + jitter: 0, + rng: Box::new(rand::rngs::StdRng::seed_from_u64(0)), + }, + peer_id, + } + } + + pub fn mov( + &mut self, + target: TreeID, + parent: TreeParentId, + id: IdFull, + position: Option, + with_check: bool, + ) -> Result<(), LoroError> { + if with_check { + if let TreeParentId::Node(parent) = parent { + if !self.trees.contains_key(&parent) { + return Err(LoroTreeError::TreeNodeParentNotFound(parent).into()); + } + } + if self.is_ancestor_of(&target, &parent) { + return Err(LoroTreeError::CyclicMoveError.into()); + } + } + if let Some(old_parent) = self.trees.get(&target).map(|x| x.parent) { + // remove old position + self.try_delete_position_cache(&old_parent, &target); + } + + let entry = self.children.entry(parent).or_default(); + let node_position = NodePosition::new(position.clone().unwrap_or_default(), id.idlp()); + debug_assert!(!entry.has_child(&node_position)); + entry.insert_child(node_position, target); + + self.trees.insert( + target, + TreeStateNode { + parent, + position, + last_move_op: id, + }, + ); + + Ok(()) + } + + fn _init_push_tree_node_in_order( + &mut self, + target: TreeID, + parent: TreeParentId, + last_move_op: IdFull, + position: Option, + ) -> Result<(), LoroError> { + debug_assert!(!self.trees.contains_key(&target)); + let entry = self.children.entry(parent).or_default(); + let node_position = + NodePosition::new(position.clone().unwrap_or_default(), last_move_op.idlp()); + debug_assert!(!entry.has_child(&node_position)); + entry.push_child_in_order(node_position, target); + self.trees.insert( + target, + TreeStateNode { + parent, + position, + last_move_op, + }, + ); + + Ok(()) + } + + #[inline(never)] + fn is_ancestor_of(&self, maybe_ancestor: &TreeID, node_id: &TreeParentId) -> bool { + if !self.trees.contains_key(maybe_ancestor) { + return false; + } + if let TreeParentId::Node(id) = node_id { + if id == maybe_ancestor { + return true; + } + } + match node_id { + TreeParentId::Node(id) => { + let parent = &self.trees.get(id).unwrap().parent; + if parent == node_id { + panic!("is_ancestor_of loop") + } + self.is_ancestor_of(maybe_ancestor, parent) + } + TreeParentId::Deleted | TreeParentId::Root => false, + TreeParentId::Unexist => unreachable!(), + } + } + + /// Get the parent of the node, if the node does not exist, return None + pub fn parent(&self, target: &TreeID) -> Option { + self.trees.get(target).map(|x| x.parent) + } + + /// If the node is unexist, return None. If the node is deleted, return true. + pub(crate) fn is_node_deleted(&self, target: &TreeID) -> Option { + match self.trees.get(target).map(|x| x.parent)? { + TreeParentId::Deleted => Some(true), + TreeParentId::Root => Some(false), + TreeParentId::Node(p) => self.is_node_deleted(&p), + TreeParentId::Unexist => unreachable!(), + } + } + + pub(crate) fn is_node_unexist(&self, target: &TreeID) -> bool { + !self.trees.contains_key(target) + } + + // Get all flat tree nodes, excluding deleted nodes. + pub(crate) fn tree_nodes(&self) -> Vec { + self.get_all_tree_nodes_under(TreeParentId::Root) + } + + // Get all flat deleted nodes + #[allow(unused)] + pub(crate) fn deleted_tree_nodes(&self) -> Vec { + self.get_all_tree_nodes_under(TreeParentId::Deleted) + } + + // Get all flat tree nodes under the parent + pub(crate) fn get_all_tree_nodes_under(&self, root: TreeParentId) -> Vec { + let mut ans = vec![]; + let children = self.children.get(&root); + let mut q = children + .map(|x| VecDeque::from_iter(x.iter().enumerate().zip(std::iter::repeat(root)))) + .unwrap_or_default(); + + while let Some(((index, (position, &target)), parent)) = q.pop_front() { + ans.push(TreeNode { + id: target, + parent, + fractional_index: position.position.clone(), + index, + last_move_op: self.trees.get(&target).map(|x| x.last_move_op).unwrap(), + }); + if let Some(children) = self.children.get(&TreeParentId::Node(target)) { + q.extend( + children + .iter() + .enumerate() + .map(|(index, (position, this_target))| { + ((index, (position, this_target)), TreeParentId::Node(target)) + }), + ); + } + } + ans + } + + pub(crate) fn get_all_hierarchy_nodes_under( + &self, + root: TreeParentId, + ) -> Vec { + let mut ans = vec![]; + let Some(children) = self.children.get(&root) else { + return ans; + }; + for (index, (position, &target)) in children.iter().enumerate() { + ans.push(TreeNodeWithChildren { + id: target, + parent: root, + fractional_index: position.position.clone(), + index, + children: self.get_all_hierarchy_nodes_under(TreeParentId::Node(target)), + }) + } + ans + } + + fn bfs_all_alive_nodes_for_fast_snapshot(&self) -> Vec { + let mut ans = vec![]; + self._bfs_all_nodes(TreeParentId::Root, &mut ans); + ans + } + + fn bfs_all_deleted_nodes_for_fast_snapshot(&self) -> Vec { + let mut ans = vec![]; + self._bfs_all_nodes(TreeParentId::Deleted, &mut ans); + ans + } + + fn _bfs_all_nodes(&self, root: TreeParentId, ans: &mut Vec) { + let children = self.children.get(&root); + if let Some(children) = children { + for (index, (position, target)) in children.iter().enumerate() { + ans.push(TreeNode { + id: *target, + parent: root, + fractional_index: position.position.clone(), + index, + last_move_op: self.trees.get(target).map(|x| x.last_move_op).unwrap(), + }); + } + + for (_, id) in children.iter() { + self._bfs_all_nodes(TreeParentId::Node(*id), ans); + } + } + } + + pub fn nodes(&self) -> Vec { + self.trees.keys().copied().collect::>() + } + + #[cfg(feature = "test_utils")] + pub fn max_counter(&self) -> i32 { + self.trees + .keys() + .filter(|&k| !self.is_node_deleted(k).unwrap()) + .map(|k| k.counter) + .max() + .unwrap_or(0) + } + + pub fn get_children<'a>( + &'a self, + parent: &TreeParentId, + ) -> Option + 'a> { + self.children.get(parent).map(|x| x.iter().map(|x| *x.1)) + } + + pub fn children_num(&self, parent: &TreeParentId) -> Option { + self.children.get(parent).map(|x| x.len()) + } + + /// Determine whether the target is the child of the node + /// + /// O(1) + pub fn is_parent(&self, target: &TreeID, parent: &TreeParentId) -> bool { + self.trees.get(target).is_some_and(|x| x.parent == *parent) + } + + /// Delete the position cache of the node + pub(crate) fn try_delete_position_cache(&mut self, parent: &TreeParentId, target: &TreeID) { + if let Some(x) = self.children.get_mut(parent) { + x.delete_child(target); + } + } + + pub(crate) fn generate_position_at( + &mut self, + target: &TreeID, + parent: &TreeParentId, + index: usize, + cfg: FiIfNotConfigured, + ) -> FractionalIndexGenResult { + match &mut self.fractional_index_config { + TreeFractionalIndexConfigInner::GenerateFractionalIndex { jitter, rng } => { + if *jitter == 0 { + self.children + .entry(*parent) + .or_default() + .generate_fi_at(index, target) + } else { + self.children + .entry(*parent) + .or_default() + .generate_fi_at_jitter(index, target, rng.as_mut(), *jitter) + } + } + TreeFractionalIndexConfigInner::MoveDisabled => match cfg { + FiIfNotConfigured::Throw => FractionalIndexGenResult::NotConfigured, + FiIfNotConfigured::UseJitterZero => self + .children + .entry(*parent) + .or_default() + .generate_fi_at(index, target), + FiIfNotConfigured::Zero => FractionalIndexGenResult::Ok(FractionalIndex::default()), + }, + } + } + + pub(crate) fn is_fractional_index_enabled(&self) -> bool { + !matches!( + self.fractional_index_config, + TreeFractionalIndexConfigInner::MoveDisabled + ) + } + + pub(crate) fn enable_generate_fractional_index(&mut self, jitter: u8) { + if let TreeFractionalIndexConfigInner::GenerateFractionalIndex { + jitter: old_jitter, .. + } = &mut self.fractional_index_config + { + *old_jitter = jitter; + return; + } + self.fractional_index_config = TreeFractionalIndexConfigInner::GenerateFractionalIndex { + jitter, + rng: Box::new(rand::rngs::StdRng::seed_from_u64(self.peer_id)), + }; + } + + pub(crate) fn disable_generate_fractional_index(&mut self) { + self.fractional_index_config = TreeFractionalIndexConfigInner::MoveDisabled; + } + + pub(crate) fn get_position(&self, target: &TreeID) -> Option { + self.trees.get(target).and_then(|x| x.position.clone()) + } + + pub(crate) fn get_index_by_tree_id(&self, target: &TreeID) -> Option { + let parent = self.parent(target)?; + (!parent.is_deleted()) + .then(|| { + self.children + .get(&parent) + .and_then(|x| x.get_index_by_child_id(target)) + }) + .flatten() + } + + pub(crate) fn get_index_by_position( + &self, + parent: &TreeParentId, + node_position: &NodePosition, + ) -> Option { + self.children.get(parent).map(|c| { + match c.get_last_insert_index_by_position(node_position) { + Ok(i) => i, + Err(i) => i, + } + }) + } + + pub(crate) fn get_id_by_index(&self, parent: &TreeParentId, index: usize) -> Option { + (!parent.is_deleted()) + .then(|| self.children.get(parent).and_then(|x| x.get_id_at(index))) + .flatten() + } + + /// Check the consistency between `self.trees` and `self.children` + /// + /// It's used for debug and test + #[allow(unused)] + pub(crate) fn check_tree_integrity(&self) { + let mut parent_children_map: FxHashMap> = + FxHashMap::default(); + for (id, node) in self.trees.iter() { + let parent = node.parent; + parent_children_map.entry(parent).or_default().insert(*id); + } + + for (parent, children) in parent_children_map.iter() { + let cached_children = self.get_children(parent).expect("parent not found"); + let cached_children = cached_children.collect::>(); + if &cached_children != children { + panic!( + "tree integrity broken: children set of node {:?} is not consistent", + parent + ); + } + } + } + + pub fn is_empty(&self) -> bool { + match self.children.get(&TreeParentId::Root) { + Some(c) => c.len() == 0, + None => true, + } + } + + pub fn get_last_move_id(&self, id: &TreeID) -> Option { + self.trees.get(id).map(|x| x.last_move_op.id()) + } +} + +pub(crate) enum FractionalIndexGenResult { + Ok(FractionalIndex), + Rearrange(Vec<(TreeID, FractionalIndex)>), + NotConfigured, +} + +impl ContainerState for TreeState { + fn container_idx(&self) -> crate::container::idx::ContainerIdx { + self.idx + } + + fn is_state_empty(&self) -> bool { + self.nodes().is_empty() + } + + // How we apply the diff is coupled with the [DiffMode] we used to calculate the diff. + // So be careful when you modify this function. + fn apply_diff_and_convert( + &mut self, + diff: crate::event::InternalDiff, + ctx: DiffApplyContext, + ) -> Diff { + let need_check = !matches!(ctx.mode, DiffMode::Checkout | DiffMode::Linear); + let mut ans = vec![]; + if let InternalDiff::Tree(tree) = &diff { + // assert never cause cycle move + for diff in tree.diff.iter() { + let last_move_op = diff.last_effective_move_op_id; + let target = diff.target; + // create associated metadata container + match &diff.action { + TreeInternalDiff::Create { parent, position } => { + self.mov(target, *parent, last_move_op, Some(position.clone()), false) + .unwrap(); + + if let TreeParentId::Node(p) = parent { + // reuse diff cache, at this time,it's "move in deleted" + if self.is_node_deleted(p).unwrap() { + continue; + } + } + + let index = self.get_index_by_tree_id(&target).unwrap(); + ans.push(TreeDiffItem { + target, + action: TreeExternalDiff::Create { + parent: *parent, + index, + position: position.clone(), + }, + }); + } + TreeInternalDiff::Move { parent, position } => { + let old_parent = self.trees.get(&target).unwrap().parent; + // If this is some, the node is still alive at the moment + let old_index = self.get_index_by_tree_id(&target); + let was_alive = !self.is_node_deleted(&target).unwrap(); + if need_check { + if self + .mov(target, *parent, last_move_op, Some(position.clone()), true) + .is_ok() + { + if self.is_node_deleted(&target).unwrap() { + if was_alive { + // delete event + ans.push(TreeDiffItem { + target, + action: TreeExternalDiff::Delete { + old_parent, + old_index: old_index.unwrap(), + }, + }); + } + // Otherwise, it's a normal move inside deleted nodes, no event is needed + } else if was_alive { + // normal move + ans.push(TreeDiffItem { + target, + action: TreeExternalDiff::Move { + parent: *parent, + index: self.get_index_by_tree_id(&target).unwrap(), + position: position.clone(), + old_parent, + old_index: old_index.unwrap(), + }, + }); + } else { + // create event + ans.push(TreeDiffItem { + target, + action: TreeExternalDiff::Create { + parent: *parent, + index: self.get_index_by_tree_id(&target).unwrap(), + position: position.clone(), + }, + }); + } + } + } else { + self.mov(target, *parent, last_move_op, Some(position.clone()), false) + .unwrap(); + + if let TreeParentId::Node(p) = parent { + // reuse diff cache, at this time,it's "move in deleted" + if self.is_node_deleted(p).unwrap() { + continue; + } + } + + let index = self.get_index_by_tree_id(&target).unwrap(); + match was_alive { + true => { + ans.push(TreeDiffItem { + target, + action: TreeExternalDiff::Move { + parent: *parent, + index, + position: position.clone(), + old_parent, + old_index: old_index.unwrap(), + }, + }); + } + false => { + ans.push(TreeDiffItem { + target, + action: TreeExternalDiff::Create { + parent: *parent, + index, + position: position.clone(), + }, + }); + } + } + }; + } + TreeInternalDiff::Delete { parent, position } => { + let mut send_event = true; + if self.is_node_deleted(&target).unwrap() { + send_event = false; + } + if send_event { + ans.push(TreeDiffItem { + target, + action: TreeExternalDiff::Delete { + old_parent: self.trees.get(&target).unwrap().parent, + old_index: self.get_index_by_tree_id(&target).unwrap(), + }, + }); + } + self.mov(target, *parent, last_move_op, position.clone(), false) + .unwrap(); + } + TreeInternalDiff::MoveInDelete { parent, position } => { + self.mov(target, *parent, last_move_op, position.clone(), false) + .unwrap(); + } + TreeInternalDiff::UnCreate => { + // maybe the node created and moved to the parent deleted + if !self.is_node_deleted(&target).unwrap() { + ans.push(TreeDiffItem { + target, + action: TreeExternalDiff::Delete { + old_parent: self.trees.get(&target).unwrap().parent, + old_index: self.get_index_by_tree_id(&target).unwrap(), + }, + }); + } + // delete it from state + let node = self.trees.remove(&target); + if let Some(node) = node { + if !node.parent.is_deleted() { + self.children + .get_mut(&node.parent) + .unwrap() + .delete_child(&target); + } + } + continue; + } + }; + } + } + + // self.check_tree_integrity(); + Diff::Tree(TreeDiff { diff: ans }) + } + + // How we apply the diff is coupled with the [DiffMode] we used to calculate the diff. + // So be careful when you modify this function. + fn apply_diff(&mut self, diff: InternalDiff, ctx: DiffApplyContext) -> LoroResult<()> { + if let InternalDiff::Tree(tree) = &diff { + let need_check = !matches!(ctx.mode, DiffMode::Checkout | DiffMode::Linear); + // assert never cause cycle move + for diff in tree.diff.iter() { + let last_move_op = diff.last_effective_move_op_id; + let target = diff.target; + // create associated metadata container + match &diff.action { + TreeInternalDiff::Create { parent, position } + | TreeInternalDiff::Move { + parent, position, .. + } => { + if need_check { + self.mov(target, *parent, last_move_op, Some(position.clone()), true) + .unwrap_or_default(); + } else { + self.mov(target, *parent, last_move_op, Some(position.clone()), false)?; + } + } + TreeInternalDiff::Delete { parent, position } => { + self.mov(target, *parent, last_move_op, position.clone(), false)?; + } + TreeInternalDiff::MoveInDelete { parent, position } => { + self.mov(target, *parent, last_move_op, position.clone(), false)?; + } + TreeInternalDiff::UnCreate => { + // delete it from state + let parent = self.trees.remove(&target); + if let Some(parent) = parent { + if !parent.parent.is_deleted() { + self.children + .get_mut(&parent.parent) + .unwrap() + .delete_child(&target); + } + } + continue; + } + }; + } + } + // self.check_tree_integrity(); + Ok(()) + } + + fn apply_local_op(&mut self, raw_op: &RawOp, _op: &Op) -> LoroResult { + let mut deleted_containers = vec![]; + match &raw_op.content { + crate::op::RawOpContent::Tree(tree) => match &**tree { + TreeOp::Create { + target, + parent, + position, + } + | TreeOp::Move { + target, + parent, + position, + } => { + let parent = TreeParentId::from(*parent); + self.mov( + *target, + parent, + raw_op.id_full(), + Some(position.clone()), + true, + )?; + } + TreeOp::Delete { target } => { + let parent = TreeParentId::Deleted; + deleted_containers.push(ContainerID::new_normal( + target.id(), + loro_common::ContainerType::Map, + )); + self.mov(*target, parent, raw_op.id_full(), None, true)?; + } + }, + _ => unreachable!(), + } + // self.check_tree_integrity(); + Ok(ApplyLocalOpReturn { deleted_containers }) + } + + fn to_diff(&mut self, _doc: &Weak) -> Diff { + let mut diffs = vec![]; + let Some(roots) = self.children.get(&TreeParentId::Root) else { + return Diff::Tree(TreeDiff { diff: vec![] }); + }; + + let mut q = VecDeque::from_iter(roots.iter()); + let mut index = 0; + let mut parent = TreeParentId::Root; + while let Some((position, node)) = q.pop_front() { + let node_parent = self.trees.get(node).unwrap().parent; + if node_parent != parent { + index = 0; + parent = node_parent; + } + let diff = TreeDiffItem { + target: *node, + action: TreeExternalDiff::Create { + parent: node_parent, + index, + position: position.position.clone(), + }, + }; + index += 1; + diffs.push(diff); + if let Some(children) = self.children.get(&TreeParentId::Node(*node)) { + // TODO: Refactor: you can include the index and parent in the q + // The code will be more robust and easy to understand + q.extend(children.iter()); + } + } + + Diff::Tree(TreeDiff { diff: diffs }) + } + + fn get_value(&mut self) -> LoroValue { + self.get_all_hierarchy_nodes_under(TreeParentId::Root) + .into_iter() + .map(|x| x.into_value()) + .collect::>() + .into() + } + + /// Get the index of the child container + fn get_child_index(&self, id: &ContainerID) -> Option { + let id = id.as_normal().unwrap(); + let tree_id = TreeID { + peer: *id.0, + counter: *id.1, + }; + if !self.trees.contains_key(&tree_id) || self.is_node_deleted(&tree_id).unwrap() { + None + } else { + Some(Index::Node(tree_id)) + } + } + + fn contains_child(&self, id: &ContainerID) -> bool { + let id = id.as_normal().unwrap(); + let tree_id = TreeID { + peer: *id.0, + counter: *id.1, + }; + self.trees.contains_key(&tree_id) && !self.is_node_deleted(&tree_id).unwrap() + } + + fn get_child_containers(&self) -> Vec { + self.trees + .keys() + .map(|n| n.associated_meta_container()) + .collect_vec() + } + + fn fork(&self, _config: &Configure) -> Self { + self.clone() + } +} + +// convert map container to LoroValue +#[allow(clippy::ptr_arg)] +pub(crate) fn get_meta_value(nodes: &mut Vec, state: &mut DocState) { + for node in nodes.iter_mut() { + let map = node.as_map_mut().unwrap().make_mut(); + let meta = map.get_mut("meta").unwrap(); + let id = meta.as_container().unwrap(); + *meta = state.get_container_deep_value(state.arena.register_container(id)); + let children = map.get_mut("children").unwrap().as_list_mut().unwrap(); + get_meta_value(children.make_mut(), state); + } +} + +#[derive(Debug, Clone)] +pub struct TreeNode { + pub id: TreeID, + pub parent: TreeParentId, + pub fractional_index: FractionalIndex, + pub index: usize, + pub last_move_op: IdFull, +} + +#[derive(Debug, Clone)] +pub struct TreeNodeWithChildren { + pub id: TreeID, + pub parent: TreeParentId, + pub fractional_index: FractionalIndex, + pub index: usize, + pub children: Vec, +} + +impl TreeNodeWithChildren { + fn into_value(self) -> LoroValue { + let mut t = FxHashMap::default(); + t.insert("id".to_string(), self.id.to_string().into()); + let p = self + .parent + .tree_id() + .map(|p| p.to_string().into()) + .unwrap_or(LoroValue::Null); + t.insert("parent".to_string(), p); + t.insert( + "meta".to_string(), + self.id.associated_meta_container().into(), + ); + t.insert("index".to_string(), (self.index as i64).into()); + t.insert( + "fractional_index".to_string(), + self.fractional_index.to_string().into(), + ); + t.insert( + "children".to_string(), + self.children + .into_iter() + .map(|x| x.into_value()) + .collect::>() + .into(), + ); + t.into() + } +} + +mod jitter { + use super::{FractionalIndexGenResult, NodeChildren}; + use fractional_index::FractionalIndex; + use loro_common::TreeID; + use rand::Rng; + + impl NodeChildren { + pub(super) fn generate_fi_at_jitter( + &self, + pos: usize, + target: &TreeID, + rng: &mut impl Rng, + jitter: u8, + ) -> FractionalIndexGenResult { + let mut reset_ids = vec![]; + let mut left = None; + let mut next_right = None; + { + let mut right = None; + let children_num = self.len(); + if children_num == 0 { + return FractionalIndexGenResult::Ok(FractionalIndex::jitter_default( + rng, jitter, + )); + } + + if pos > 0 { + left = self.get_node_position_at(pos - 1); + } + if pos < children_num { + right = self.get_elem_at(pos); + } + + let left_fi = left.map(|x| &x.position); + // if left and right have the same fractional indexes, we need to scan further to + // find all the ids that need to be reset + if let Some(left_fi) = left_fi { + if Some(left_fi) == right.map(|x| &x.0.position) { + // TODO: the min length between left and right + reset_ids.push(*right.unwrap().1); + for i in (pos + 1)..children_num { + let this_position = &self.get_node_position_at(i).unwrap().position; + if this_position == left_fi { + reset_ids.push(*self.get_elem_at(i).unwrap().1); + } else { + next_right = Some(this_position.clone()); + break; + } + } + } + } + + if reset_ids.is_empty() { + return FractionalIndexGenResult::Ok( + FractionalIndex::new_jitter( + left.map(|x| &x.position), + right.map(|x| &x.0.position), + rng, + jitter, + ) + .unwrap(), + ); + } + } + let positions = FractionalIndex::generate_n_evenly_jitter( + left.map(|x| &x.position), + next_right.as_ref(), + reset_ids.len() + 1, + rng, + jitter, + ) + .unwrap(); + FractionalIndexGenResult::Rearrange( + Some(*target) + .into_iter() + .chain(reset_ids) + .zip(positions) + .collect(), + ) + } + } +} + +mod snapshot { + use std::{borrow::Cow, collections::BTreeSet}; + + use fractional_index::FractionalIndex; + use loro_common::{IdFull, PeerID, TreeID}; + use rustc_hash::FxHashMap; + + use serde_columnar::columnar; + + use crate::{ + encoding::{arena::PositionArena, value_register::ValueRegister}, + state::{ + decode_counter, decode_lamport_from_delta, decode_peer_from_table, decode_peer_table, + state_decode_error, FastStateSnapshot, + }, + }; + + use super::{TreeNode, TreeParentId, TreeState}; + #[columnar(vec, ser, de, iterable)] + #[derive(Debug, Clone)] + struct EncodedTreeNodeId { + #[columnar(strategy = "DeltaRle")] + peer_idx: usize, + #[columnar(strategy = "DeltaRle")] + counter: i32, + } + + #[columnar(vec, ser, de, iterable)] + #[derive(Debug, Clone)] + struct EncodedTreeNode { + /// If this field is 0, it means none, its parent is root + /// If this field is 1, its parent is the deleted root + #[columnar(strategy = "DeltaRle")] + parent_idx_plus_two: usize, + #[columnar(strategy = "DeltaRle")] + last_set_peer_idx: usize, + #[columnar(strategy = "DeltaRle")] + last_set_counter: i32, + #[columnar(strategy = "DeltaRle")] + last_set_lamport_sub_counter: i32, + fractional_index_idx: usize, + } + + #[columnar(ser, de)] + struct EncodedTree<'a> { + #[columnar(class = "vec", iter = "EncodedTreeNodeId")] + node_ids: Vec, + #[columnar(class = "vec", iter = "EncodedTreeNode")] + nodes: Vec, + #[columnar(borrow)] + fractional_indexes: Cow<'a, [u8]>, + #[columnar(borrow)] + reserved_has_effect_bool_rle: Cow<'a, [u8]>, + } + + fn encode( + state: &TreeState, + input: Vec, + deleted_nodes: Vec, + ) -> (ValueRegister, EncodedTree<'_>) { + let mut peers: ValueRegister = ValueRegister::new(); + let mut position_set = BTreeSet::default(); + let mut nodes = Vec::with_capacity(input.len()); + let mut node_ids = Vec::with_capacity(input.len()); + let mut position_register = ValueRegister::new(); + let mut id_to_idx = FxHashMap::default(); + for node in input.iter().chain(deleted_nodes.iter()) { + position_set.insert(node.fractional_index.clone()); + let idx = node_ids.len(); + node_ids.push(EncodedTreeNodeId { + peer_idx: peers.register(&node.id.peer), + counter: node.id.counter, + }); + id_to_idx.insert(node.id, idx); + } + + for p in position_set { + position_register.register(&p); + } + + for node in input { + let n = state.trees.get(&node.id).unwrap(); + let last_set_id = n.last_move_op; + nodes.push(EncodedTreeNode { + parent_idx_plus_two: match node.parent { + TreeParentId::Deleted => 1, + TreeParentId::Root => 0, + TreeParentId::Node(id) => id_to_idx.get(&id).unwrap() + 2, + TreeParentId::Unexist => unreachable!(), + }, + last_set_peer_idx: peers.register(&last_set_id.peer), + last_set_counter: last_set_id.counter, + last_set_lamport_sub_counter: last_set_id.lamport as i32 - last_set_id.counter, + fractional_index_idx: position_register.register(&node.fractional_index), + }) + } + + for node in deleted_nodes { + let n = state.trees.get(&node.id).unwrap(); + let last_set_id = n.last_move_op; + nodes.push(EncodedTreeNode { + parent_idx_plus_two: match node.parent { + TreeParentId::Deleted => 1, + TreeParentId::Root => 0, + TreeParentId::Node(id) => id_to_idx.get(&id).unwrap() + 2, + TreeParentId::Unexist => unreachable!(), + }, + last_set_peer_idx: peers.register(&last_set_id.peer), + last_set_counter: last_set_id.counter, + last_set_lamport_sub_counter: last_set_id.lamport as i32 - last_set_id.counter, + fractional_index_idx: position_register.register(&node.fractional_index), + }) + } + + let position_vec = position_register.unwrap_vec(); + let positions = PositionArena::from_positions(position_vec.iter().map(|p| p.as_bytes())); + ( + peers, + EncodedTree { + node_ids, + nodes, + fractional_indexes: positions.encode().into(), + reserved_has_effect_bool_rle: vec![].into(), + }, + ) + } + + impl FastStateSnapshot for TreeState { + /// Encodes the TreeState into a compact binary format for efficient serialization. + /// + /// The encoding schema: + /// 1. Encodes all nodes using a breadth-first search traversal, ensuring a consistent order. + /// 2. Uses a ValueRegister to deduplicate and index PeerIDs and TreeIDs, reducing redundancy. + /// - PeerIDs are stored once and referenced by index. + /// - TreeIDs are decomposed into peer index and counter for compact representation. + /// 3. Encodes fractional indexes using a PositionArena for space efficiency + /// 4. Utilizes delta encoding and run-length encoding for further size reduction: + /// - Delta encoding stores differences between consecutive values. + /// - Run-length encoding compresses sequences of repeated values. + /// 5. Stores parent relationships using indices, with special values for root and deleted nodes. + /// 6. Encodes last move operation details (peer_idx, counter[Delta], lamport clock[Delta]) for each node. + fn encode_snapshot_fast(&mut self, mut w: W) { + let all_alive_nodes = self.bfs_all_alive_nodes_for_fast_snapshot(); + let all_deleted_nodes = self.bfs_all_deleted_nodes_for_fast_snapshot(); + let (peers, encoded) = encode(self, all_alive_nodes, all_deleted_nodes); + let peers = peers.unwrap_vec(); + leb128::write::unsigned(&mut w, peers.len() as u64).unwrap(); + for peer in peers { + w.write_all(&peer.to_le_bytes()).unwrap(); + } + + let vec = serde_columnar::to_vec(&encoded).unwrap(); + w.write_all(&vec).unwrap(); + } + + fn decode_value(_: &[u8]) -> loro_common::LoroResult<(loro_common::LoroValue, &[u8])> { + unreachable!() + } + + fn decode_snapshot_fast( + idx: crate::container::idx::ContainerIdx, + (_, mut bytes): (loro_common::LoroValue, &[u8]), + ctx: crate::state::ContainerCreationContext, + ) -> loro_common::LoroResult + where + Self: Sized, + { + let peers = decode_peer_table(&mut bytes, "Decode tree state failed")?; + + let mut tree = TreeState::new(idx, ctx.peer); + let encoded: EncodedTree = serde_columnar::from_bytes(bytes).map_err(|err| { + state_decode_error(format!("Decode tree state failed: invalid metadata: {err}")) + })?; + let fractional_indexes = + PositionArena::decode(&encoded.fractional_indexes)?.try_parse_to_positions()?; + if encoded.node_ids.len() != encoded.nodes.len() { + return Err(state_decode_error( + "Decode tree state failed: node id/node length mismatch", + )); + } + let node_ids = encoded + .node_ids + .iter() + .map(|x| { + Ok(TreeID::new( + decode_peer_from_table(&peers, x.peer_idx, "Decode tree state failed")?, + decode_counter(x.counter, "Decode tree state failed")?, + )) + }) + .collect::>>()?; + for (node_id, node) in node_ids.iter().zip(encoded.nodes) { + // PERF: we don't need to mov the deleted node, instead we can cache them + // If the parent is TreeParentId::Deleted, then all the nodes afterwards are deleted + tree._init_push_tree_node_in_order( + *node_id, + match node.parent_idx_plus_two { + 0 => TreeParentId::Root, + 1 => TreeParentId::Deleted, + n => { + let id = *node_ids.get(n - 2).ok_or_else(|| { + state_decode_error( + "Decode tree state failed: parent index out of range", + ) + })?; + TreeParentId::from(Some(id)) + } + }, + IdFull::new( + decode_peer_from_table( + &peers, + node.last_set_peer_idx, + "Decode tree state failed", + )?, + node.last_set_counter, + decode_lamport_from_delta( + node.last_set_counter, + node.last_set_lamport_sub_counter, + "Decode tree state failed", + )?, + ), + Some(FractionalIndex::from_bytes( + fractional_indexes + .get(node.fractional_index_idx) + .ok_or_else(|| { + state_decode_error( + "Decode tree state failed: fractional index out of range", + ) + })? + .clone(), + )), + ) + .map_err(|err| { + state_decode_error(format!("Decode tree state failed: invalid node: {err}")) + })?; + } + + Ok(tree) + } + } + + #[cfg(test)] + mod tests { + use loro_common::{ContainerType, LoroValue}; + + use crate::{container::idx::ContainerIdx, state::ContainerCreationContext}; + + use super::*; + + #[test] + fn tree_fast_snapshot_rejects_corrupt_state_metadata() { + let idx = ContainerIdx::from_index_and_type(0, ContainerType::Tree); + let configure = Default::default(); + let ctx = ContainerCreationContext { + configure: &configure, + peer: 0, + }; + + assert!(TreeState::decode_snapshot_fast(idx, (LoroValue::Null, &[1]), ctx).is_err()); + } + + #[test] + fn tree_fast_snapshot_rejects_negative_node_counter() { + let idx = ContainerIdx::from_index_and_type(0, ContainerType::Tree); + let configure = Default::default(); + let ctx = ContainerCreationContext { + configure: &configure, + peer: 0, + }; + + let position = fractional_index::FractionalIndex::default(); + let positions = PositionArena::from_positions([position.as_bytes()]); + let encoded = EncodedTree { + node_ids: vec![EncodedTreeNodeId { + peer_idx: 0, + counter: -1, + }], + nodes: vec![EncodedTreeNode { + parent_idx_plus_two: 0, + last_set_peer_idx: 0, + last_set_counter: 0, + last_set_lamport_sub_counter: 0, + fractional_index_idx: 0, + }], + fractional_indexes: positions.encode().into(), + reserved_has_effect_bool_rle: vec![].into(), + }; + + let mut bytes = Vec::new(); + leb128::write::unsigned(&mut bytes, 1).unwrap(); + bytes.extend_from_slice(&1_u64.to_le_bytes()); + bytes.extend_from_slice(&serde_columnar::to_vec(&encoded).unwrap()); + + assert!(TreeState::decode_snapshot_fast(idx, (LoroValue::Null, &bytes), ctx).is_err()); + } + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/state/unknown_state.rs b/3rdparty/loro/crates/loro-internal/src/state/unknown_state.rs new file mode 100644 index 0000000000..d3b1113955 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/state/unknown_state.rs @@ -0,0 +1,176 @@ +use std::sync::Weak; + +use loro_common::{ContainerID, LoroResult, LoroValue}; + +use crate::{ + configure::Configure, + container::idx::ContainerIdx, + event::{Diff, Index, InternalDiff}, + op::{Op, RawOp}, + LoroDocInner, +}; + +use super::{ApplyLocalOpReturn, ContainerState, DiffApplyContext}; + +#[derive(Debug, Clone)] +pub struct UnknownState { + idx: ContainerIdx, +} + +impl UnknownState { + pub fn new(idx: ContainerIdx) -> Self { + Self { idx } + } +} + +impl ContainerState for UnknownState { + fn container_idx(&self) -> ContainerIdx { + self.idx + } + + fn is_state_empty(&self) -> bool { + false + } + + fn apply_diff_and_convert(&mut self, _diff: InternalDiff, _ctx: DiffApplyContext) -> Diff { + unreachable!() + } + + fn apply_diff(&mut self, _diff: InternalDiff, _ctx: DiffApplyContext) -> LoroResult<()> { + unreachable!() + } + + fn apply_local_op(&mut self, _raw_op: &RawOp, _op: &Op) -> LoroResult { + unreachable!() + } + + #[doc = r" Convert a state to a diff, such that an empty state will be transformed into the same as this state when it's applied."] + fn to_diff(&mut self, _doc: &Weak) -> Diff { + Diff::Unknown + } + + fn get_value(&mut self) -> LoroValue { + unreachable!() + } + + #[doc = r" Get the index of the child container"] + #[allow(unused)] + fn get_child_index(&self, id: &ContainerID) -> Option { + None + } + + fn contains_child(&self, _id: &ContainerID) -> bool { + false + } + + #[allow(unused)] + fn get_child_containers(&self) -> Vec { + vec![] + } + + fn fork(&self, _config: &Configure) -> Self { + self.clone() + } +} + +mod snapshot { + use loro_common::LoroValue; + + use crate::state::FastStateSnapshot; + + use super::UnknownState; + + impl FastStateSnapshot for UnknownState { + fn encode_snapshot_fast(&mut self, _w: W) {} + + fn decode_value(bytes: &[u8]) -> loro_common::LoroResult<(loro_common::LoroValue, &[u8])> { + Ok((LoroValue::Null, bytes)) + } + + fn decode_snapshot_fast( + idx: crate::container::idx::ContainerIdx, + _v: (loro_common::LoroValue, &[u8]), + _ctx: crate::state::ContainerCreationContext, + ) -> loro_common::LoroResult + where + Self: Sized, + { + Ok(UnknownState::new(idx)) + } + } +} + +#[cfg(test)] +mod tests { + use std::sync::Weak; + + use loro_common::{ContainerID, ContainerType}; + + use crate::{ + configure::Configure, + container::idx::ContainerIdx, + event::Diff, + state::{ContainerCreationContext, ContainerState, FastStateSnapshot}, + InternalString, + }; + + use super::UnknownState; + + fn unknown_idx() -> ContainerIdx { + ContainerIdx::from_index_and_type(7, ContainerType::Unknown(3)) + } + + #[test] + fn unknown_state_reports_identity_without_material_value() { + let mut state = UnknownState::new(unknown_idx()); + assert_eq!(state.container_idx(), unknown_idx()); + assert!(!state.is_state_empty()); + + assert!(matches!(state.to_diff(&Weak::new()), Diff::Unknown)); + assert_eq!(state.get_child_containers(), Vec::::new()); + assert!(!state.contains_child(&ContainerID::Root { + name: InternalString::from("child"), + container_type: ContainerType::Text, + })); + assert_eq!( + state.get_child_index(&ContainerID::Root { + name: InternalString::from("child"), + container_type: ContainerType::Text, + }), + None + ); + } + + #[test] + fn unknown_state_fork_keeps_container_identity() { + let state = UnknownState::new(unknown_idx()); + let forked = state.fork(&Configure::default()); + + assert_eq!(forked.container_idx(), unknown_idx()); + assert!(!forked.is_state_empty()); + } + + #[test] + fn unknown_snapshot_roundtrip_preserves_bytes_and_index() { + let mut state = UnknownState::new(unknown_idx()); + let mut encoded = Vec::new(); + state.encode_snapshot_fast(&mut encoded); + assert!(encoded.is_empty()); + + let input = b"opaque-unknown-payload"; + let (value, rest) = UnknownState::decode_value(input).unwrap(); + assert_eq!(value, loro_common::LoroValue::Null); + assert_eq!(rest, input); + + let decoded = UnknownState::decode_snapshot_fast( + unknown_idx(), + (value, rest), + ContainerCreationContext { + configure: &Configure::default(), + peer: 0, + }, + ) + .unwrap(); + assert_eq!(decoded.container_idx(), unknown_idx()); + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/subscription.rs b/3rdparty/loro/crates/loro-internal/src/subscription.rs new file mode 100644 index 0000000000..71138334f6 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/subscription.rs @@ -0,0 +1,222 @@ +use super::{ + arena::SharedArena, + event::{DiffEvent, DocDiff}, +}; +use crate::{ + container::idx::ContainerIdx, utils::subscription::SubscriberSet, ContainerDiff, LoroDoc, + Subscription, +}; +use loro_common::{ContainerID, ID}; +use rustc_hash::FxHashMap; +use smallvec::SmallVec; +use std::{collections::VecDeque, sync::Arc}; + +use crate::sync::Mutex; +/// The callback of the local update. +pub type LocalUpdateCallback = Box) -> bool + Send + Sync + 'static>; +/// The callback of the peer id change. The second argument is the next counter for the peer. +pub type PeerIdUpdateCallback = Box bool + Send + Sync + 'static>; +#[allow(clippy::unused_unit)] +pub type Subscriber = Arc Fn(DiffEvent<'a>) + Send + Sync>; + +impl LoroDoc { + /// Subscribe to the changes of the peer id. + pub fn subscribe_peer_id_change(&self, callback: PeerIdUpdateCallback) -> Subscription { + let (s, enable) = self.peer_id_change_subs.inner().insert((), callback); + enable(); + s + } +} + +struct ObserverInner { + subscriber_set: SubscriberSet, Subscriber>, + queue: Arc>>, +} + +impl Default for ObserverInner { + fn default() -> Self { + Self { + subscriber_set: SubscriberSet::new(), + queue: Arc::new(Mutex::new(VecDeque::new())), + } + } +} + +pub struct Observer { + inner: ObserverInner, + arena: SharedArena, +} + +impl std::fmt::Debug for Observer { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Observer").finish() + } +} + +impl Observer { + pub fn new(arena: SharedArena) -> Self { + Self { + arena, + inner: ObserverInner::default(), + } + } + + pub fn subscribe(&self, id: &ContainerID, callback: Subscriber) -> Subscription { + let idx = self.arena.register_container(id); + let inner = &self.inner; + let (sub, enable) = inner.subscriber_set.insert(Some(idx), callback); + enable(); + sub + } + + pub fn subscribe_root(&self, callback: Subscriber) -> Subscription { + let inner = &self.inner; + let (sub, enable) = inner.subscriber_set.insert(None, callback); + enable(); + sub + } + + pub(crate) fn emit(&self, doc_diff: DocDiff) { + let success = self.emit_inner(doc_diff); + if success { + let mut e = self.inner.queue.lock().pop_front(); + while let Some(event) = e { + self.emit_inner(event); + e = self.inner.queue.lock().pop_front(); + } + } + } + + // When emitting changes, we need to make sure that the observer is not locked. + fn emit_inner(&self, doc_diff: DocDiff) -> bool { + let inner = &self.inner; + let mut container_events_map: FxHashMap> = + Default::default(); + for container_diff in doc_diff.diff.iter() { + self.arena + .with_ancestors(container_diff.idx, |ancestor, _| { + if inner.subscriber_set.may_include(&Some(ancestor)) { + container_events_map + .entry(ancestor) + .or_default() + .push(container_diff); + } + }); + } + + { + // Check whether we are calling events recursively. + // If so, push the event to the queue + if inner.subscriber_set.is_recursive_calling(&None) + || container_events_map + .keys() + .any(|x| inner.subscriber_set.is_recursive_calling(&Some(*x))) + { + drop(container_events_map); + inner.queue.lock().push_back(doc_diff); + return false; + } + } + + for (container_idx, container_diffs) in container_events_map { + inner + .subscriber_set + .retain(&Some(container_idx), &mut |callback| { + (callback)(DiffEvent { + current_target: Some(self.arena.get_container_id(container_idx).unwrap()), + events: &container_diffs, + event_meta: &doc_diff, + }); + true + }) + .unwrap(); + } + + let events: Vec<_> = doc_diff.diff.iter().collect(); + inner + .subscriber_set + .retain(&None, &mut |callback| { + (callback)(DiffEvent { + current_target: None, + events: &events, + event_meta: &doc_diff, + }); + true + }) + .unwrap(); + + true + } +} + +#[cfg(test)] +mod test { + use std::sync::atomic::{AtomicUsize, Ordering}; + + use super::*; + use crate::{cursor::PosType, handler::HandlerTrait, LoroDoc}; + + #[test] + fn test_recursive_events() { + let loro = Arc::new(LoroDoc::new()); + let loro_cp = loro.clone(); + let count = Arc::new(AtomicUsize::new(0)); + let count_cp = Arc::clone(&count); + let _g = loro_cp.subscribe_root(Arc::new(move |_| { + count_cp.fetch_add(1, Ordering::SeqCst); + let mut txn = loro.txn().unwrap(); + let text = loro.get_text("id"); + if text.get_value().as_string().unwrap().len() > 10 { + return; + } + text.insert_with_txn(&mut txn, 0, "123", PosType::Unicode) + .unwrap(); + txn.commit().unwrap(); + })); + + let loro = loro_cp; + let mut txn = loro.txn().unwrap(); + let text = loro.get_text("id"); + text.insert_with_txn(&mut txn, 0, "123", PosType::Unicode) + .unwrap(); + txn.commit().unwrap(); + let count = count.load(Ordering::SeqCst); + assert!(count > 2, "{}", count); + } + + #[test] + fn unsubscribe() { + let loro = Arc::new(LoroDoc::new()); + let count = Arc::new(AtomicUsize::new(0)); + let count_cp = Arc::clone(&count); + let sub = loro.subscribe_root(Arc::new(move |_| { + count_cp.fetch_add(1, Ordering::SeqCst); + })); + + let text = loro.get_text("id"); + + assert_eq!(count.load(Ordering::SeqCst), 0); + { + let mut txn = loro.txn().unwrap(); + text.insert_with_txn(&mut txn, 0, "123", PosType::Unicode) + .unwrap(); + txn.commit().unwrap(); + } + assert_eq!(count.load(Ordering::SeqCst), 1); + { + let mut txn = loro.txn().unwrap(); + text.insert_with_txn(&mut txn, 0, "123", PosType::Unicode) + .unwrap(); + txn.commit().unwrap(); + } + assert_eq!(count.load(Ordering::SeqCst), 2); + sub.unsubscribe(); + { + let mut txn = loro.txn().unwrap(); + text.insert_with_txn(&mut txn, 0, "123", PosType::Unicode) + .unwrap(); + txn.commit().unwrap(); + } + assert_eq!(count.load(Ordering::SeqCst), 2); + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/sync.rs b/3rdparty/loro/crates/loro-internal/src/sync.rs new file mode 100644 index 0000000000..3cc5f88490 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/sync.rs @@ -0,0 +1,169 @@ +#[cfg(loom)] +pub use loom::thread; +#[cfg(not(loom))] +pub use std::thread; + +#[cfg(loom)] +mod raw { + pub use loom::sync::{ + LockResult, Mutex as RawMutex, MutexGuard, RwLock as RawRwLock, RwLockReadGuard, + RwLockWriteGuard, + }; +} +#[cfg(not(loom))] +mod raw { + pub use std::sync::{ + LockResult, Mutex as RawMutex, MutexGuard, RwLock as RawRwLock, RwLockReadGuard, + RwLockWriteGuard, + }; +} + +pub use raw::{MutexGuard, RwLockReadGuard, RwLockWriteGuard}; + +#[cfg(loom)] +pub use loom::sync::atomic::{AtomicBool, AtomicI64, AtomicU64, AtomicU8, AtomicUsize}; +#[cfg(not(loom))] +pub use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU64, AtomicU8, AtomicUsize}; + +#[cfg(loom)] +pub(crate) use my_thread_local::ThreadLocal; +#[cfg(not(loom))] +pub(crate) use thread_local::ThreadLocal; + +fn expect_not_poisoned(result: raw::LockResult, lock_kind: &str) -> T { + result.unwrap_or_else(|_| panic!("poisoned {lock_kind}")) +} + +#[derive(Debug)] +pub struct Mutex { + inner: raw::RawMutex, +} + +impl Mutex { + pub fn new(value: T) -> Self { + Self { + inner: raw::RawMutex::new(value), + } + } +} + +impl Mutex { + pub fn lock(&self) -> MutexGuard<'_, T> { + self.lock_with_kind("mutex") + } + + pub(crate) fn lock_with_kind(&self, lock_kind: &str) -> MutexGuard<'_, T> { + expect_not_poisoned(self.inner.lock(), lock_kind) + } + + pub(crate) fn is_locked(&self) -> bool { + self.inner.try_lock().is_err() + } +} + +impl Default for Mutex { + fn default() -> Self { + Self::new(T::default()) + } +} + +#[derive(Debug)] +pub struct RwLock { + inner: raw::RawRwLock, +} + +impl RwLock { + pub fn new(value: T) -> Self { + Self { + inner: raw::RawRwLock::new(value), + } + } + + pub fn into_inner(self) -> T { + expect_not_poisoned(self.inner.into_inner(), "rwlock") + } +} + +impl RwLock { + pub fn read(&self) -> RwLockReadGuard<'_, T> { + expect_not_poisoned(self.inner.read(), "rwlock") + } + + pub fn write(&self) -> RwLockWriteGuard<'_, T> { + expect_not_poisoned(self.inner.write(), "rwlock") + } +} + +impl Default for RwLock { + fn default() -> Self { + Self::new(T::default()) + } +} + +#[cfg(loom)] +mod my_thread_local { + use std::sync::Arc; + + use super::thread; + use super::Mutex; + use rustc_hash::FxHashMap; + + #[derive(Debug)] + pub(crate) struct ThreadLocal { + content: Arc>>>, + } + + impl ThreadLocal { + pub fn new() -> Self { + Self { + content: Arc::new(Mutex::new(FxHashMap::default())), + } + } + + pub fn get_or_default(&self) -> Arc { + let mut content = self.content.lock(); + let v = content + .entry(thread::current().id()) + .or_insert_with(|| Arc::new(T::default())); + v.clone() + } + } + + impl Clone for ThreadLocal { + fn clone(&self) -> Self { + Self { + content: self.content.clone(), + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + #[should_panic(expected = "poisoned mutex")] + fn mutex_lock_panics_after_poison() { + let lock = Mutex::new(7); + let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _guard = lock.lock(); + panic!("poison mutex"); + })); + + drop(lock.lock()); + } + + #[test] + #[should_panic(expected = "poisoned rwlock")] + fn rwlock_read_panics_after_poison() { + let lock = RwLock::new(7); + let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let mut guard = lock.write(); + *guard = 9; + panic!("poison rwlock"); + })); + + drop(lock.read()); + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/tests/import_atomicity.rs b/3rdparty/loro/crates/loro-internal/src/tests/import_atomicity.rs new file mode 100644 index 0000000000..0e1d8f6526 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/tests/import_atomicity.rs @@ -0,0 +1,304 @@ +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; + +use crate::{ + cursor::PosType, + encoding::json_schema::json::{JsonOpContent, JsonSchema, ListOp}, + loro::ExportMode, + state::fail_next_import_state_apply_for_test, + version::{Frontiers, VersionVector}, + LoroDoc, LoroError, TreeParentId, +}; + +fn pending_len(doc: &LoroDoc) -> usize { + doc.oplog().lock().pending_changes_len() +} + +fn corrupt_snapshot_state_bytes(snapshot: &mut [u8]) { + let body_start = 22; + let oplog_len = + u32::from_le_bytes(snapshot[body_start..body_start + 4].try_into().unwrap()) as usize; + let state_len_pos = body_start + 4 + oplog_len; + let state_len = u32::from_le_bytes( + snapshot[state_len_pos..state_len_pos + 4] + .try_into() + .unwrap(), + ) as usize; + assert!(state_len > 0); + let state_start = state_len_pos + 4; + snapshot[state_start] ^= 0xff; + + let checksum = xxhash_rust::xxh32::xxh32(&snapshot[20..], u32::from_le_bytes(*b"LORO")); + snapshot[16..20].copy_from_slice(&checksum.to_le_bytes()); +} + +fn make_json_list_update_with_four_ops(peer: u64) -> (LoroDoc, JsonSchema) { + let doc = LoroDoc::new(); + doc.set_peer_id(peer).unwrap(); + let map = doc.get_map("map"); + let list = doc.get_list("list"); + let text = doc.get_text("text"); + + let mut txn = doc.txn().unwrap(); + map.insert_with_txn(&mut txn, "prefix", "map-value".into()) + .unwrap(); + list.insert_with_txn(&mut txn, 0, "seed".into()).unwrap(); + text.insert_with_txn(&mut txn, 0, "text-value", PosType::Unicode) + .unwrap(); + list.insert_with_txn(&mut txn, 1, "tail".into()).unwrap(); + txn.commit().unwrap(); + + let json = doc.export_json_updates(&Default::default(), &doc.oplog_vv(), false); + assert_eq!(json.changes.len(), 1); + assert_eq!(json.changes[0].ops.len(), 4); + (doc, json) +} + +fn move_last_list_insert_far_out_of_bounds(json: &mut JsonSchema) { + let last_change = json.changes.last_mut().unwrap(); + let last_op = last_change.ops.last_mut().unwrap(); + match &mut last_op.content { + JsonOpContent::List(ListOp::Insert { pos, .. }) => { + *pos = 1_000; + } + other => panic!("expected list insert op, got {other:?}"), + } +} + +fn make_multi_peer_frontier_doc() -> LoroDoc { + let base = LoroDoc::new_auto_commit(); + base.set_peer_id(1).unwrap(); + base.get_map("map").insert("base", 0).unwrap(); + base.get_text("text").insert_unicode(0, "base").unwrap(); + let tree = base.get_tree("tree"); + let root = tree.create(TreeParentId::Root).unwrap(); + tree.get_meta(root).unwrap().insert("base", 0).unwrap(); + + let base_updates = base.export(ExportMode::all_updates()).unwrap(); + + let peer2 = base.fork(); + peer2.set_peer_id(2).unwrap(); + peer2.get_map("map").insert("p2", 2).unwrap(); + peer2.get_text("text").insert_unicode(0, "p2").unwrap(); + peer2.commit_then_renew(); + let peer2_updates = peer2.export(ExportMode::updates(&base.oplog_vv())).unwrap(); + + let peer3 = base.fork(); + peer3.set_peer_id(3).unwrap(); + peer3.get_map("map").insert("p3", 3).unwrap(); + let peer3_tree = peer3.get_tree("tree"); + let node = peer3_tree.create(TreeParentId::Root).unwrap(); + peer3_tree.get_meta(node).unwrap().insert("p3", 3).unwrap(); + peer3.commit_then_renew(); + let peer3_updates = peer3.export(ExportMode::updates(&base.oplog_vv())).unwrap(); + + let target = LoroDoc::new(); + target.import(&base_updates).unwrap(); + target.import(&peer2_updates).unwrap(); + target.import(&peer3_updates).unwrap(); + target +} + +fn assert_doc_unchanged( + doc: &LoroDoc, + vv: &VersionVector, + frontiers: &Frontiers, + state: &crate::LoroValue, +) { + assert_eq!(&doc.oplog_vv(), vv); + assert_eq!(&doc.oplog_frontiers(), frontiers); + assert_eq!(&doc.get_deep_value(), state); +} + +#[test] +fn failed_dependency_import_rolls_back_single_pending_change() { + let src = LoroDoc::new_auto_commit(); + src.set_peer_id(1).unwrap(); + let map = src.get_map("map"); + map.insert("seed", "base").unwrap(); + let update_base = src + .export(ExportMode::updates(&VersionVector::default())) + .unwrap(); + let version_base = src.oplog_vv(); + + map.insert("later", "value").unwrap(); + let update_later = src.export(ExportMode::updates(&version_base)).unwrap(); + + let dst = LoroDoc::new(); + dst.import(&update_later).unwrap(); + assert_eq!(pending_len(&dst), 1); + let vv_before_import = dst.oplog_vv(); + let frontiers_before_import = dst.oplog_frontiers(); + let state_before_import = dst.get_deep_value(); + + fail_next_import_state_apply_for_test(); + let err = dst.import(&update_base).unwrap_err(); + assert!( + err.to_string().contains("state apply failpoint"), + "unexpected error: {err:?}" + ); + assert_eq!(pending_len(&dst), 1); + assert_eq!(dst.oplog_vv(), vv_before_import); + assert_eq!(dst.oplog_frontiers(), frontiers_before_import); + assert_eq!(dst.get_deep_value(), state_before_import); + + dst.import(&update_base).unwrap(); + assert_eq!(pending_len(&dst), 0); + assert_eq!(dst.oplog_vv(), src.oplog_vv()); + assert_eq!(dst.oplog_frontiers(), src.oplog_frontiers()); + assert_eq!(dst.get_deep_value(), src.get_deep_value()); +} + +#[test] +fn failed_dependency_import_rolls_back_multiple_pending_changes() { + let base = LoroDoc::new_auto_commit(); + base.set_peer_id(1).unwrap(); + let map = base.get_map("map"); + map.insert("seed", "base").unwrap(); + let update_base = base + .export(ExportMode::updates(&VersionVector::default())) + .unwrap(); + let version_base = base.oplog_vv(); + + let peer2 = LoroDoc::new_auto_commit(); + peer2.set_peer_id(2).unwrap(); + peer2.import(&update_base).unwrap(); + peer2.get_map("map").insert("p2", "B").unwrap(); + let update_peer2 = peer2.export(ExportMode::updates(&version_base)).unwrap(); + + let peer3 = LoroDoc::new_auto_commit(); + peer3.set_peer_id(3).unwrap(); + peer3.import(&update_base).unwrap(); + peer3.get_map("map").insert("p3", "C").unwrap(); + let update_peer3 = peer3.export(ExportMode::updates(&version_base)).unwrap(); + + let expected = LoroDoc::new(); + expected.import(&update_base).unwrap(); + expected.import(&update_peer2).unwrap(); + expected.import(&update_peer3).unwrap(); + + let dst = LoroDoc::new(); + dst.import(&update_peer2).unwrap(); + dst.import(&update_peer3).unwrap(); + assert_eq!(pending_len(&dst), 2); + + let vv_before_import = dst.oplog_vv(); + let frontiers_before_import: Frontiers = dst.oplog_frontiers(); + let state_before_import = dst.get_deep_value(); + + fail_next_import_state_apply_for_test(); + let err = dst.import(&update_base).unwrap_err(); + assert!( + err.to_string().contains("state apply failpoint"), + "unexpected error: {err:?}" + ); + assert_eq!(pending_len(&dst), 2); + assert_eq!(dst.oplog_vv(), vv_before_import); + assert_eq!(dst.oplog_frontiers(), frontiers_before_import); + assert_eq!(dst.get_deep_value(), state_before_import); + + dst.import(&update_base).unwrap(); + assert_eq!(pending_len(&dst), 0); + assert_eq!(dst.oplog_vv(), expected.oplog_vv()); + assert_eq!(dst.oplog_frontiers(), expected.oplog_frontiers()); + assert_eq!(dst.get_deep_value(), expected.get_deep_value()); +} + +#[test] +fn failed_import_keeps_multi_peer_frontiers_intact() { + let target = make_multi_peer_frontier_doc(); + let vv_before_import = target.oplog_vv(); + assert!(vv_before_import.iter().count() >= 3); + let frontiers_before_import = target.oplog_frontiers(); + let state_before_import = target.get_deep_value(); + + let (_, mut bad_json) = make_json_list_update_with_four_ops(4); + move_last_list_insert_far_out_of_bounds(&mut bad_json); + let bad_json = serde_json::to_string(&bad_json).unwrap(); + + let err = target.import_json_updates(&bad_json).unwrap_err(); + assert!( + err.to_string().contains("list diff"), + "expected state list bounds validation, got {err:?}" + ); + assert_eq!(target.oplog_vv(), vv_before_import); + assert_eq!(target.oplog_frontiers(), frontiers_before_import); + assert_eq!(target.get_deep_value(), state_before_import); +} + +#[test] +fn malformed_json_import_returns_error_without_mutating_doc() { + let doc = make_multi_peer_frontier_doc(); + let vv_before_import = doc.oplog_vv(); + let frontiers_before_import = doc.oplog_frontiers(); + let state_before_import = doc.get_deep_value(); + + let err = doc + .import_json_updates("[3,{ \"(' k\" :\n\n42222 }]") + .unwrap_err(); + assert_eq!(err, LoroError::InvalidJsonSchema); + assert_doc_unchanged( + &doc, + &vv_before_import, + &frontiers_before_import, + &state_before_import, + ); +} + +#[test] +fn failed_import_does_not_emit_events() { + let doc = LoroDoc::new(); + let hit = Arc::new(AtomicUsize::new(0)); + let hit_cloned = hit.clone(); + let _sub = doc.subscribe_root(Arc::new(move |_event| { + hit_cloned.fetch_add(1, Ordering::SeqCst); + })); + + let (_, mut bad_json) = make_json_list_update_with_four_ops(7); + move_last_list_insert_far_out_of_bounds(&mut bad_json); + let bad_json = serde_json::to_string(&bad_json).unwrap(); + let err = doc.import_json_updates(&bad_json).unwrap_err(); + assert!( + err.to_string().contains("list diff"), + "expected state list bounds validation, got {err:?}" + ); + assert_eq!(hit.load(Ordering::SeqCst), 0); + assert!(doc.drop_pending_events().is_empty()); + + let (_, good_json) = make_json_list_update_with_four_ops(8); + let good_json = serde_json::to_string(&good_json).unwrap(); + doc.import_json_updates(&good_json).unwrap(); + assert!(hit.load(Ordering::SeqCst) > 0); +} + +#[test] +fn corrupt_snapshot_import_rolls_back_empty_doc() { + let src = LoroDoc::new_auto_commit(); + src.set_peer_id(9).unwrap(); + src.get_text("text").insert_unicode(0, "snapshot").unwrap(); + src.get_list("list").push("value").unwrap(); + let snapshot = src.export(ExportMode::Snapshot).unwrap(); + let mut corrupt_snapshot = snapshot.clone(); + corrupt_snapshot_state_bytes(&mut corrupt_snapshot); + + let dst = LoroDoc::new(); + let vv_before_import = dst.oplog_vv(); + let frontiers_before_import = dst.oplog_frontiers(); + let state_before_import = dst.get_deep_value(); + let err = dst.import(&corrupt_snapshot).unwrap_err(); + assert!( + err.to_string().contains("decode_snapshot") + || err.to_string().contains("Decode") + || err.to_string().contains("snapshot"), + "unexpected error: {err:?}" + ); + assert_eq!(dst.oplog_vv(), vv_before_import); + assert_eq!(dst.oplog_frontiers(), frontiers_before_import); + assert_eq!(dst.get_deep_value(), state_before_import); + assert!(dst.oplog().lock().is_empty()); + + dst.import(&snapshot).unwrap(); + assert_eq!(dst.get_deep_value(), src.get_deep_value()); +} diff --git a/3rdparty/loro/crates/loro-internal/src/tests/mod.rs b/3rdparty/loro/crates/loro-internal/src/tests/mod.rs new file mode 100644 index 0000000000..83923c87eb --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/tests/mod.rs @@ -0,0 +1,36 @@ +#[ctor::ctor] +fn init() { + dev_utils::setup_test_log(); +} + +mod import_atomicity; +mod op_count; + +use crate::{op::ListSlice, LoroValue}; + +pub const PROPTEST_FACTOR_10: usize = 1; +pub const PROPTEST_FACTOR_1: usize = 0; + +#[test] +fn size_of() { + use crate::change::Change; + use crate::{ + container::{map::MapSet, ContainerID}, + id::ID, + op::{Op, RawOpContent}, + span::IdSpan, + InternalString, + }; + + println!("Change {}", std::mem::size_of::()); + println!("Op {}", std::mem::size_of::()); + println!("InsertContent {}", std::mem::size_of::()); + println!("MapSet {}", std::mem::size_of::()); + println!("ListSlice {}", std::mem::size_of::()); + println!("LoroValue {}", std::mem::size_of::()); + println!("ID {}", std::mem::size_of::()); + println!("Vec {}", std::mem::size_of::>()); + println!("IdSpan {}", std::mem::size_of::()); + println!("ContainerID {}", std::mem::size_of::()); + println!("InternalString {}", std::mem::size_of::()); +} diff --git a/3rdparty/loro/crates/loro-internal/src/tests/op_count.rs b/3rdparty/loro/crates/loro-internal/src/tests/op_count.rs new file mode 100644 index 0000000000..1980d30004 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/tests/op_count.rs @@ -0,0 +1,107 @@ +use crate::{ + loro::ExportMode, state::fail_next_import_state_apply_for_test, version::VersionVector, LoroDoc, +}; +use std::sync::atomic::Ordering::{Acquire, Release}; + +fn assert_visible_op_count_cache_matches_exact(doc: &LoroDoc) { + let exact = doc.oplog().lock().visible_op_count_exact(); + assert_eq!(doc.visible_op_count.load(Acquire), exact); + + let _state = doc.app_state().lock(); + assert_eq!(doc.len_ops(), exact); +} + +#[test] +fn len_ops_is_available_while_state_is_locked_with_local_ops() { + let doc = LoroDoc::new_auto_commit(); + doc.set_peer_id(1).unwrap(); + doc.get_map("map").insert("k", 1).unwrap(); + + let _state = doc.app_state().lock(); + assert_eq!(doc.len_ops(), 1); +} + +#[test] +fn len_ops_tracks_snapshot_import_while_state_is_locked() { + let src = LoroDoc::new_auto_commit(); + src.set_peer_id(2).unwrap(); + src.get_map("map").insert("k", 1).unwrap(); + src.commit_then_renew(); + let expected = src.len_ops(); + let snapshot = src.export(ExportMode::Snapshot).unwrap(); + + let dst = LoroDoc::new(); + dst.import(&snapshot).unwrap(); + + let _state = dst.app_state().lock(); + assert_eq!(dst.len_ops(), expected); +} + +#[test] +fn len_ops_prefers_exact_oplog_value_when_lock_order_allows() { + let doc = LoroDoc::new_auto_commit(); + doc.set_peer_id(3).unwrap(); + doc.get_map("map").insert("k", 1).unwrap(); + + doc.visible_op_count.store(0, Release); + assert_eq!(doc.len_ops(), 1); +} + +#[test] +fn visible_op_count_cache_tracks_pending_activation() { + let src = LoroDoc::new_auto_commit(); + src.set_peer_id(4).unwrap(); + let map = src.get_map("map"); + map.insert("base", "base").unwrap(); + let update_base = src + .export(ExportMode::updates(&VersionVector::default())) + .unwrap(); + let version_base = src.oplog_vv(); + + map.insert("later", "later").unwrap(); + let update_later = src.export(ExportMode::updates(&version_base)).unwrap(); + + let dst = LoroDoc::new(); + let status = dst.import(&update_later).unwrap(); + assert!(status.pending.is_some()); + assert_visible_op_count_cache_matches_exact(&dst); + assert_eq!(dst.len_ops(), 0); + + let status = dst.import(&update_base).unwrap(); + assert!(status.pending.is_none()); + assert_eq!(dst.len_ops(), src.len_ops()); + assert_visible_op_count_cache_matches_exact(&dst); +} + +#[test] +fn visible_op_count_cache_rolls_back_after_failed_pending_activation() { + let src = LoroDoc::new_auto_commit(); + src.set_peer_id(5).unwrap(); + let map = src.get_map("map"); + map.insert("base", "base").unwrap(); + let update_base = src + .export(ExportMode::updates(&VersionVector::default())) + .unwrap(); + let version_base = src.oplog_vv(); + + map.insert("later", "later").unwrap(); + let update_later = src.export(ExportMode::updates(&version_base)).unwrap(); + + let dst = LoroDoc::new(); + dst.import(&update_later).unwrap(); + assert_eq!(dst.len_ops(), 0); + assert_visible_op_count_cache_matches_exact(&dst); + + fail_next_import_state_apply_for_test(); + let err = dst.import(&update_base).unwrap_err(); + assert!( + err.to_string().contains("state apply failpoint"), + "unexpected error: {err:?}" + ); + assert_eq!(dst.len_ops(), 0); + assert_visible_op_count_cache_matches_exact(&dst); + + dst.import(&update_base).unwrap(); + assert_eq!(dst.len_ops(), src.len_ops()); + assert_visible_op_count_cache_matches_exact(&dst); +} diff --git a/3rdparty/loro/crates/loro-internal/src/txn.rs b/3rdparty/loro/crates/loro-internal/src/txn.rs new file mode 100644 index 0000000000..817b53cb1f --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/txn.rs @@ -0,0 +1,1018 @@ +use std::{ + borrow::Cow, + mem::take, + sync::{Arc, Weak}, +}; + +use enum_as_inner::EnumAsInner; +use generic_btree::rle::{HasLength as RleHasLength, Mergeable as GBSliceable}; +use loro_common::{ContainerType, IdLp, IdSpan, LoroResult}; +use loro_delta::{array_vec::ArrayVec, DeltaRopeBuilder}; +use rle::{HasLength, Mergable, RleVec, Sliceable}; +use rustc_hash::FxHashMap; +use smallvec::{smallvec, SmallVec}; + +use crate::{ + change::{Change, Lamport, Timestamp}, + container::{ + idx::ContainerIdx, + list::list_op::{DeleteSpan, InnerListOp}, + richtext::Style, + IntoContainerId, + }, + delta::{ResolvedMapDelta, ResolvedMapValue, StyleMeta, StyleMetaItem, TreeDiff, TreeDiffItem}, + encoding::export_fast_updates_in_range, + event::{Diff, ListDeltaMeta, TextDiff}, + handler::{Handler, ValueOrHandler}, + id::{Counter, PeerID, ID}, + lock::{LoroMutex, LoroMutexGuard}, + loro::CommitOptions, + op::{Op, RawOp, RawOpContent}, + pre_commit::{ChangeModifier, PreCommitCallbackPayload}, + span::HasIdSpan, + version::Frontiers, + ChangeMeta, InternalString, LoroDoc, LoroDocInner, LoroError, LoroValue, +}; + +use super::{ + arena::SharedArena, + event::{InternalContainerDiff, InternalDocDiff}, + handler::{ListHandler, MapHandler, TextHandler, TreeHandler}, + oplog::OpLog, + state::DocState, +}; + +impl crate::LoroDoc { + /// Create a new transaction. + /// Every ops created inside one transaction will be packed into a single + /// [Change]. + /// + /// There can only be one active transaction at a time for a [LoroDoc]. + #[inline(always)] + pub fn txn(&self) -> Result { + self.txn_with_origin("") + } + + /// Create a new transaction with specified origin. + /// + /// The origin will be propagated to the events. + /// There can only be one active transaction at a time for a [LoroDoc]. + pub fn txn_with_origin(&self, origin: &str) -> Result { + if !self.can_edit() { + return Err(LoroError::TransactionError( + String::from("LoroDoc is in readonly detached mode. To make it writable in detached mode, call `set_detached_editing(true)`.").into_boxed_str(), + )); + } + + let mut txn = Transaction::new_with_origin(self.inner.clone(), origin.into())?; + + let obs = self.observer.clone(); + let local_update_subs_weak = self.local_update_subs.downgrade(); + txn.set_on_commit(Box::new(move |state, oplog, id_span| { + let mut state = state.lock(); + let events = state.take_events(); + drop(state); + for event in events { + obs.emit(event); + } + + if id_span.atom_len() == 0 { + return; + } + + if let Some(local_update_subs) = local_update_subs_weak.upgrade() { + if !local_update_subs.inner().is_empty() { + let bytes = { export_fast_updates_in_range(&oplog.lock(), &[id_span]) }; + local_update_subs.emit(&(), bytes); + } + } + })); + + Ok(txn) + } + + pub fn start_auto_commit(&self) { + self.auto_commit + .store(true, std::sync::atomic::Ordering::Release); + let mut self_txn = self.txn.lock(); + if self_txn.is_some() || !self.can_edit() { + return; + } + + let txn = self + .txn() + .expect("auto-commit should be able to create a transaction"); + self_txn.replace(txn); + } + + #[inline] + pub fn renew_txn_if_auto_commit(&self, options: Option) { + if self.auto_commit.load(std::sync::atomic::Ordering::Acquire) && self.can_edit() { + let mut self_txn = self.txn.lock(); + if self_txn.is_some() { + return; + } + + let mut txn = self + .txn() + .expect("auto-commit should be able to renew a transaction"); + if let Some(options) = options { + txn.set_options(options); + } + self_txn.replace(txn); + } + } + + #[inline] + pub(crate) fn _renew_txn_if_auto_commit_with_guard( + &self, + options: Option, + mut guard: LoroMutexGuard>, + ) { + if self.auto_commit.load(std::sync::atomic::Ordering::Acquire) && self.can_edit() { + if guard.is_some() { + return; + } + + let mut txn = self + .txn() + .expect("auto-commit should be able to renew a transaction"); + if let Some(options) = options { + txn.set_options(options); + } + guard.replace(txn); + } + } +} + +pub(crate) type OnCommitFn = + Box>, &Arc>, IdSpan) + Sync + Send>; + +pub struct Transaction { + peer: PeerID, + origin: InternalString, + start_counter: Counter, + next_counter: Counter, + start_lamport: Lamport, + next_lamport: Lamport, + doc: Weak, + frontiers: Frontiers, + local_ops: RleVec<[Op; 1]>, // TODO: use a more efficient data structure + event_hints: FxHashMap>, + pub(super) arena: SharedArena, + finished: bool, + on_commit: Option, + timestamp: Option, + msg: Option>, + latest_timestamp: Timestamp, + pub(super) is_peer_first_appearance: bool, +} + +impl std::fmt::Debug for Transaction { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Transaction") + .field("peer", &self.peer) + .field("origin", &self.origin) + .field("start_counter", &self.start_counter) + .field("next_counter", &self.next_counter) + .field("start_lamport", &self.start_lamport) + .field("next_lamport", &self.next_lamport) + .field("frontiers", &self.frontiers) + .field("local_ops", &self.local_ops) + .field("event_hints", &self.event_hints) + .field("arena", &self.arena) + .field("finished", &self.finished) + .field("on_commit", &self.on_commit.is_some()) + .field("timestamp", &self.timestamp) + .finish() + } +} + +/// We can infer local events directly from the local behavior. This enum is used to +/// record them, so that we can avoid recalculate them when we commit the transaction. +/// +/// For example, when we insert a text in wasm, users use the utf16 index to send the +/// command. However, internally loro will convert it to unicode index. But the users +/// still need events that are in utf16 index. To avoid the round trip, we record the +/// events here. +#[derive(Debug, Clone, EnumAsInner)] +pub(super) enum EventHint { + Mark { + start: u32, + end: u32, + style: Style, + }, + InsertText { + /// pos is a Unicode index. If wasm, it's a UTF-16 index. + pos: u32, + event_len: u32, + unicode_len: u32, + styles: StyleMeta, + }, + /// pos is a Unicode index. If wasm, it's a UTF-16 index. + DeleteText { + span: DeleteSpan, + unicode_len: usize, + }, + InsertList { + len: u32, + pos: usize, + }, + SetList { + index: usize, + value: LoroValue, + }, + Move { + value: LoroValue, + from: u32, + to: u32, + }, + DeleteList(DeleteSpan), + Map { + key: InternalString, + value: Option, + }, + // use vec because we could bring back some node that has children + Tree(SmallVec<[TreeDiffItem; 1]>), + MarkEnd, + #[cfg(feature = "counter")] + Counter(f64), +} + +impl generic_btree::rle::HasLength for EventHint { + fn rle_len(&self) -> usize { + match self { + EventHint::Mark { .. } => 1, + EventHint::InsertText { + unicode_len: len, .. + } => *len as usize, + EventHint::DeleteText { unicode_len, .. } => *unicode_len, + EventHint::InsertList { len, .. } => *len as usize, + EventHint::DeleteList(d) => d.len(), + EventHint::Map { .. } => 1, + EventHint::Tree(_) => 1, + EventHint::MarkEnd => 1, + EventHint::Move { .. } => 1, + EventHint::SetList { .. } => 1, + #[cfg(feature = "counter")] + EventHint::Counter(_) => 1, + } + } +} + +impl generic_btree::rle::Mergeable for EventHint { + fn can_merge(&self, rhs: &Self) -> bool { + match (self, rhs) { + ( + EventHint::InsertText { + pos, + unicode_len: _, + event_len, + styles, + }, + EventHint::InsertText { + pos: r_pos, + styles: r_styles, + .. + }, + ) => *pos + *event_len == *r_pos && styles == r_styles, + (EventHint::InsertList { pos, len }, EventHint::InsertList { pos: pos_right, .. }) => { + pos + *len as usize == *pos_right + } + // We don't merge delete text because it's hard to infer the correct pos to split: + // `range` param is in unicode range, but the delete text event is in UTF-16 range. + // Without the original text, it's impossible to convert the range. + (EventHint::DeleteText { span, .. }, EventHint::DeleteText { span: r, .. }) => { + span.is_mergable(r, &()) + } + (EventHint::DeleteList(l), EventHint::DeleteList(r)) => l.is_mergable(r, &()), + _ => false, + } + } + + fn merge_right(&mut self, rhs: &Self) { + match (self, rhs) { + ( + EventHint::InsertText { + event_len, + unicode_len: len, + .. + }, + EventHint::InsertText { + event_len: r_event_len, + unicode_len: r_len, + .. + }, + ) => { + *len += *r_len; + *event_len += *r_event_len; + } + ( + EventHint::InsertList { len, pos: _ }, + EventHint::InsertList { len: r_len, pos: _ }, + ) => *len += *r_len, + (EventHint::DeleteList(l), EventHint::DeleteList(r)) => l.merge(r, &()), + ( + EventHint::DeleteText { span, unicode_len }, + EventHint::DeleteText { + span: r_span, + unicode_len: r_len, + }, + ) => { + *unicode_len += *r_len; + span.merge(r_span, &()); + } + _ => unreachable!(), + } + } + + fn merge_left(&mut self, _: &Self) { + unreachable!() + } +} + +impl Transaction { + #[inline] + pub fn new(doc: Arc) -> LoroResult { + Self::new_with_origin(doc.clone(), "".into()) + } + + pub fn new_with_origin(doc: Arc, origin: InternalString) -> LoroResult { + let oplog_lock = doc.oplog.lock(); + let mut state_lock = doc.state.lock(); + if state_lock.is_in_txn() { + return Err(LoroError::DuplicatedTransactionError); + } + + state_lock.start_txn(origin, crate::event::EventTriggerKind::Local); + let arena = state_lock.arena.clone(); + let frontiers = state_lock.frontiers.clone(); + let peer = state_lock.peer.load(std::sync::atomic::Ordering::Relaxed); + let next_counter = oplog_lock.next_id(peer).counter; + let next_lamport = oplog_lock.dag.frontiers_to_next_lamport(&frontiers); + let latest_timestamp = oplog_lock.get_greatest_timestamp(&frontiers); + if let Err(err) = + oplog_lock.check_change_greater_than_last_peer_id(peer, next_counter, &frontiers) + { + state_lock.abort_txn(); + return Err(err); + } + drop(state_lock); + drop(oplog_lock); + Ok(Self { + peer, + doc: Arc::downgrade(&doc), + arena, + frontiers, + timestamp: None, + next_counter, + next_lamport, + origin: Default::default(), + start_counter: next_counter, + start_lamport: next_lamport, + event_hints: Default::default(), + local_ops: RleVec::new(), + finished: false, + on_commit: None, + msg: None, + latest_timestamp, + is_peer_first_appearance: false, + }) + } + + pub fn set_origin(&mut self, origin: InternalString) { + self.origin = origin; + } + + pub fn set_timestamp(&mut self, time: Timestamp) { + self.timestamp = Some(time); + } + + pub fn set_msg(&mut self, msg: Option>) { + self.msg = msg; + } + + pub fn local_ops(&self) -> &RleVec<[Op; 1]> { + &self.local_ops + } + + pub fn peer(&self) -> &PeerID { + &self.peer + } + + pub fn timestamp(&self) -> &Option { + &self.timestamp + } + + pub fn frontiers(&self) -> &Frontiers { + &self.frontiers + } + pub fn msg(&self) -> &Option> { + &self.msg + } + + pub fn lamport(&self) -> &Lamport { + &self.start_lamport + } + + pub(crate) fn set_on_commit(&mut self, f: OnCommitFn) { + self.on_commit = Some(f); + } + + pub(crate) fn take_on_commit(&mut self) -> Option { + self.on_commit.take() + } + + pub fn commit(mut self) -> Result, LoroError> { + self._commit() + } + + #[tracing::instrument(level = "debug", skip(self))] + fn _commit(&mut self) -> Result, LoroError> { + if self.finished { + return Ok(None); + } + + let Some(doc) = self.doc.upgrade() else { + return Ok(None); + }; + self.finished = true; + if self.local_ops.is_empty() { + let mut state = doc.state.lock(); + state.abort_txn(); + return Ok(Some(self.take_options())); + } + + let ops = std::mem::take(&mut self.local_ops); + let deps = take(&mut self.frontiers); + let change = Change { + lamport: self.start_lamport, + ops, + deps, + id: ID::new(self.peer, self.start_counter), + timestamp: self.latest_timestamp.max( + self.timestamp + .unwrap_or_else(|| doc.oplog.lock().get_timestamp_for_next_txn()), + ), + commit_msg: take(&mut self.msg), + }; + + let change_meta = ChangeMeta::from_change(&change); + { + // add change to uncommit field of oplog + let mut oplog = doc.oplog.lock(); + oplog.set_uncommitted_change(change); + } + + let modifier = ChangeModifier::default(); + doc.pre_commit_subs.emit( + &(), + PreCommitCallbackPayload { + change_meta, + origin: self.origin.to_string(), + modifier: modifier.clone(), + }, + ); + + let mut oplog = doc.oplog.lock(); + let mut state = doc.state.lock(); + + let Some(mut change) = oplog.uncommitted_change.take() else { + state.abort_txn(); + drop(state); + drop(oplog); + return Err(LoroError::internal( + "missing uncommitted change while committing transaction", + )); + }; + modifier.modify_change(&mut change); + let diff = if state.is_recording() { + Some(change_to_diff( + &change, + doc.clone(), + std::mem::take(&mut self.event_hints), + )) + } else { + None + }; + + let last_id = change.id_last(); + if let Err(err) = oplog.import_local_change(change) { + state.abort_txn(); + drop(state); + drop(oplog); + return Err(err); + } + + state.commit_txn( + Frontiers::from_id(last_id), + diff.map(|arr| InternalDocDiff { + by: crate::event::EventTriggerKind::Local, + origin: self.origin.clone(), + diff: Cow::Owned( + arr.into_iter() + .map(|x| InternalContainerDiff { + idx: x.idx, + bring_back: false, + diff: (x.diff.into()), + diff_mode: crate::diff_calc::DiffMode::Linear, + }) + .collect(), + ), + new_version: Cow::Borrowed(oplog.frontiers()), + }), + ); + drop(state); + drop(oplog); + if let Some(on_commit) = self.on_commit.take() { + assert!(!doc.txn.is_locked()); + on_commit(&doc.state.clone(), &doc.oplog.clone(), self.id_span()); + } + Ok(None) + } + + fn take_options(&self) -> CommitOptions { + let mut options = CommitOptions::new(); + if !self.origin.is_empty() { + options = options.origin(self.origin.as_str()); + } + if let Some(msg) = self.msg.as_ref() { + options = options.commit_msg(msg); + } + if let Some(timestamp) = self.timestamp { + options = options.timestamp(timestamp); + } + options + } + + pub(super) fn apply_local_op( + &mut self, + container: ContainerIdx, + content: RawOpContent, + event: EventHint, + // check whether context and txn are referring to the same state context + doc: &LoroDoc, + ) -> LoroResult<()> { + // A transaction must be applied to the document it was created for. + // `insert_with_txn`/`delete_with_txn` are public API, so this guards + // against misuse (passing one doc's txn to another doc's handler), which + // would otherwise stamp the target doc's state/oplog with the wrong + // peer+counter. Compare doc identity via the `Weak` pointer (no atomic + // upgrade) on the hot path; only upgrade to fill in the error on the + // rare mismatch. + if self.doc.as_ptr() != Arc::as_ptr(&doc.inner) { + let found = doc + .state + .lock() + .peer + .load(std::sync::atomic::Ordering::Relaxed); + let expected = self + .doc + .upgrade() + .map(|d| d.state.lock().peer.load(std::sync::atomic::Ordering::Relaxed)) + .unwrap_or(found); + return Err(LoroError::UnmatchedContext { expected, found }); + } + + let len = content.content_len(); + assert!(len > 0); + let raw_op = RawOp { + id: ID { + peer: self.peer, + counter: self.next_counter, + }, + lamport: self.next_lamport, + container, + content, + }; + + let mut oplog = doc.oplog.lock(); + let mut state = doc.state.lock(); + if state.is_deleted(container) { + return Err(LoroError::ContainerDeleted { + container: Box::new(state.arena.idx_to_id(container).unwrap()), + }); + } + + let op = self.arena.convert_raw_op(&raw_op); + state.apply_local_op(&raw_op, &op)?; + { + if !self.is_peer_first_appearance && !oplog.dag.latest_vv_contains_peer(self.peer) { + self.is_peer_first_appearance = true; + } + // update version info + let dep_id = Frontiers::from_id(ID::new(self.peer, self.next_counter - 1)); + let start_id = ID::new(self.peer, self.next_counter); + self.next_counter += len as Counter; + oplog.dag.update_version_on_new_local_op( + if self.local_ops.is_empty() { + &self.frontiers + } else { + &dep_id + }, + start_id, + self.next_lamport, + len, + ); + // Local ops are always visible; bump the cached count incrementally + // instead of recomputing it from the version vectors every op. + oplog.inc_visible_op_count(len); + self.next_lamport += len as Lamport; + // set frontiers to the last op id + let last_id = start_id.inc(len as Counter - 1); + state.frontiers = Frontiers::from_id(last_id); + }; + drop(state); + drop(oplog); + debug_assert_eq!( + event.rle_len(), + op.atom_len(), + "event:{:#?} \nop:{:#?}", + &event, + &op + ); + + let container_hints = self.event_hints.entry(container).or_default(); + + match container_hints.last_mut() { + Some(last) if last.can_merge(&event) => { + last.merge_right(&event); + } + _ => { + container_hints.push(event); + } + } + self.local_ops.push(op); + Ok(()) + } + + /// id can be a str, ContainerID, or ContainerIdRaw. + /// if it's str it will use Root container, which will not be None + pub fn get_text(&self, id: I) -> TextHandler { + let id = id.into_container_id(&self.arena, ContainerType::Text); + Handler::new_attached(id, LoroDoc::from_inner(self.doc.upgrade().unwrap())) + .into_text() + .unwrap() + } + + /// id can be a str, ContainerID, or ContainerIdRaw. + /// if it's str it will use Root container, which will not be None + pub fn get_list(&self, id: I) -> ListHandler { + let id = id.into_container_id(&self.arena, ContainerType::List); + Handler::new_attached(id, LoroDoc::from_inner(self.doc.upgrade().unwrap())) + .into_list() + .unwrap() + } + + /// id can be a str, ContainerID, or ContainerIdRaw. + /// if it's str it will use Root container, which will not be None + pub fn get_map(&self, id: I) -> MapHandler { + let id = id.into_container_id(&self.arena, ContainerType::Map); + Handler::new_attached(id, LoroDoc::from_inner(self.doc.upgrade().unwrap())) + .into_map() + .unwrap() + } + + /// id can be a str, ContainerID, or ContainerIdRaw. + /// if it's str it will use Root container, which will not be None + pub fn get_tree(&self, id: I) -> TreeHandler { + let id = id.into_container_id(&self.arena, ContainerType::Tree); + Handler::new_attached(id, LoroDoc::from_inner(self.doc.upgrade().unwrap())) + .into_tree() + .unwrap() + } + pub fn next_id(&self) -> ID { + ID { + peer: self.peer, + counter: self.next_counter, + } + } + + #[inline] + pub fn id_span(&self) -> IdSpan { + IdSpan::new(self.peer, self.start_counter, self.next_counter) + } + + pub fn next_idlp(&self) -> IdLp { + IdLp { + peer: self.peer, + lamport: self.next_lamport, + } + } + + pub fn is_empty(&self) -> bool { + self.local_ops.is_empty() + } + + pub(crate) fn len(&self) -> usize { + (self.next_counter - self.start_counter) as usize + } + + pub(crate) fn set_options(&mut self, options: CommitOptions) { + self.origin = options.origin.unwrap_or_default(); + self.msg = options.commit_msg; + self.timestamp = options.timestamp; + } + + pub(crate) fn set_default_options(&mut self, default_options: crate::loro::CommitOptions) { + if self.origin.is_empty() { + self.origin = default_options.origin.unwrap_or_default(); + } + if self.msg.is_none() { + self.msg = default_options.commit_msg; + } + if self.timestamp.is_none() { + self.timestamp = default_options.timestamp; + } + } +} + +impl Drop for Transaction { + #[tracing::instrument(level = "debug", skip(self))] + fn drop(&mut self) { + if !self.finished { + // TODO: should we abort here or commit here? + // what if commit fails? + self._commit().unwrap(); + } + } +} + +#[derive(Debug, Clone)] +pub(crate) struct TxnContainerDiff { + pub(crate) idx: ContainerIdx, + pub(crate) diff: Diff, +} + +// PERF: could be compacter +fn change_to_diff( + change: &Change, + doc: Arc, + event_hints: FxHashMap>, +) -> Vec { + let mut ans: Vec = Vec::with_capacity(change.ops.len()); + let peer = change.id.peer; + let mut lamport = change.lamport; + + // Group ops by container first to match our new structure + let mut ops_by_container: FxHashMap> = FxHashMap::default(); + for op in change.ops.iter() { + ops_by_container + .entry(op.container) + .or_default() + .push(op.clone()); + } + + // Process each container's hints and ops together + for (container_idx, hints) in event_hints { + let Some(container_ops) = ops_by_container.get_mut(&container_idx) else { + continue; + }; + + let mut op_index = 0; + let mut hint_iter = hints.into_iter(); + let mut current_hint = hint_iter.next(); + + while op_index < container_ops.len() { + let Some(hint) = current_hint.take() else { + unreachable!("Missing hint for op"); + }; + + // Collect ops that belong to this hint + let mut ops_for_hint: SmallVec<[Op; 1]> = smallvec![container_ops[op_index].clone()]; + let mut total_len = container_ops[op_index].atom_len(); + + // If hint spans multiple ops, collect them + while total_len < hint.rle_len() { + op_index += 1; + let next_op_len = container_ops[op_index].atom_len(); + let op = if next_op_len + total_len > hint.rle_len() { + let new_len = hint.rle_len() - total_len; + let left = container_ops[op_index].slice(0, new_len); + let right = container_ops[op_index].slice(new_len, next_op_len); + container_ops[op_index] = right; + op_index -= 1; + left + } else { + container_ops[op_index].clone() + }; + + total_len += op.atom_len(); + ops_for_hint.push(op); + } + + op_index += 1; + assert_eq!(total_len, hint.rle_len(), "Op/hint length mismatch"); + + // Move to next hint + current_hint = hint_iter.next(); + + // Generate diff based on hint type + match hint { + EventHint::Mark { start, end, style } => { + let mut meta = StyleMeta::default(); + meta.insert( + style.key.clone(), + StyleMetaItem { + lamport, + peer: change.id.peer, + value: style.data, + }, + ); + let diff = DeltaRopeBuilder::new() + .retain(start as usize, Default::default()) + .retain( + (end - start) as usize, + meta.to_option_map().unwrap_or_default().into(), + ) + .build(); + ans.push(TxnContainerDiff { + idx: container_idx, + diff: Diff::Text(diff), + }); + } + EventHint::InsertText { styles, pos, .. } => { + let mut delta: TextDiff = DeltaRopeBuilder::new() + .retain(pos as usize, Default::default()) + .build(); + for op in ops_for_hint.iter() { + let InnerListOp::InsertText { slice, .. } = op.content.as_list().unwrap() + else { + unreachable!() + }; + + delta.push_insert( + slice.clone().into(), + styles.to_option_map().unwrap_or_default().into(), + ); + } + ans.push(TxnContainerDiff { + idx: container_idx, + diff: Diff::Text(delta), + }) + } + EventHint::DeleteText { + span, + unicode_len: _, + // we don't need to iter over ops here, because we already + // know what the events should be + } => ans.push(TxnContainerDiff { + idx: container_idx, + diff: Diff::Text( + DeltaRopeBuilder::new() + .retain(span.start() as usize, Default::default()) + .delete(span.len()) + .build(), + ), + }), + EventHint::InsertList { pos, .. } => { + // We should use pos from event hint because index in op may + // be using op index for the MovableList + let mut index = pos; + for op in ops_for_hint.iter() { + let (range, _) = op.content.as_list().unwrap().as_insert().unwrap(); + let values = doc + .arena + .get_values(range.to_range()) + .into_iter() + .map(|v| ValueOrHandler::from_value(v, &doc)); + let len = values.len(); + ans.push(TxnContainerDiff { + idx: container_idx, + diff: Diff::List( + DeltaRopeBuilder::new() + .retain(index, Default::default()) + .insert_many(values, Default::default()) + .build(), + ), + }); + index += len; + } + } + EventHint::DeleteList(s) => { + ans.push(TxnContainerDiff { + idx: container_idx, + diff: Diff::List( + DeltaRopeBuilder::new() + .retain(s.start() as usize, Default::default()) + .delete(s.len()) + .build(), + ), + }); + } + EventHint::Map { key, value } => { + // Mirror `MapState::apply_diff_and_convert`: a marker write surfaces to + // subscribers as the deterministic Container, not the raw binary value. + let parent_id = doc.arena.idx_to_id(container_idx); + let value = value.map(|v| match parent_id.as_ref() { + Some(pid) => { + loro_common::translate_mergeable_marker_value(pid, key.as_ref(), v) + } + None => v, + }); + ans.push(TxnContainerDiff { + idx: container_idx, + diff: Diff::Map(ResolvedMapDelta::new().with_entry( + key, + ResolvedMapValue { + value: value.map(|v| ValueOrHandler::from_value(v, &doc)), + idlp: IdLp::new(peer, lamport), + }, + )), + }); + } + EventHint::Tree(tree_diff) => { + let mut diff = TreeDiff::default(); + diff.diff.extend(tree_diff); + ans.push(TxnContainerDiff { + idx: container_idx, + diff: Diff::Tree(diff), + }); + } + EventHint::Move { from, to, value } => { + let mut a = DeltaRopeBuilder::new() + .retain(from as usize, Default::default()) + .delete(1) + .build(); + a.compose( + &DeltaRopeBuilder::new() + .retain(to as usize, Default::default()) + .insert( + ArrayVec::from([ValueOrHandler::from_value(value, &doc)]), + ListDeltaMeta { from_move: true }, + ) + .build(), + ); + ans.push(TxnContainerDiff { + idx: container_idx, + diff: Diff::List(a), + }); + } + EventHint::SetList { index, value } => { + ans.push(TxnContainerDiff { + idx: container_idx, + diff: Diff::List( + DeltaRopeBuilder::new() + .retain(index, Default::default()) + .delete(1) + .insert( + ArrayVec::from([ValueOrHandler::from_value(value, &doc)]), + Default::default(), + ) + .build(), + ), + }); + } + EventHint::MarkEnd => { + // do nothing + } + #[cfg(feature = "counter")] + EventHint::Counter(diff) => { + ans.push(TxnContainerDiff { + idx: container_idx, + diff: Diff::Counter(diff), + }); + } + } + + // Update lamport for this hint's operations + lamport += ops_for_hint + .iter() + .map(|x| x.content_len() as Lamport) + .sum::(); + } + } + + ans +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{cursor::PosType, version::Frontiers}; + + #[test] + fn txn_creation_rolls_back_in_txn_after_peer_conflict() { + let doc = LoroDoc::new(); + doc.set_detached_editing(true); + doc.set_peer_id(7).unwrap(); + + let text = doc.get_text("text"); + let mut txn = doc.txn().unwrap(); + text.insert_with_txn(&mut txn, 0, "a", PosType::Unicode) + .unwrap(); + txn.commit().unwrap(); + + doc.checkout(&Frontiers::default()).unwrap(); + doc.set_peer_id(7).unwrap(); + + let err = doc + .txn() + .expect_err("stale detached frontiers should reject reusing the same peer"); + assert!(matches!( + err, + LoroError::ConcurrentOpsWithSamePeerID { peer: 7, .. } + )); + assert!(!doc.app_state().lock().is_in_txn()); + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/undo.rs b/3rdparty/loro/crates/loro-internal/src/undo.rs new file mode 100644 index 0000000000..e135e7462a --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/undo.rs @@ -0,0 +1,1086 @@ +use std::{cell::RefCell, collections::VecDeque, sync::Arc}; + +use crate::sync::{AtomicU64, Mutex}; +use either::Either; +use loro_common::{ + ContainerID, Counter, CounterSpan, HasIdSpan, IdSpan, LoroError, LoroResult, LoroValue, PeerID, +}; +use parking_lot::lock_api::ReentrantMutex; +use rustc_hash::{FxHashMap, FxHashSet}; +use tracing::{debug_span, info_span, instrument}; + +use crate::{ + change::{get_sys_timestamp, Timestamp}, + cursor::{AbsolutePosition, Cursor}, + delta::TreeExternalDiff, + event::{Diff, EventTriggerKind}, + version::Frontiers, + ContainerDiff, DiffEvent, DocDiff, LoroDoc, Subscription, +}; + +/// A batch of diffs. +/// +/// You can use `loroDoc.apply_diff(diff)` to apply the diff to the document. +#[derive(Debug, Clone, Default)] +pub struct DiffBatch { + pub cid_to_events: FxHashMap, + pub order: Vec, +} + +impl DiffBatch { + pub fn new(diff: Vec) -> Self { + let mut map: FxHashMap = Default::default(); + let mut order: Vec = Vec::with_capacity(diff.len()); + for d in diff.into_iter() { + for item in d.diff.into_iter() { + let old = map.insert(item.id.clone(), item.diff); + assert!(old.is_none(), "Duplicate container ID in diff events"); + order.push(item.id.clone()); + } + } + + Self { + cid_to_events: map, + order, + } + } + + pub fn compose(&mut self, other: &Self) { + if other.cid_to_events.is_empty() { + return; + } + + for (id, diff) in other.iter() { + if let Some(this_diff) = self.cid_to_events.get_mut(id) { + this_diff.compose_ref(diff); + } else { + self.cid_to_events.insert(id.clone(), diff.clone()); + self.order.push(id.clone()); + } + } + } + + pub fn transform(&mut self, other: &Self, left_priority: bool) { + if other.cid_to_events.is_empty() || self.cid_to_events.is_empty() { + return; + } + + for (idx, diff) in self.cid_to_events.iter_mut() { + if let Some(b_diff) = other.cid_to_events.get(idx) { + diff.transform(b_diff, left_priority); + } + } + } + + pub fn clear(&mut self) { + self.cid_to_events.clear(); + self.order.clear(); + } + + pub fn iter(&self) -> impl Iterator + '_ { + self.order + .iter() + .map(|cid| (cid, self.cid_to_events.get(cid).unwrap())) + } + + #[allow(clippy::should_implement_trait)] + pub fn into_iter(self) -> impl Iterator { + let mut cid_to_events = self.cid_to_events; + self.order.into_iter().map(move |cid| { + let d = cid_to_events.remove(&cid).unwrap(); + (cid, d) + }) + } +} + +fn transform_cursor( + cursor_with_pos: &mut CursorWithPos, + remote_diff: &DiffBatch, + doc: &LoroDoc, + container_remap: &FxHashMap, +) { + let mut container_changed = false; + let mut cid = &cursor_with_pos.cursor.container; + while let Some(new_cid) = container_remap.get(cid) { + cid = new_cid; + container_changed = true; + } + + if cursor_with_pos.cursor.id.is_none() { + // We don't need to transform a cursor that always points to the leftmost or rightmost position + if container_changed { + cursor_with_pos.cursor.container = cid.clone(); + } + return; + } + + if let Some(diff) = remote_diff.cid_to_events.get(cid) { + let new_pos = diff.transform_cursor(cursor_with_pos.pos.pos, false); + cursor_with_pos.pos.pos = new_pos; + }; + + let new_pos = cursor_with_pos.pos.pos; + match doc.get_handler(cid.clone()).unwrap() { + crate::handler::Handler::Text(h) => { + let Some(new_cursor) = h.get_cursor_internal(new_pos, cursor_with_pos.pos.side, false) + else { + return; + }; + + cursor_with_pos.cursor = new_cursor; + } + crate::handler::Handler::List(h) => { + let Some(new_cursor) = h.get_cursor(new_pos, cursor_with_pos.pos.side) else { + return; + }; + + cursor_with_pos.cursor = new_cursor; + } + crate::handler::Handler::MovableList(h) => { + let Some(new_cursor) = h.get_cursor(new_pos, cursor_with_pos.pos.side) else { + return; + }; + + cursor_with_pos.cursor = new_cursor; + } + crate::handler::Handler::Map(_) => {} + crate::handler::Handler::Tree(_) => {} + crate::handler::Handler::Unknown(_) => {} + #[cfg(feature = "counter")] + crate::handler::Handler::Counter(_) => {} + } +} + +/// UndoManager is responsible for managing undo/redo from the current peer's perspective. +/// +/// Undo/local is local: it cannot be used to undone the changes made by other peers. +/// If you want to undo changes made by other peers, you may need to use the time travel feature. +/// +/// PeerID cannot be changed during the lifetime of the UndoManager +pub struct UndoManager { + peer: Arc, + container_remap: Arc>>, + inner: Arc>>, + _peer_id_change_sub: Subscription, + _undo_sub: Subscription, + doc: LoroDoc, +} + +impl std::fmt::Debug for UndoManager { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("UndoManager") + .field("peer", &self.peer) + .field("container_remap", &self.container_remap) + .field("inner", &self.inner) + .finish() + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum UndoOrRedo { + Undo, + Redo, +} + +impl UndoOrRedo { + fn opposite(&self) -> UndoOrRedo { + match self { + Self::Undo => Self::Redo, + Self::Redo => Self::Undo, + } + } +} + +/// When a undo/redo item is pushed, the undo manager will call the on_push callback to get the meta data of the undo item. +/// The returned cursors will be recorded for a new pushed undo item. +pub type OnPush = Box< + dyn for<'a> Fn(UndoOrRedo, CounterSpan, Option>) -> UndoItemMeta + Send + Sync, +>; +pub type OnPop = Box; + +struct UndoManagerInner { + next_counter: Option, + undo_stack: Stack, + redo_stack: Stack, + processing_undo: bool, + last_undo_time: i64, + merge_interval_in_ms: i64, + max_stack_size: usize, + exclude_origin_prefixes: Vec>, + last_popped_selection: Option>, + on_push: Option, + on_pop: Option, + group: Option, +} + +impl std::fmt::Debug for UndoManagerInner { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("UndoManagerInner") + .field("latest_counter", &self.next_counter) + .field("undo_stack", &self.undo_stack) + .field("redo_stack", &self.redo_stack) + .field("processing_undo", &self.processing_undo) + .field("last_undo_time", &self.last_undo_time) + .field("merge_interval", &self.merge_interval_in_ms) + .field("max_stack_size", &self.max_stack_size) + .field("exclude_origin_prefixes", &self.exclude_origin_prefixes) + .field("group", &self.group) + .finish() + } +} + +#[derive(Debug, Clone, Default)] +struct UndoGroup { + start_counter: Counter, + affected_cids: FxHashSet, +} + +impl UndoGroup { + pub fn new(start_counter: Counter) -> Self { + Self { + start_counter, + affected_cids: Default::default(), + } + } +} + +#[derive(Debug)] +struct Stack { + stack: VecDeque<(VecDeque, Arc>)>, + size: usize, +} + +#[derive(Debug, Clone)] +struct StackItem { + span: CounterSpan, + meta: UndoItemMeta, +} + +/// The metadata of an undo item. +/// +/// The cursors inside the metadata will be transformed by remote operations as well. +/// So that when the item is popped, users can restore the cursors position correctly. +#[derive(Debug, Default, Clone)] +pub struct UndoItemMeta { + pub value: LoroValue, + pub cursors: Vec, +} + +#[derive(Debug, Clone)] +pub struct CursorWithPos { + pub cursor: Cursor, + pub pos: AbsolutePosition, +} + +impl UndoItemMeta { + pub fn new() -> Self { + Self { + value: LoroValue::Null, + cursors: Default::default(), + } + } + + /// It's assumed that the cursor is just acquired before the ops that + /// need to be undo/redo. + /// + /// We need to rely on the validity of the original pos value + pub fn add_cursor(&mut self, cursor: &Cursor) { + self.cursors.push(CursorWithPos { + cursor: cursor.clone(), + pos: AbsolutePosition { + pos: cursor.origin_pos, + side: cursor.side, + }, + }); + } + + pub fn set_value(&mut self, value: LoroValue) { + self.value = value; + } +} + +impl Stack { + pub fn new() -> Self { + let mut stack = VecDeque::new(); + stack.push_back((VecDeque::new(), Arc::new(Mutex::new(Default::default())))); + Stack { stack, size: 0 } + } + + /// Peek the top-most StackItem's metadata without modifying the stack. + /// + /// Returns None if the stack is empty. + fn peek_top_meta(&self) -> Option { + if self.is_empty() { + return None; + } + + for (items, _) in self.stack.iter().rev() { + if let Some(item) = items.back() { + return Some(item.meta.clone()); + } + } + + None + } + + pub fn pop(&mut self) -> Option<(StackItem, Arc>)> { + while self.stack.back().unwrap().0.is_empty() && self.stack.len() > 1 { + let (_, diff) = self.stack.pop_back().unwrap(); + let diff = diff.lock(); + if !diff.cid_to_events.is_empty() { + self.stack.back_mut().unwrap().1.lock().compose(&diff); + } + } + + if self.stack.len() == 1 && self.stack.back().unwrap().0.is_empty() { + // If the stack is empty, we need to clear the remote diff + self.stack.back_mut().unwrap().1.lock().clear(); + return None; + } + + self.size -= 1; + let last = self.stack.back_mut().unwrap(); + last.0.pop_back().map(|x| (x, last.1.clone())) + // If this row in stack is empty, we don't pop it right away + // Because we still need the remote diff to be available. + // Cursor position transformation relies on the remote diff in the same row. + } + + pub fn push(&mut self, span: CounterSpan, meta: UndoItemMeta) { + self.push_with_merge(span, meta, false, None) + } + + pub fn push_with_merge( + &mut self, + span: CounterSpan, + meta: UndoItemMeta, + can_merge: bool, + group: Option<&UndoGroup>, + ) { + let last = self.stack.back_mut().unwrap(); + let last_remote_diff = last.1.lock(); + + // Check if the remote diff is disjoint with the current undo group + let is_disjoint_group = group.is_some_and(|g| { + g.affected_cids.iter().all(|cid| { + last_remote_diff + .cid_to_events + .get(cid) + .is_none_or(|diff| diff.is_empty()) + }) + }); + + // Can't merge if remote diffs exist and it's not disjoint with the current undo group + let should_create_new_entry = + !last_remote_diff.cid_to_events.is_empty() && !is_disjoint_group; + + if should_create_new_entry { + // Create a new entry in the stack + drop(last_remote_diff); + let mut v = VecDeque::new(); + v.push_back(StackItem { span, meta }); + self.stack + .push_back((v, Arc::new(Mutex::new(DiffBatch::default())))); + self.size += 1; + return; + } + + // Try to merge with the previous entry if allowed + if can_merge { + if let Some(last_span) = last.0.back_mut() { + if last_span.span.end == span.start { + // Merge spans by extending the end of the last span + last_span.span.end = span.end; + return; + } + } + } + + // Add as a new item to the existing entry + self.size += 1; + last.0.push_back(StackItem { span, meta }); + } + + pub fn compose_remote_event(&mut self, diff: &[&ContainerDiff]) { + if self.is_empty() { + return; + } + + let remote_diff = &mut self.stack.back_mut().unwrap().1; + let mut remote_diff = remote_diff.lock(); + for e in diff { + if let Some(d) = remote_diff.cid_to_events.get_mut(&e.id) { + d.compose_ref(&e.diff); + } else { + remote_diff + .cid_to_events + .insert(e.id.clone(), e.diff.clone()); + remote_diff.order.push(e.id.clone()); + } + } + } + + pub fn transform_based_on_this_delta(&mut self, diff: &DiffBatch) { + if self.is_empty() { + return; + } + let remote_diff = &mut self.stack.back_mut().unwrap().1; + remote_diff.lock().transform(diff, false); + } + + pub fn clear(&mut self) { + self.stack = VecDeque::new(); + self.stack.push_back((VecDeque::new(), Default::default())); + self.size = 0; + } + + pub fn is_empty(&self) -> bool { + self.size == 0 + } + + pub fn len(&self) -> usize { + self.size + } + + fn discard_empty_front_rows(&mut self) { + // Undo pop can leave an empty front row that only carries remote diffs. + // There is no older stack item for that diff to transform during trimming. + while self + .stack + .front() + .is_some_and(|(items, _)| items.is_empty()) + { + self.stack.pop_front(); + } + } + + fn ensure_trailing_empty_row(&mut self) { + if self.stack.is_empty() { + self.stack + .push_back((VecDeque::new(), Arc::new(Mutex::new(Default::default())))); + } + } + + fn pop_front(&mut self) { + if self.is_empty() { + return; + } + + self.discard_empty_front_rows(); + self.size -= 1; + let first = self.stack.front_mut().unwrap(); + let f = first.0.pop_front(); + assert!(f.is_some()); + if first.0.is_empty() { + self.stack.pop_front(); + } + + self.ensure_trailing_empty_row(); + } + + fn set_top_meta(&mut self, meta: UndoItemMeta) { + let Some(top) = self.stack.back_mut() else { + return; + }; + let Some(last) = top.0.back_mut() else { + return; + }; + last.meta = meta; + } +} + +impl Default for Stack { + fn default() -> Self { + Stack::new() + } +} + +impl UndoManagerInner { + fn new(last_counter: Counter) -> Self { + Self { + next_counter: Some(last_counter), + undo_stack: Default::default(), + redo_stack: Default::default(), + processing_undo: false, + merge_interval_in_ms: 0, + last_undo_time: 0, + max_stack_size: usize::MAX, + exclude_origin_prefixes: vec![], + last_popped_selection: None, + on_pop: None, + on_push: None, + group: None, + } + } + + /// Returns true if a given container diff is disjoint with the current group. + /// They are disjoint if they have no overlap in changed container ids. + fn is_disjoint_with_group(&self, diff: &[&ContainerDiff]) -> bool { + let Some(group) = &self.group else { + return false; + }; + + diff.iter().all(|d| !group.affected_cids.contains(&d.id)) + } + + fn record_checkpoint(this: &RefCell, latest_counter: Counter, event: Option) { + let previous_counter = this.borrow().next_counter; + + if Some(latest_counter) == this.borrow().next_counter { + return; + } + + if this.borrow().next_counter.is_none() { + this.borrow_mut().next_counter = Some(latest_counter); + return; + } + + if let Some(group) = &mut this.borrow_mut().group { + event.iter().for_each(|e| { + e.events.iter().for_each(|e| { + group.affected_cids.insert(e.id.clone()); + }) + }); + } + + let now = get_sys_timestamp() as Timestamp; + let span = CounterSpan::new(this.borrow().next_counter.unwrap(), latest_counter); + let meta = this + .borrow() + .on_push + .as_ref() + .map(|x| x(UndoOrRedo::Undo, span, event)) + .unwrap_or_default(); + + let mut this = this.borrow_mut(); + let this: &mut Self = &mut this; + // Wether the change is within the accepted merge interval + let in_merge_interval = now - this.last_undo_time < this.merge_interval_in_ms; + + // If group is active, but there is nothing in the group, don't merge + // If the group is active and it's not the first push in the group, merge + let group_should_merge = this.group.is_some() + && match ( + previous_counter, + this.group.as_ref().map(|g| g.start_counter), + ) { + (Some(previous), Some(active)) => previous != active, + _ => true, + }; + + let should_merge = !this.undo_stack.is_empty() && (in_merge_interval || group_should_merge); + + if should_merge { + this.undo_stack + .push_with_merge(span, meta, true, this.group.as_ref()); + } else { + this.last_undo_time = now; + this.undo_stack.push(span, meta); + } + + this.next_counter = Some(latest_counter); + this.redo_stack.clear(); + while this.undo_stack.len() > this.max_stack_size { + this.undo_stack.pop_front(); + } + } +} + +fn get_counter_end(doc: &LoroDoc, peer: PeerID) -> Counter { + doc.oplog().lock().vv().get(&peer).cloned().unwrap_or(0) +} + +impl UndoManager { + pub fn new(doc: &LoroDoc) -> Self { + let peer = Arc::new(AtomicU64::new(doc.peer_id())); + let peer_clone = peer.clone(); + let peer_clone2 = peer.clone(); + let inner = Arc::new(ReentrantMutex::new(RefCell::new(UndoManagerInner::new( + get_counter_end(doc, doc.peer_id()), + )))); + let inner_clone = inner.clone(); + let inner_clone2 = inner.clone(); + let remap_containers = Arc::new(Mutex::new(FxHashMap::default())); + let remap_containers_clone = remap_containers.clone(); + let undo_sub = doc.subscribe_root(Arc::new(move |event| match event.event_meta.by { + EventTriggerKind::Local => { + // TODO: PERF undo can be significantly faster if we can get + // the DiffBatch for undo here + let lock = inner_clone.lock(); + if lock.borrow().processing_undo { + return; + } + if let Some(id) = event + .event_meta + .to + .iter() + .find(|x| x.peer == peer_clone.load(std::sync::atomic::Ordering::Relaxed)) + { + let should_exclude = lock + .borrow() + .exclude_origin_prefixes + .iter() + .any(|x| event.event_meta.origin.starts_with(&**x)); + if should_exclude { + // If the event is from the excluded origin, we don't record it + // in the undo stack. But we need to record its effect like it's + // a remote event. + let mut inner = lock.borrow_mut(); + inner.undo_stack.compose_remote_event(event.events); + inner.redo_stack.compose_remote_event(event.events); + inner.next_counter = Some(id.counter + 1); + } else { + UndoManagerInner::record_checkpoint(&lock, id.counter + 1, Some(event)); + } + } + } + EventTriggerKind::Import => { + let lock = inner_clone.lock(); + let mut inner = lock.borrow_mut(); + + for e in event.events { + if let Diff::Tree(tree) = &e.diff { + for item in &tree.diff { + let target = item.target; + if let TreeExternalDiff::Create { .. } = &item.action { + // If the concurrent event is a create event, it may bring the deleted tree node back, + // so we need to remove it from the remap of the container. + remap_containers_clone + .lock() + .remove(&target.associated_meta_container()); + } + } + } + } + + let is_import_disjoint = inner.is_disjoint_with_group(event.events); + + inner.undo_stack.compose_remote_event(event.events); + inner.redo_stack.compose_remote_event(event.events); + + // If the import is not disjoint, we end the active group + // all subsequent changes will be new undo items + if !is_import_disjoint { + inner.group = None; + } + } + EventTriggerKind::Checkout => { + let lock = inner_clone.lock(); + let mut inner = lock.borrow_mut(); + inner.undo_stack.clear(); + inner.redo_stack.clear(); + inner.next_counter = None; + } + })); + + let sub = doc.subscribe_peer_id_change(Box::new(move |id| { + let lock = inner_clone2.lock(); + let mut inner = lock.borrow_mut(); + inner.undo_stack.clear(); + inner.redo_stack.clear(); + inner.next_counter = Some(id.counter); + peer_clone2.store(id.peer, std::sync::atomic::Ordering::Relaxed); + true + })); + + UndoManager { + peer, + container_remap: remap_containers, + inner, + _peer_id_change_sub: sub, + _undo_sub: undo_sub, + doc: doc.clone(), + } + } + + pub fn group_start(&self) -> LoroResult<()> { + let lock = self.inner.lock(); + let mut inner = lock.borrow_mut(); + + if inner.group.is_some() { + return Err(LoroError::UndoGroupAlreadyStarted); + } + + inner.group = + Some(UndoGroup::new(inner.next_counter.ok_or_else(|| { + LoroError::Unknown("UndoManager is not ready".into()) + })?)); + + Ok(()) + } + + pub fn group_end(&self) { + self.inner.lock().borrow_mut().group = None; + } + + pub fn peer(&self) -> PeerID { + self.peer.load(std::sync::atomic::Ordering::Relaxed) + } + + pub fn set_merge_interval(&self, interval: i64) { + self.inner.lock().borrow_mut().merge_interval_in_ms = interval; + } + + pub fn set_max_undo_steps(&self, size: usize) { + self.inner.lock().borrow_mut().max_stack_size = size; + } + + pub fn add_exclude_origin_prefix(&self, prefix: &str) { + self.inner + .lock() + .borrow_mut() + .exclude_origin_prefixes + .push(prefix.into()); + } + + pub fn record_new_checkpoint(&self) -> LoroResult<()> { + // Use implicit-style barrier to preserve next-commit options across + // an empty commit before undo/redo processing. + self.doc.with_barrier(|| {}); + let counter = get_counter_end(&self.doc, self.peer()); + UndoManagerInner::record_checkpoint(&self.inner.lock(), counter, None); + Ok(()) + } + + #[instrument(skip_all)] + pub fn undo(&self) -> LoroResult { + self.perform( + |x| &mut x.undo_stack, + |x| &mut x.redo_stack, + UndoOrRedo::Undo, + ) + } + + #[instrument(skip_all)] + pub fn redo(&self) -> LoroResult { + self.perform( + |x| &mut x.redo_stack, + |x| &mut x.undo_stack, + UndoOrRedo::Redo, + ) + } + + fn perform( + &self, + get_stack: impl Fn(&mut UndoManagerInner) -> &mut Stack, + get_opposite: impl Fn(&mut UndoManagerInner) -> &mut Stack, + kind: UndoOrRedo, + ) -> LoroResult { + let doc = &self.doc.clone(); + // When in the undo/redo loop, the new undo/redo stack item should restore the selection + // to the state it was in before the item that was popped two steps ago from the stack. + // + // ┌────────────┐ + // │Selection 1 │ + // └─────┬──────┘ + // │ Some + // ▼ ops + // ┌────────────┐ + // │Selection 2 │ + // └─────┬──────┘ + // │ Some + // ▼ ops + // ┌────────────┐ + // │Selection 3 │◁ ─ ─ ─ ─ ─ ─ ─ Restore ─ ─ ─ + // └─────┬──────┘ │ + // │ + // │ │ + // │ ┌ ─ ─ ─ ─ ─ ─ ─ + // Enter the │ Undo ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─▶ Push Redo │ + // undo/redo ─ ─ ─ ▶ ▼ └ ─ ─ ─ ─ ─ ─ ─ + // loop ┌────────────┐ │ + // │Selection 2 │◁─ ─ ─ Restore ─ + // └─────┬──────┘ │ │ + // │ + // │ │ │ + // │ ┌ ─ ─ ─ ─ ─ ─ ─ + // │ Undo ─ ─ ─ ─ ▶ Push Redo │ │ + // ▼ └ ─ ─ ─ ─ ─ ─ ─ + // ┌────────────┐ │ │ + // │Selection 1 │ + // └─────┬──────┘ │ │ + // │ Redo ◀ ─ ─ ─ ─ ─ ─ ─ ─ + // ▼ │ + // ┌────────────┐ + // ┌ Restore ─ ▷│Selection 2 │ │ + // └─────┬──────┘ + // │ │ │ + // ┌ ─ ─ ─ ─ ─ ─ ─ │ + // Push Undo │◀─ ─ ─ ─ ─ ─ ─ │ Redo ◀ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┘ + // └ ─ ─ ─ ─ ─ ─ ─ ▼ + // │ ┌────────────┐ + // │Selection 3 │ + // │ └─────┬──────┘ + // ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ▶ │ Undo + // ▼ + // ┌────────────┐ + // │Selection 2 │ + // └────────────┘ + // + // Because users may change the selections during the undo/redo loop, it's + // more stable to keep the selection stored in the last stack item + // rather than using the current selection directly. + self.record_new_checkpoint()?; + let end_counter = get_counter_end(doc, self.peer()); + let mut top = { + let lock = self.inner.lock(); + let mut inner = lock.borrow_mut(); + inner.processing_undo = true; + get_stack(&mut inner).pop() + }; + + let mut executed = false; + while let Some((mut span, remote_diff)) = top { + let mut next_push_selection = None; + { + let inner = self.inner.clone(); + // We need to clone this because otherwise will be applied to the same remote diff + let remote_change_clone = remote_diff.lock().clone(); + let commit = doc.undo_internal( + IdSpan { + peer: self.peer(), + counter: span.span, + }, + &mut self.container_remap.lock(), + Some(&remote_change_clone), + &mut |diff| { + info_span!("transform remote diff").in_scope(|| { + let inner = inner.lock(); + // + get_stack(&mut inner.borrow_mut()).transform_based_on_this_delta(diff); + }); + }, + )?; + drop(commit); + let inner = self.inner.lock(); + let mut is_some = false; + + if let Some(on_pop) = inner.borrow().on_pop.as_ref() { + is_some = true; + for cursor in span.meta.cursors.iter_mut() { + // We need to transform cursor here. + // Note that right now is already done, + // remote_diff is also transformed by it now (that's what we need). + transform_cursor( + cursor, + &remote_diff.lock(), + doc, + &self.container_remap.lock(), + ); + } + + on_pop(kind, span.span, span.meta.clone()); + } + if is_some { + let take = inner.borrow_mut().last_popped_selection.take(); + next_push_selection = take; + inner.borrow_mut().last_popped_selection = Some(span.meta.cursors); + } + } + let new_counter = get_counter_end(doc, self.peer()); + if end_counter != new_counter { + let inner = self.inner.lock(); + let mut meta = inner + .borrow() + .on_push + .as_ref() + .map(|x| { + x( + kind.opposite(), + CounterSpan::new(end_counter, new_counter), + None, + ) + }) + .unwrap_or_default(); + + if matches!(kind, UndoOrRedo::Undo) + && get_opposite(&mut inner.borrow_mut()).is_empty() + { + // If it's the first undo, we use the cursors from the users + } else if let Some(inner) = next_push_selection.take() { + // Otherwise, we use the cursors from the undo/redo loop + meta.cursors = inner; + } + + get_opposite(&mut inner.borrow_mut()) + .push(CounterSpan::new(end_counter, new_counter), meta); + inner.borrow_mut().next_counter = Some(new_counter); + executed = true; + break; + } else { + // continue to pop the undo item as this undo is a no-op + top = get_stack(&mut self.inner.lock().borrow_mut()).pop(); + continue; + } + } + + self.inner.lock().borrow_mut().processing_undo = false; + Ok(executed) + } + + pub fn can_undo(&self) -> bool { + !self.inner.lock().borrow().undo_stack.is_empty() + } + + pub fn can_redo(&self) -> bool { + !self.inner.lock().borrow().redo_stack.is_empty() + } + + pub fn undo_count(&self) -> usize { + self.inner.lock().borrow().undo_stack.len() + } + + pub fn redo_count(&self) -> usize { + self.inner.lock().borrow().redo_stack.len() + } + + /// Get the metadata of the top undo stack item, if any. + pub fn top_undo_meta(&self) -> Option { + self.inner.lock().borrow().undo_stack.peek_top_meta() + } + + /// Get the metadata of the top redo stack item, if any. + pub fn top_redo_meta(&self) -> Option { + self.inner.lock().borrow().redo_stack.peek_top_meta() + } + + /// Get the value associated with the top undo stack item, if any. + pub fn top_undo_value(&self) -> Option { + self.top_undo_meta().map(|m| m.value) + } + + /// Get the value associated with the top redo stack item, if any. + pub fn top_redo_value(&self) -> Option { + self.top_redo_meta().map(|m| m.value) + } + + pub fn set_on_push(&self, on_push: Option) { + self.inner.lock().borrow_mut().on_push = on_push; + } + + pub fn set_on_pop(&self, on_pop: Option) { + self.inner.lock().borrow_mut().on_pop = on_pop; + } + + pub fn clear(&self) { + self.inner.lock().borrow_mut().undo_stack.clear(); + self.inner.lock().borrow_mut().redo_stack.clear(); + } + + /// Clear only the redo stack, preserving the undo stack. + pub fn clear_redo(&self) { + self.inner.lock().borrow_mut().redo_stack.clear(); + } + + /// Clear only the undo stack, preserving the redo stack. + pub fn clear_undo(&self) { + self.inner.lock().borrow_mut().undo_stack.clear(); + } + + pub fn set_top_undo_meta(&self, meta: UndoItemMeta) { + self.inner.lock().borrow_mut().undo_stack.set_top_meta(meta); + } + + pub fn set_top_redo_meta(&self, meta: UndoItemMeta) { + self.inner.lock().borrow_mut().redo_stack.set_top_meta(meta); + } +} + +/// Undo the given spans of operations. +/// +/// # Parameters +/// +/// - `spans`: A vector of tuples where each tuple contains an `IdSpan` and its associated `Frontiers`. +/// - `IdSpan`: Represents a span of operations identified by an ID. +/// - `Frontiers`: Represents the deps of the given id_span +/// - `latest_frontiers`: The latest frontiers of the document +/// - `calc_diff`: A closure that takes two `Frontiers` and calculates the difference between them, returning a `DiffBatch`. +/// +/// # Returns +/// +/// - `DiffBatch`: Applying this batch on the `latest_frontiers` will undo the ops in the given spans. +pub(crate) fn undo( + spans: Vec<(IdSpan, Frontiers)>, + last_frontiers_or_last_bi: Either<&Frontiers, &DiffBatch>, + calc_diff: impl Fn(&Frontiers, &Frontiers) -> DiffBatch, + on_last_event_a: &mut dyn FnMut(&DiffBatch), +) -> DiffBatch { + // The process of performing undo is: + // + // 0. Split the span into a series of continuous spans. There is no external dep within each continuous span. + // + // For each continuous span_i: + // + // 1. a. Calculate the event of checkout from id_span.last to id_span.deps, call it Ai. It undo the ops in the current span. + // b. Calculate A'i = Ai + T(Ci-1, Ai) if i > 0, otherwise A'i = Ai. + // NOTE: A'i can undo the ops in the current span and the previous spans, if it's applied on the id_span.last version. + // 2. Calculate the event of checkout from id_span.last to [the next span's last id] or [the latest version], call it Bi. + // 3. Transform event A'i based on Bi, call it Ci + // 4. If span_i is the last span, apply Ci to the current state. + + // ------------------------------------------------------- + // 0. Split the span into a series of continuous spans + // ------------------------------------------------------- + + let mut last_ci: Option = None; + for i in 0..spans.len() { + debug_span!("Undo", ?i, "Undo span {:?}", &spans[i]).in_scope(|| { + let (this_id_span, this_deps) = &spans[i]; + // --------------------------------------- + // 1.a Calc event A_i + // --------------------------------------- + let mut event_a_i = debug_span!("1. Calc event A_i").in_scope(|| { + // Checkout to the last id of the id_span + calc_diff(&this_id_span.id_last().into(), this_deps) + }); + + // println!("event_a_i: {:?}", event_a_i); + + // --------------------------------------- + // 2. Calc event B_i + // --------------------------------------- + let stack_diff_batch; + let event_b_i = 'block: { + let next = if i + 1 < spans.len() { + spans[i + 1].0.id_last().into() + } else { + match last_frontiers_or_last_bi { + Either::Left(last_frontiers) => last_frontiers.clone(), + Either::Right(right) => break 'block right, + } + }; + stack_diff_batch = Some(calc_diff(&this_id_span.id_last().into(), &next)); + stack_diff_batch.as_ref().unwrap() + }; + + // println!("event_b_i: {:?}", event_b_i); + + // event_a_prime can undo the ops in the current span and the previous spans + let mut event_a_prime = if let Some(mut last_ci) = last_ci.take() { + // ------------------------------------------------------------------------------ + // 1.b Transform and apply Ci-1 based on Ai, call it A'i + // ------------------------------------------------------------------------------ + last_ci.transform(&event_a_i, true); + + event_a_i.compose(&last_ci); + event_a_i + } else { + event_a_i + }; + if i == spans.len() - 1 { + on_last_event_a(&event_a_prime); + } + // -------------------------------------------------- + // 3. Transform event A'_i based on B_i, call it C_i + // -------------------------------------------------- + event_a_prime.transform(event_b_i, true); + + // println!("event_a_prime: {:?}", event_a_prime); + + let c_i = event_a_prime; + last_ci = Some(c_i); + }); + } + + last_ci.unwrap() +} diff --git a/3rdparty/loro/crates/loro-internal/src/utils/kv_wrapper.rs b/3rdparty/loro/crates/loro-internal/src/utils/kv_wrapper.rs new file mode 100644 index 0000000000..9c9988bf49 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/utils/kv_wrapper.rs @@ -0,0 +1,173 @@ +use crate::sync::Mutex; +use bytes::Bytes; +use loro_kv_store::{mem_store::MemKvConfig, MemKvStore}; +use std::{collections::BTreeSet, ops::Bound, sync::Arc}; + +use crate::kv_store::KvStore; + +/// This thin wrapper aims to limit the ability to modify the kv store and make +/// it easy to find all the modifications. +pub(crate) struct KvWrapper { + kv: Arc>, +} + +impl Clone for KvWrapper { + /// Deep clone the inner kv store. + fn clone(&self) -> Self { + Self { + kv: self.kv.lock().clone_store(), + } + } +} + +impl KvWrapper { + pub fn new_mem() -> Self { + Self { + // kv: Arc::new(Mutex::new(BTreeMap::new())), + kv: Arc::new(Mutex::new(MemKvStore::new( + // set false because it's depended by GC snapshot's import & export + MemKvConfig::default().should_encode_none(false), + ))), + } + } + + pub fn arc_clone(&self) -> Self { + Self { + kv: self.kv.clone(), + } + } + + pub fn import(&self, bytes: Bytes) -> Result<(), String> { + let mut kv = self.kv.lock(); + // Only reached while decoding a snapshot/state blob whose integrity is + // already guaranteed by the document-level checksum in + // `parse_header_and_body(.., true)`, so skip the redundant per-block + // checksum. + kv.import_all_unchecked(bytes) + } + + pub fn export(&self) -> Bytes { + let mut kv = self.kv.lock(); + kv.export_all() + } + + pub fn get(&self, key: &[u8]) -> Option { + let kv = self.kv.lock(); + kv.get(key) + } + + pub(crate) fn keys(&self) -> BTreeSet> { + let kv = self.kv.lock(); + kv.scan(Bound::Unbounded, Bound::Unbounded) + .map(|(k, _)| k.to_vec()) + .collect() + } + + /// Snapshot all entries while holding only the KV lock. + /// + /// Callers that also need the arena lock must use this instead of entering arena code from + /// inside a KV-locked closure. Lazy arena parent resolution can acquire this same KV lock. + pub(crate) fn scan_all_entries(&self) -> Vec<(Bytes, Bytes)> { + let kv = self.kv.lock(); + kv.scan(Bound::Unbounded, Bound::Unbounded).collect() + } + + pub(crate) fn scan_all_keys(&self) -> Vec { + let kv = self.kv.lock(); + kv.scan(Bound::Unbounded, Bound::Unbounded) + .map(|(k, _)| k) + .collect() + } + + pub fn set_all(&self, updates: Vec<(Bytes, Bytes)>) { + let mut kv = self.kv.lock(); + for (k, v) in updates { + kv.set(&k, v); + } + } + + #[allow(unused)] + pub(crate) fn contains_key(&self, key: &[u8]) -> bool { + self.kv.lock().contains_key(key) + } + + pub(crate) fn remove_same(&self, old_kv: &KvWrapper) { + if Arc::ptr_eq(&self.kv, &old_kv.kv) { + let mut this = self.kv.lock(); + let keys = this + .scan(Bound::Unbounded, Bound::Unbounded) + .map(|(k, _)| k) + .collect::>(); + for k in keys { + this.remove(&k); + } + return; + } + + let other = old_kv.kv.lock(); + let mut this = self.kv.lock(); + for (k, v) in other.scan(Bound::Unbounded, Bound::Unbounded) { + if this.get(&k) == Some(v) { + this.remove(&k); + } + } + } + + pub(crate) fn remove(&self, k: &[u8]) -> Option { + self.kv.lock().remove(k) + } + + /// Remove all keys not in the given set + pub(crate) fn retain_keys(&self, keys: &BTreeSet>) { + let mut kv = self.kv.lock(); + let mut to_remove = BTreeSet::new(); + for (k, _) in kv.scan(Bound::Unbounded, Bound::Unbounded) { + if !keys.contains(&*k) { + to_remove.insert(k); + } + } + + for k in to_remove { + kv.remove(&k); + } + } + + pub(crate) fn insert(&self, k: &[u8], v: Bytes) { + self.kv.lock().set(k, v); + } + + pub(crate) fn is_empty(&self) -> bool { + self.kv.lock().is_empty() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn set_all_applies_precollected_updates() { + let kv = KvWrapper::new_mem(); + + kv.set_all(vec![( + Bytes::from_static(b"key"), + Bytes::from_static(b"value"), + )]); + assert_eq!(kv.get(b"key"), Some(Bytes::from_static(b"value"))); + } + + #[test] + fn remove_same_streams_different_stores() { + let kv = KvWrapper::new_mem(); + let old_kv = KvWrapper::new_mem(); + kv.insert(b"same", Bytes::from_static(b"value")); + kv.insert(b"changed", Bytes::from_static(b"new")); + old_kv.insert(b"same", Bytes::from_static(b"value")); + old_kv.insert(b"changed", Bytes::from_static(b"old")); + + kv.remove_same(&old_kv); + + assert_eq!(kv.get(b"same"), None); + assert_eq!(kv.get(b"changed"), Some(Bytes::from_static(b"new"))); + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/utils/lazy.rs b/3rdparty/loro/crates/loro-internal/src/utils/lazy.rs new file mode 100644 index 0000000000..175e0d2451 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/utils/lazy.rs @@ -0,0 +1,21 @@ +#[derive(Debug, Clone)] +pub enum LazyLoad> { + Src(Src), + Dst(Dst), +} + +impl> LazyLoad { + pub fn get_mut(&mut self) -> &mut Dst { + match self { + LazyLoad::Src(src) => { + let dst = Dst::from(std::mem::take(src)); + *self = LazyLoad::Dst(dst); + match self { + LazyLoad::Dst(dst) => dst, + _ => unreachable!(), + } + } + LazyLoad::Dst(dst) => dst, + } + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/utils/mod.rs b/3rdparty/loro/crates/loro-internal/src/utils/mod.rs new file mode 100644 index 0000000000..9feef9bec8 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/utils/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod kv_wrapper; +pub(crate) mod lazy; +pub(crate) mod query_by_len; +pub mod string_slice; +pub(crate) mod subscription; +pub(crate) mod utf16; diff --git a/3rdparty/loro/crates/loro-internal/src/utils/query_by_len.rs b/3rdparty/loro/crates/loro-internal/src/utils/query_by_len.rs new file mode 100644 index 0000000000..fd631189ff --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/utils/query_by_len.rs @@ -0,0 +1,192 @@ +use std::marker::PhantomData; + +use generic_btree::{BTreeTrait, FindResult, Query}; + +use crate::container::richtext::richtext_state::{RichtextStateChunk, RichtextTreeTrait}; + +/// An easy way to implement [Query] by using key index +/// +/// This query implementation will +/// - prefer next element when this element has zero length. +/// - prefer next element rather than previous element when the left is at the middle of two elements. +pub struct IndexQuery, B: BTreeTrait> { + pub left: usize, + _data: PhantomData<(T, B)>, +} + +/// An easy way to implement [Query] by using key index +/// +/// This query implementation will +/// - prefer next element when this element has zero length. +/// - prefer next element rather than previous element when the left is at the middle of two elements. +pub struct IndexQueryWithEntityIndex, B: BTreeTrait> { + pub left: usize, + pub entity_index: usize, + _data: PhantomData<(T, B)>, +} + +/// An easy way to implement [Query] by using key index +/// +/// This query implementation will +/// - prefer next element when this element has zero length. +/// - prefer next element rather than previous element when the left is at the middle of two elements. +pub struct EntityIndexQueryWithEventIndex { + pub left: usize, + pub event_index: usize, +} + +impl, B: BTreeTrait> IndexQueryWithEntityIndex { + pub fn entity_index(&self) -> usize { + self.entity_index + } +} + +/// The default query implementation will +/// +/// - prefer next element when this element has zero length. +/// - prefer next element rather than previous element when the left is at the middle of two elements. +pub trait QueryByLen { + fn get_cache_entity_len(cache: &B::Cache) -> usize; + fn get_cache_len(cache: &B::Cache) -> usize; + fn get_elem_len(elem: &B::Elem) -> usize; + fn get_offset_and_found(left: usize, elem: &B::Elem) -> (usize, bool); +} + +impl, B: BTreeTrait> Query for IndexQuery { + type QueryArg = usize; + + fn init(target: &Self::QueryArg) -> Self { + Self { + left: *target, + _data: PhantomData, + } + } + + fn find_node( + &mut self, + _: &Self::QueryArg, + child_caches: &[generic_btree::Child], + ) -> generic_btree::FindResult { + let mut last_left = self.left; + for (i, cache) in child_caches.iter().enumerate() { + let len = T::get_cache_len(&cache.cache); + if self.left >= len { + last_left = self.left; + self.left -= len; + } else { + return FindResult::new_found(i, self.left); + } + } + + self.left = last_left; + FindResult::new_missing(child_caches.len() - 1, last_left) + } + + fn confirm_elem( + &mut self, + _q: &Self::QueryArg, + elem: &::Elem, + ) -> (usize, bool) { + T::get_offset_and_found(self.left, elem) + } +} + +impl, B: BTreeTrait> Query for IndexQueryWithEntityIndex { + type QueryArg = usize; + + fn init(target: &Self::QueryArg) -> Self { + Self { + left: *target, + entity_index: 0, + _data: PhantomData, + } + } + + fn find_node( + &mut self, + _: &Self::QueryArg, + child_caches: &[generic_btree::Child], + ) -> generic_btree::FindResult { + let mut last_left = self.left; + for (i, cache) in child_caches.iter().enumerate() { + let len = T::get_cache_len(&cache.cache); + if self.left >= len { + self.entity_index += T::get_cache_entity_len(&cache.cache); + last_left = self.left; + self.left -= len; + } else { + return FindResult::new_found(i, self.left); + } + } + + self.left = last_left; + FindResult::new_missing(child_caches.len() - 1, last_left) + } + + fn confirm_elem( + &mut self, + _q: &Self::QueryArg, + elem: &::Elem, + ) -> (usize, bool) { + let (offset, found) = T::get_offset_and_found(self.left, elem); + self.entity_index += offset; + (offset, found) + } +} + +impl Query for EntityIndexQueryWithEventIndex { + type QueryArg = usize; + + fn init(target: &Self::QueryArg) -> Self { + Self { + left: *target, + event_index: 0, + } + } + + fn find_node( + &mut self, + _: &Self::QueryArg, + child_caches: &[generic_btree::Child], + ) -> generic_btree::FindResult { + let mut last_left = self.left; + let mut last_event_left = self.left; + for (i, cache) in child_caches.iter().enumerate() { + let len = cache.cache.entity_len as usize; + if self.left >= len { + last_event_left = self.event_index; + self.event_index += cache.cache.event_len() as usize; + last_left = self.left; + self.left -= len; + } else { + return FindResult::new_found(i, self.left); + } + } + + self.left = last_left; + self.event_index = last_event_left; + FindResult::new_missing(child_caches.len() - 1, last_left) + } + + fn confirm_elem(&mut self, _q: &Self::QueryArg, elem: &RichtextStateChunk) -> (usize, bool) { + let left = self.left; + match elem { + RichtextStateChunk::Text(s) => { + if s.len() as usize >= left { + self.event_index += s.convert_unicode_offset_to_event_offset(left); + return (left, true); + } + + self.event_index += s.event_len() as usize; + (left, false) + } + RichtextStateChunk::Style { .. } => { + if left == 0 { + return (0, true); + } + + (left, false) + } + } + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/utils/string_slice.rs b/3rdparty/loro/crates/loro-internal/src/utils/string_slice.rs new file mode 100644 index 0000000000..c0096fa9f5 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/utils/string_slice.rs @@ -0,0 +1,464 @@ +use std::{fmt::Debug, ops::Deref}; + +use append_only_bytes::BytesSlice; +use generic_btree::rle::{HasLength, Mergeable, Sliceable, TryInsert}; +use rle::Mergable; +use serde::{Deserialize, Deserializer, Serialize}; + +use crate::{ + container::richtext::richtext_state::{unicode_to_utf8_index, utf16_to_utf8_index}, + delta::DeltaValue, +}; + +use super::utf16::{count_unicode_chars, count_utf16_len}; + +#[derive(Clone)] +pub struct StringSlice { + bytes: Variant, +} + +impl PartialEq for StringSlice { + fn eq(&self, other: &Self) -> bool { + self.as_str() == other.as_str() + } +} + +impl Eq for StringSlice {} + +#[derive(Clone, PartialEq, Eq)] +enum Variant { + BytesSlice(BytesSlice), + Owned(String), +} + +impl From for StringSlice { + fn from(s: String) -> Self { + Self { + bytes: Variant::Owned(s), + } + } +} + +impl From for StringSlice { + fn from(s: BytesSlice) -> Self { + Self::new(s) + } +} + +impl From<&str> for StringSlice { + fn from(s: &str) -> Self { + Self { + bytes: Variant::Owned(s.to_string()), + } + } +} + +impl Debug for StringSlice { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("StringSlice") + .field("bytes", &self.as_str()) + .finish() + } +} + +impl StringSlice { + pub fn new(s: BytesSlice) -> Self { + std::str::from_utf8(&s).unwrap(); + Self { + bytes: Variant::BytesSlice(s), + } + } + + pub fn as_str(&self) -> &str { + match &self.bytes { + // SAFETY: `bytes` is always valid utf8 + Variant::BytesSlice(s) => unsafe { std::str::from_utf8_unchecked(s) }, + Variant::Owned(s) => s, + } + } + + pub fn len_bytes(&self) -> usize { + match &self.bytes { + Variant::BytesSlice(s) => s.len(), + Variant::Owned(s) => s.len(), + } + } + + fn bytes(&self) -> &[u8] { + match &self.bytes { + Variant::BytesSlice(s) => s.deref(), + Variant::Owned(s) => s.as_bytes(), + } + } + + pub fn len_unicode(&self) -> usize { + count_unicode_chars(self.bytes()) + } + + pub fn len_utf16(&self) -> usize { + count_utf16_len(self.bytes()) + } + + pub fn is_empty(&self) -> bool { + self.bytes().is_empty() + } + + pub fn extend(&mut self, s: &str) { + match &mut self.bytes { + Variant::BytesSlice(_) => { + *self = Self { + bytes: Variant::Owned(format!("{}{}", self.as_str(), s)), + } + } + Variant::Owned(v) => { + v.push_str(s); + } + } + } +} + +impl std::fmt::Display for StringSlice { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +impl<'de> Deserialize<'de> for StringSlice { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let s = String::deserialize(deserializer)?; + Ok(s.into()) + } +} + +impl Serialize for StringSlice { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(self.as_str()) + } +} + +impl DeltaValue for StringSlice { + fn value_extend(&mut self, other: Self) -> Result<(), Self> { + match (&mut self.bytes, &other.bytes) { + (Variant::BytesSlice(s), Variant::BytesSlice(o)) => match s.try_merge(o) { + Ok(_) => Ok(()), + Err(_) => Err(other), + }, + (Variant::Owned(s), _) => { + s.push_str(other.as_str()); + Ok(()) + } + _ => Err(other), + } + } + + fn take(&mut self, length: usize) -> Self { + let length = if cfg!(feature = "wasm") { + utf16_to_utf8_index(self.as_str(), length).unwrap() + } else { + unicode_to_utf8_index(self.as_str(), length).unwrap() + }; + + match &mut self.bytes { + Variant::BytesSlice(s) => { + let mut other = s.slice_clone(length..); + s.slice_(..length); + std::mem::swap(s, &mut other); + Self { + bytes: Variant::BytesSlice(other), + } + } + Variant::Owned(s) => { + let mut other = s.split_off(length); + std::mem::swap(s, &mut other); + Self { + bytes: Variant::Owned(other), + } + } + } + } + + /// Unicode length of the string + /// Utf16 length when in WASM + fn length(&self) -> usize { + if cfg!(feature = "wasm") { + count_utf16_len(self.bytes()) + } else { + count_unicode_chars(self.bytes()) + } + } +} + +impl HasLength for StringSlice { + fn rle_len(&self) -> usize { + if cfg!(feature = "wasm") { + count_utf16_len(self.bytes()) + } else { + count_unicode_chars(self.bytes()) + } + } +} + +impl TryInsert for StringSlice { + fn try_insert(&mut self, pos: usize, elem: Self) -> Result<(), Self> + where + Self: Sized, + { + match &mut self.bytes { + Variant::BytesSlice(_) => Err(elem), + Variant::Owned(s) => { + if s.capacity() >= s.len() + elem.len_bytes() { + let pos = if cfg!(feature = "wasm") { + utf16_to_utf8_index(s.as_str(), pos).unwrap() + } else { + unicode_to_utf8_index(s.as_str(), pos).unwrap() + }; + s.insert_str(pos, elem.as_str()); + Ok(()) + } else { + Err(elem) + } + } + } + + // match (&mut self.bytes, &elem.bytes) { + // (Variant::Owned(a), Variant::Owned(b)) + // // TODO: Extract magic num + // if a.capacity() >= a.len() + b.len() && a.capacity() < 128 => + // { + // a.insert_str(pos, b.as_str()); + // Ok(()) + // } + // _ => Err(elem), + // } + } +} + +impl Mergeable for StringSlice { + fn can_merge(&self, rhs: &Self) -> bool { + match (&self.bytes, &rhs.bytes) { + (Variant::BytesSlice(a), Variant::BytesSlice(b)) => a.can_merge(b), + (Variant::Owned(a), Variant::Owned(b)) => a.len() + b.len() <= a.capacity(), + _ => false, + } + } + + fn merge_right(&mut self, rhs: &Self) { + match (&mut self.bytes, &rhs.bytes) { + (Variant::BytesSlice(a), Variant::BytesSlice(b)) => a.merge(b, &()), + (Variant::Owned(a), Variant::Owned(b)) => a.push_str(b.as_str()), + _ => {} + } + } + + fn merge_left(&mut self, left: &Self) { + match (&mut self.bytes, &left.bytes) { + (Variant::BytesSlice(a), Variant::BytesSlice(b)) => { + let mut new = b.clone(); + new.merge(a, &()); + *a = new; + } + (Variant::Owned(a), Variant::Owned(b)) => { + a.insert_str(0, b.as_str()); + } + _ => {} + } + } +} + +impl Sliceable for StringSlice { + fn _slice(&self, range: std::ops::Range) -> Self { + let range = if cfg!(feature = "wasm") { + let start = utf16_to_utf8_index(self.as_str(), range.start).unwrap(); + let end = utf16_to_utf8_index(self.as_str(), range.end).unwrap(); + start..end + } else { + let start = unicode_to_utf8_index(self.as_str(), range.start).unwrap(); + let end = unicode_to_utf8_index(self.as_str(), range.end).unwrap(); + start..end + }; + + let bytes = match &self.bytes { + Variant::BytesSlice(s) => Variant::BytesSlice(s.slice_clone(range)), + Variant::Owned(s) => Variant::Owned(s[range].to_string()), + }; + + Self { bytes } + } + + fn split(&mut self, pos: usize) -> Self { + let pos = if cfg!(feature = "wasm") { + utf16_to_utf8_index(self.as_str(), pos).unwrap() + } else { + unicode_to_utf8_index(self.as_str(), pos).unwrap() + }; + + let bytes = match &mut self.bytes { + Variant::BytesSlice(s) => { + let other = s.slice_clone(pos..); + s.slice_(..pos); + Variant::BytesSlice(other) + } + Variant::Owned(s) => { + let other = s.split_off(pos); + Variant::Owned(other) + } + }; + + Self { bytes } + } +} + +impl Default for StringSlice { + fn default() -> Self { + StringSlice { + bytes: Variant::Owned(String::with_capacity(32)), + } + } +} + +impl loro_delta::delta_trait::DeltaValue for StringSlice {} +#[allow(dead_code)] +pub fn unicode_range_to_byte_range(s: &str, start: usize, end: usize) -> (usize, usize) { + debug_assert!(start <= end); + let start_unicode_index = start; + let end_unicode_index = end; + let mut current_utf8_index = 0; + let mut start_byte = 0; + let mut end_byte = s.len(); + for (current_unicode_index, c) in s.chars().enumerate() { + if current_unicode_index == start_unicode_index { + start_byte = current_utf8_index; + } + + if current_unicode_index == end_unicode_index { + end_byte = current_utf8_index; + break; + } + + current_utf8_index += c.len_utf8(); + } + + (start_byte, end_byte) +} + +#[cfg(test)] +mod tests { + use append_only_bytes::AppendOnlyBytes; + use generic_btree::rle::{HasLength, Mergeable, Sliceable, TryInsert}; + + use crate::delta::DeltaValue; + + use super::*; + + #[test] + fn owned_slice_lengths_and_unicode_boundaries_use_logical_char_indices() { + let mut slice = StringSlice::from("aé😀b"); + assert_eq!(slice.as_str(), "aé😀b"); + assert_eq!(slice.len_bytes(), 8); + assert_eq!(slice.len_unicode(), 4); + assert_eq!(slice.len_utf16(), 5); + let event_len = if cfg!(feature = "wasm") { 5 } else { 4 }; + assert_eq!(slice.length(), event_len); + assert_eq!(slice.rle_len(), event_len); + + let middle_range = if cfg!(feature = "wasm") { 1..4 } else { 1..3 }; + let middle = slice._slice(middle_range); + assert_eq!(middle.as_str(), "é😀"); + + let right = slice.split(2); + assert_eq!(slice.as_str(), "aé"); + assert_eq!(right.as_str(), "😀b"); + + let prefix = slice.take(1); + assert_eq!(prefix.as_str(), "a"); + assert_eq!(slice.as_str(), "é"); + } + + #[test] + fn owned_slice_insert_and_merge_respect_capacity_contracts() { + let mut backing = String::with_capacity(16); + backing.push_str("a😀"); + let mut slice = StringSlice::from(backing); + slice.try_insert(1, StringSlice::from("é")).unwrap(); + assert_eq!(slice.as_str(), "aé😀"); + + let mut tight = StringSlice::from(String::from("ab")); + let rejected = tight.try_insert(1, StringSlice::from("c")).unwrap_err(); + assert_eq!(tight.as_str(), "ab"); + assert_eq!(rejected.as_str(), "c"); + + let mut left_text = String::with_capacity(8); + left_text.push_str("ab"); + let mut left = StringSlice::from(left_text); + let right = StringSlice::from("cd"); + assert!(left.can_merge(&right)); + left.merge_right(&right); + assert_eq!(left.as_str(), "abcd"); + + let mut suffix = StringSlice::from("cd"); + suffix.merge_left(&StringSlice::from("ab")); + assert_eq!(suffix.as_str(), "abcd"); + } + + #[test] + fn bytes_slice_variant_keeps_append_only_slice_semantics() { + let mut bytes = AppendOnlyBytes::new(); + bytes.push_str("left"); + let left_bytes = bytes.slice(..); + bytes.push_str("right"); + let right_bytes = bytes.slice(4..); + + let mut left = StringSlice::from(left_bytes.clone()); + let right = StringSlice::from(right_bytes.clone()); + assert_eq!(left.as_str(), "left"); + assert_eq!(right.as_str(), "right"); + assert!(left.can_merge(&right)); + left.value_extend(right).unwrap(); + assert_eq!(left.as_str(), "leftright"); + + let mut whole = StringSlice::from(bytes.slice(..)); + let tail = whole.split(4); + assert_eq!(whole.as_str(), "left"); + assert_eq!(tail.as_str(), "right"); + assert_eq!( + StringSlice::from(bytes.slice(..))._slice(1..5).as_str(), + "eftr" + ); + + let mut converted_to_owned = StringSlice::from(left_bytes); + converted_to_owned.extend("!"); + assert_eq!(converted_to_owned.as_str(), "left!"); + + let mut byte_slice = StringSlice::from(right_bytes); + let rejected = byte_slice + .try_insert(1, StringSlice::from("!")) + .unwrap_err(); + assert_eq!(byte_slice.as_str(), "right"); + assert_eq!(rejected.as_str(), "!"); + } + + #[test] + fn serde_display_default_and_range_conversion_are_string_like() { + let empty = StringSlice::default(); + assert!(empty.is_empty()); + + let slice = StringSlice::from("aé😀b"); + assert_eq!(slice.to_string(), "aé😀b"); + assert_eq!(format!("{slice:?}"), r#"StringSlice { bytes: "aé😀b" }"#); + + let encoded = serde_json::to_string(&slice).unwrap(); + assert_eq!(encoded, r#""aé😀b""#); + let decoded: StringSlice = serde_json::from_str(&encoded).unwrap(); + assert_eq!(decoded, slice); + + let (start, end) = unicode_range_to_byte_range("aé😀b", 1, 3); + assert_eq!(&"aé😀b"[start..end], "é😀"); + assert_eq!(unicode_range_to_byte_range("aé😀b", 2, 4), (3, 8)); + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/utils/subscription.rs b/3rdparty/loro/crates/loro-internal/src/utils/subscription.rs new file mode 100644 index 0000000000..764259ea13 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/utils/subscription.rs @@ -0,0 +1,668 @@ +use either::Either; +/* +This file is modified from the original file in the following repo: +https://github.com/zed-industries/zed + + +Copyright 2022 - 2024 Zed Industries, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + + http://www.apache.org/licenses/LICENSE-2.0 + + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + + + +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + + 1. Definitions. + + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + + END OF TERMS AND CONDITIONS + +*/ +use crate::sync::{thread, AtomicBool, Mutex}; +use smallvec::SmallVec; +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::atomic::Ordering; +use std::sync::{Arc, Weak}; +use std::{fmt::Debug, mem}; +use thread::ThreadId; + +#[derive(Debug)] +pub enum SubscriptionError { + CannotEmitEventDueToRecursiveCall, +} + +impl std::fmt::Display for Subscription { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "SubscriptionError") + } +} + +pub struct SubscriberSet( + Arc>>, +); + +impl Clone for SubscriberSet { + fn clone(&self) -> Self { + SubscriberSet(self.0.clone()) + } +} + +struct SubscriberSetState { + subscribers: BTreeMap>, ThreadId>>, + dropped_subscribers: BTreeSet<(EmitterKey, usize)>, + next_subscriber_id: usize, +} + +struct Subscriber { + active: Arc, + callback: Callback, + /// This field is used to drop the subscription when the subscriber is dropped. + _sub: InnerSubscription, +} + +impl SubscriberSet +where + EmitterKey: 'static + Ord + Clone + Debug + Send + Sync, + Callback: 'static + Send + Sync, +{ + pub fn new() -> Self { + Self(Arc::new(Mutex::new(SubscriberSetState { + subscribers: Default::default(), + dropped_subscribers: Default::default(), + next_subscriber_id: 0, + }))) + } + + /// Inserts a new [`Subscription`] for the given `emitter_key`. + /// + /// By default, subscriptions are inert, meaning that they won't be listed when calling + /// `[SubscriberSet::remove]` or `[SubscriberSet::retain]`. This method returns a tuple + /// of a [`Subscription`] and an `impl FnOnce`, and you can use the latter to activate + /// the [`Subscription`]. + pub fn insert( + &self, + emitter_key: EmitterKey, + callback: Callback, + ) -> (Subscription, impl FnOnce()) { + let active = Arc::new(AtomicBool::new(false)); + let mut lock = self.0.lock(); + let subscriber_id = post_inc(&mut lock.next_subscriber_id); + let this = Arc::downgrade(&self.0); + let emitter_key_1 = emitter_key.clone(); + let inner_sub = InnerSubscription { + unsubscribe: Arc::new(Mutex::new(Some(Box::new(move || { + let Some(this) = this.upgrade() else { + return; + }; + + let mut lock = this.lock(); + let Some(subscribers) = lock.subscribers.get_mut(&emitter_key) else { + // remove was called with this emitter_key + return; + }; + + if let Either::Left(subscribers) = subscribers { + subscribers.remove(&subscriber_id); + if subscribers.is_empty() { + lock.subscribers.remove(&emitter_key); + } + return; + } + + // We didn't manage to remove the subscription, which means it was dropped + // while invoking the callback. Mark it as dropped so that we can remove it + // later. + lock.dropped_subscribers + .insert((emitter_key, subscriber_id)); + })))), + }; + let subscription = Subscription { + unsubscribe: Arc::downgrade(&inner_sub.unsubscribe), + }; + + lock.subscribers + .entry(emitter_key_1) + .or_insert_with(|| Either::Left(BTreeMap::new())) + .as_mut() + .unwrap_left() + .insert( + subscriber_id, + Subscriber { + active: active.clone(), + callback, + _sub: inner_sub, + }, + ); + (subscription, move || active.store(true, Ordering::Relaxed)) + } + + #[allow(unused)] + pub fn remove(&self, emitter: &EmitterKey) -> impl IntoIterator { + let mut lock = self.0.lock(); + let subscribers = lock.subscribers.remove(emitter); + subscribers + .and_then(|x| x.left().map(|s| s.into_values())) + .into_iter() + .flatten() + .filter_map(|subscriber| { + if subscriber.active.load(Ordering::Relaxed) { + Some(subscriber.callback) + } else { + None + } + }) + } + + pub fn is_recursive_calling(&self, emitter: &EmitterKey) -> bool { + if let Some(Either::Right(thread_id)) = self.0.lock().subscribers.get(emitter) { + *thread_id == thread::current().id() + } else { + false + } + } + + /// Call the given callback for each subscriber to the given emitter. + /// If the callback returns false, the subscriber is removed. + pub fn retain( + &self, + emitter: &EmitterKey, + f: &mut dyn FnMut(&mut Callback) -> bool, + ) -> Result<(), SubscriptionError> { + let mut subscribers = { + let inner = loop { + let mut subscriber_set_state = self.0.lock(); + let Some(set) = subscriber_set_state.subscribers.get_mut(emitter) else { + return Ok(()); + }; + match set { + Either::Left(_) => { + break std::mem::replace(set, Either::Right(thread::current().id())) + .unwrap_left(); + } + Either::Right(lock_thread) => { + if thread::current().id() == *lock_thread { + return Err(SubscriptionError::CannotEmitEventDueToRecursiveCall); + } else { + // return Ok(()); + drop(subscriber_set_state); + #[cfg(loom)] + loom::thread::yield_now(); + #[cfg(not(loom))] + std::thread::sleep(std::time::Duration::from_millis(10)); + } + } + } + }; + inner + }; + + subscribers.retain(|_, subscriber| { + if subscriber.active.load(Ordering::Relaxed) { + f(&mut subscriber.callback) + } else { + true + } + }); + + let mut lock = self.0.lock(); + + // Add any new subscribers that were added while invoking the callback. + if let Some(Either::Left(new_subscribers)) = lock.subscribers.remove(emitter) { + subscribers.extend(new_subscribers); + } + + // Remove any dropped subscriptions that were dropped while invoking the callback. + for (dropped_emitter, dropped_subscription_id) in mem::take(&mut lock.dropped_subscribers) { + if *emitter == dropped_emitter { + subscribers.remove(&dropped_subscription_id); + } else { + lock.dropped_subscribers + .insert((dropped_emitter, dropped_subscription_id)); + } + } + + lock.subscribers + .insert(emitter.clone(), Either::Left(subscribers)); + Ok(()) + } + + pub fn is_empty(&self) -> bool { + self.0.lock().subscribers.is_empty() + } + + pub fn may_include(&self, emitter: &EmitterKey) -> bool { + self.0.lock().subscribers.contains_key(emitter) + } +} + +impl Default for SubscriberSet +where + EmitterKey: 'static + Ord + Clone + Debug + Send + Sync, + Callback: 'static + Send + Sync, +{ + fn default() -> Self { + Self::new() + } +} + +impl std::fmt::Debug for SubscriberSet +where + EmitterKey: 'static + Ord + Clone + Debug + Send + Sync, + Callback: 'static + Send + Sync, +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let lock = self.0.lock(); + f.debug_struct("SubscriberSet") + .field("subscriber_count", &lock.subscribers.len()) + .field("dropped_subscribers_count", &lock.dropped_subscribers.len()) + .field("next_subscriber_id", &lock.next_subscriber_id) + .finish() + } +} + +fn post_inc(next_subscriber_id: &mut usize) -> usize { + let ans = *next_subscriber_id; + *next_subscriber_id += 1; + ans +} +type Callback = Box; + +/// A handle to a subscription created by GPUI. When dropped, the subscription +/// is cancelled and the callback will no longer be invoked. +#[must_use] +pub struct Subscription { + unsubscribe: Weak>>, +} + +impl std::fmt::Debug for Subscription { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Subscription").finish() + } +} + +impl Subscription { + /// Detaches the subscription from this handle. The callback will + /// continue to be invoked until the doc has been subscribed to + /// are dropped + pub fn detach(self) { + if let Some(unsubscribe) = self.unsubscribe.upgrade() { + unsubscribe.lock().take(); + } + } + + /// Unsubscribes the subscription. + #[inline] + pub fn unsubscribe(self) { + drop(self) + } +} + +impl Drop for Subscription { + fn drop(&mut self) { + if let Some(unsubscribe) = self.unsubscribe.upgrade() { + let unsubscribe = unsubscribe.lock().take(); + if let Some(unsubscribe) = unsubscribe { + unsubscribe(); + } + } + } +} + +struct InnerSubscription { + unsubscribe: Arc>>, +} + +impl Drop for InnerSubscription { + fn drop(&mut self) { + self.unsubscribe.lock().take(); + } +} + +/// A wrapper around `SubscriberSet` that automatically handles recursive event emission. +/// +/// This struct differs from `SubscriberSet` in the following ways: +/// 1. It automatically handles the `CannotEmitEventDueToRecursiveCall` error that can occur in `SubscriberSet`. +/// 2. When a recursive event emission is detected, it queues the event instead of throwing an error. +/// 3. After the current event processing is complete, it automatically processes the queued events. +/// +/// This behavior ensures that all events are processed in the order they were emitted, even in cases +/// where recursive event emission would normally cause an error. +#[derive(Clone)] +pub struct SubscriberSetWithQueue { + subscriber_set: SubscriberSet, + queue: Arc>>>, +} + +impl Debug + for SubscriberSetWithQueue +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SubscriberSetWithQueue").finish() + } +} + +impl Default + for SubscriberSetWithQueue +where + EmitterKey: 'static + Ord + Clone + Debug + Send + Sync, + Callback: 'static + Send + Sync + for<'a> FnMut(&'a Payload) -> bool, + Payload: Send + Sync + Debug, +{ + fn default() -> Self { + Self::new() + } +} + +pub struct WeakSubscriberSetWithQueue { + subscriber_set: Weak>>, + queue: Weak>>>, +} + +impl Debug + for WeakSubscriberSetWithQueue +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("WeakSubscriberSetWithQueue").finish() + } +} + +impl Clone + for WeakSubscriberSetWithQueue +{ + fn clone(&self) -> Self { + Self { + subscriber_set: self.subscriber_set.clone(), + queue: self.queue.clone(), + } + } +} + +impl WeakSubscriberSetWithQueue { + pub fn upgrade(self) -> Option> { + Some(SubscriberSetWithQueue { + subscriber_set: SubscriberSet(self.subscriber_set.upgrade()?), + queue: self.queue.upgrade()?, + }) + } +} + +impl SubscriberSetWithQueue +where + EmitterKey: 'static + Ord + Clone + Debug + Send + Sync, + Callback: 'static + Send + Sync + for<'a> FnMut(&'a Payload) -> bool, + Payload: Send + Sync + Debug, +{ + pub fn new() -> Self { + Self { + subscriber_set: SubscriberSet::new(), + queue: Arc::new(Mutex::new(Default::default())), + } + } + + pub fn downgrade(&self) -> WeakSubscriberSetWithQueue { + WeakSubscriberSetWithQueue { + subscriber_set: Arc::downgrade(&self.subscriber_set.0), + queue: Arc::downgrade(&self.queue), + } + } + + pub fn inner(&self) -> &SubscriberSet { + &self.subscriber_set + } + + pub fn emit(&self, key: &EmitterKey, payload: Payload) { + let mut pending_events: SmallVec<[Payload; 1]> = SmallVec::new(); + pending_events.push(payload); + while let Some(payload) = pending_events.pop() { + let result = self + .subscriber_set + .retain(key, &mut |callback| (callback)(&payload)); + match result { + Ok(_) => { + let mut queue = self.queue.lock(); + if let Some(new_pending_events) = queue.remove(key) { + pending_events.extend(new_pending_events); + } + } + Err(SubscriptionError::CannotEmitEventDueToRecursiveCall) => { + let mut queue = self.queue.lock(); + queue.entry(key.clone()).or_default().push(payload); + } + } + } + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_inner_subscription_drop() { + let subscriber_set = SubscriberSet:: bool + Send + Sync>>::new(); + let (subscription, activate) = subscriber_set.insert(1, Box::new(move |_: &i32| true)); + activate(); + drop(subscriber_set); + assert!(subscription.unsubscribe.upgrade().is_none()); + } + + #[test] + fn test_inner_subscription_drop_2() { + let subscriber_set = SubscriberSet:: bool + Send + Sync>>::new(); + let (subscription, activate) = subscriber_set.insert(1, Box::new(move |_: &i32| false)); + activate(); + subscriber_set + .retain(&1, &mut |callback| callback(&1)) + .unwrap(); + assert!(subscription.unsubscribe.upgrade().is_none()); + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/utils/utf16.rs b/3rdparty/loro/crates/loro-internal/src/utils/utf16.rs new file mode 100644 index 0000000000..cc65d22bd0 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/utils/utf16.rs @@ -0,0 +1,116 @@ +pub fn count_utf16_len(utf8_str: &[u8]) -> usize { + let mut utf16_count = 0; + + let mut iter = utf8_str.iter(); + + while let Some(&byte) = iter.next() { + if byte & 0b1000_0000 == 0 { + utf16_count += 1; + } else if byte & 0b1110_0000 == 0b1100_0000 { + let _ = iter.next(); + + utf16_count += 1; + } else if byte & 0b1111_0000 == 0b1110_0000 { + let _ = iter.next(); + let _ = iter.next(); + + utf16_count += 1; + } else if byte & 0b1111_1000 == 0b1111_0000 { + let u = (((byte & 0b0000_0111) as u32) << 18) + | (((iter.next().unwrap() & 0b0011_1111) as u32) << 12); + + let _ = iter.next(); + let _ = iter.next(); + if u >= 0x10000 { + utf16_count += 2; + } else { + utf16_count += 1; + } + } else { + unreachable!() + } + } + + utf16_count +} + +// TODO: FIXME: Tests +/// Count unicode chars in a utf8 string +pub fn count_unicode_chars(s: &[u8]) -> usize { + std::str::from_utf8(s).unwrap().chars().count() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_ascii() { + let input = "Hello, world!"; + let expected = 13; + assert_eq!(count_utf16_len(input.as_bytes()), expected); + } + + #[test] + fn test_2_byte() { + let input = "Привет, мир!"; + let expected = 12; + assert_eq!(count_utf16_len(input.as_bytes()), expected); + } + + #[test] + fn test_3_byte() { + let input = "こんにちは世界"; + let expected = 7; + assert_eq!(count_utf16_len(input.as_bytes()), expected); + } + + #[test] + fn test_4_byte() { + let input = "👋🌍"; + let expected = 4; + assert_eq!(count_utf16_len(input.as_bytes()), expected); + } + + #[test] + fn test_crazy_emoji_bytes() { + let input = "👩‍👩‍👧‍👧👨‍👨‍👧"; + let expected = 19; + assert_eq!(count_utf16_len(input.as_bytes()), expected); + } + + #[test] + fn test_empty_string() { + let input = ""; + let expected = 0; + assert_eq!(count_utf16_len(input.as_bytes()), expected); + } + + #[test] + fn test_single_char() { + let input = "a"; + let expected = 1; + assert_eq!(count_utf16_len(input.as_bytes()), expected); + } + + #[test] + fn test_utf8_with_bom() { + let input = "\u{FEFF}Hello, world!"; // UTF-8 with BOM + let expected = 14; + assert_eq!(count_utf16_len(input.as_bytes()), expected); + } + + #[test] + fn test_long_string() { + let input = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; + let expected = 52; + assert_eq!(count_utf16_len(input.as_bytes()), expected); + } + + #[test] + fn test_utf8_with_null_char() { + let input = "Hello\u{0000}world!"; + let expected = 12; + assert_eq!(count_utf16_len(input.as_bytes()), expected); + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/value.rs b/3rdparty/loro/crates/loro-internal/src/value.rs new file mode 100644 index 0000000000..8104eec606 --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/value.rs @@ -0,0 +1,1222 @@ +use crate::{ + container::richtext::richtext_state::{unicode_to_utf8_index, utf16_to_utf8_index}, + delta::{Delta, DeltaItem, Meta}, + event::{Diff, Index, Path, TextDiff, TextDiffItem, TextMeta}, + handler::ValueOrHandler, + utils::string_slice::StringSlice, +}; +use generic_btree::rle::HasLength; +use loro_common::ContainerType; +pub use loro_common::LoroValue; + +// TODO: rename this trait +pub trait ToJson { + fn to_json_value(&self) -> serde_json::Value; + fn to_json(&self) -> String { + self.to_json_value().to_string() + } + fn to_json_pretty(&self) -> String { + serde_json::to_string_pretty(&self.to_json_value()).unwrap() + } + fn from_json(s: &str) -> Self; +} + +impl ToJson for LoroValue { + fn to_json_value(&self) -> serde_json::Value { + serde_json::to_value(self).unwrap() + } + + fn to_json(&self) -> String { + serde_json::to_string(self).unwrap() + } + + fn to_json_pretty(&self) -> String { + serde_json::to_string_pretty(self).unwrap() + } + + #[allow(unused)] + fn from_json(s: &str) -> Self { + serde_json::from_str(s).unwrap() + } +} + +impl ToJson for DeltaItem { + fn to_json_value(&self) -> serde_json::Value { + match self { + DeltaItem::Retain { + retain: len, + attributes: meta, + } => { + let mut map = serde_json::Map::new(); + map.insert("retain".into(), serde_json::to_value(len).unwrap()); + if !meta.is_empty() { + map.insert("attributes".into(), meta.to_json_value()); + } + serde_json::Value::Object(map) + } + DeltaItem::Insert { + insert: value, + attributes: meta, + } => { + let mut map = serde_json::Map::new(); + map.insert("insert".into(), serde_json::to_value(value).unwrap()); + if !meta.is_empty() { + map.insert("attributes".into(), meta.to_json_value()); + } + serde_json::Value::Object(map) + } + DeltaItem::Delete { + delete: len, + attributes: _, + } => { + let mut map = serde_json::Map::new(); + map.insert("delete".into(), serde_json::to_value(len).unwrap()); + serde_json::Value::Object(map) + } + } + } + + fn from_json(s: &str) -> Self { + let map: serde_json::Map = serde_json::from_str(s).unwrap(); + if map.contains_key("retain") { + let len = map["retain"].as_u64().unwrap(); + let meta = if let Some(meta) = map.get("attributes") { + TextMeta::from_json(meta.to_string().as_str()) + } else { + TextMeta::default() + }; + DeltaItem::Retain { + retain: len as usize, + attributes: meta, + } + } else if map.contains_key("insert") { + let value = map["insert"].as_str().unwrap().to_string().into(); + let meta = if let Some(meta) = map.get("attributes") { + TextMeta::from_json(meta.to_string().as_str()) + } else { + TextMeta::default() + }; + DeltaItem::Insert { + insert: value, + attributes: meta, + } + } else if map.contains_key("delete") { + let len = map["delete"].as_u64().unwrap(); + DeltaItem::Delete { + delete: len as usize, + attributes: Default::default(), + } + } else { + panic!("Invalid delta item: {}", s); + } + } +} + +fn diff_item_to_json_value(item: &TextDiffItem) -> (serde_json::Value, Option) { + match item { + loro_delta::DeltaItem::Retain { len, attr } => { + let mut map = serde_json::Map::new(); + map.insert("retain".into(), serde_json::to_value(len).unwrap()); + if !attr.is_empty() { + map.insert("attributes".into(), attr.to_json_value()); + } + (serde_json::Value::Object(map), None) + } + loro_delta::DeltaItem::Replace { + value, + attr, + delete, + } => { + let mut a = None; + let mut b = None; + if value.rle_len() > 0 { + let mut map = serde_json::Map::new(); + map.insert("insert".into(), serde_json::to_value(value).unwrap()); + if !attr.is_empty() { + map.insert("attributes".into(), attr.to_json_value()); + } + a = Some(serde_json::Value::Object(map)); + } + if *delete > 0 { + let mut map = serde_json::Map::new(); + map.insert("delete".into(), serde_json::to_value(delete).unwrap()); + b = Some(serde_json::Value::Object(map)); + } + + if a.is_none() { + a = std::mem::take(&mut b); + if a.is_none() { + let mut map = serde_json::Map::new(); + map.insert("retain".into(), serde_json::to_value(0).unwrap()); + a = Some(serde_json::Value::Object(map)); + } + } + (a.unwrap(), b) + } + } +} + +fn diff_item_from_json(v: serde_json::Value) -> TextDiffItem { + let serde_json::Value::Object(map) = v else { + panic!("Invalid delta item: {:?}", v); + }; + if map.contains_key("retain") { + let len = map["retain"].as_u64().unwrap(); + let meta = if let Some(meta) = map.get("attributes") { + TextMeta::from_json(meta.to_string().as_str()) + } else { + TextMeta::default() + }; + TextDiffItem::Retain { + len: len as usize, + attr: meta, + } + } else if map.contains_key("insert") { + let value = map["insert"].as_str().unwrap().to_string().into(); + let meta = if let Some(meta) = map.get("attributes") { + TextMeta::from_json(meta.to_string().as_str()) + } else { + TextMeta::default() + }; + TextDiffItem::Replace { + value, + attr: meta, + delete: 0, + } + } else if map.contains_key("delete") { + let len = map["delete"].as_u64().unwrap(); + TextDiffItem::new_delete(len as usize) + } else { + panic!("Invalid delta item: {:?}", map); + } +} + +impl ToJson for TextDiff { + fn to_json_value(&self) -> serde_json::Value { + let mut vec = Vec::new(); + for item in self.iter() { + let (a, b) = diff_item_to_json_value(item); + vec.push(a); + if let Some(b) = b { + vec.push(b); + } + } + serde_json::Value::Array(vec) + } + + fn from_json(s: &str) -> Self { + let vec: Vec = serde_json::from_str(s).unwrap(); + let mut ans = TextDiff::new(); + for item in vec.into_iter() { + ans.push(diff_item_from_json(item)); + } + ans + } +} + +impl ToJson for Delta { + fn to_json_value(&self) -> serde_json::Value { + let mut vec = Vec::new(); + for item in self.iter() { + vec.push(item.to_json_value()); + } + serde_json::Value::Array(vec) + } + + fn from_json(s: &str) -> Self { + let vec: Vec = serde_json::from_str(s).unwrap(); + let mut ans = Delta::new(); + for item in vec.into_iter() { + ans.push(DeltaItem::from_json(item.to_string().as_str())); + } + ans + } +} + +#[derive(Debug, PartialEq, Eq)] +enum TypeHint { + Map, + Text, + List, + Tree, + #[cfg(feature = "counter")] + Counter, +} + +pub trait ApplyDiff { + fn apply_diff_shallow(&mut self, diff: &[Diff]); + fn apply_diff(&mut self, diff: &[Diff]); + fn apply(&mut self, path: &Path, diff: &[Diff]); +} + +impl ApplyDiff for LoroValue { + fn apply_diff_shallow(&mut self, diff: &[Diff]) { + match self { + LoroValue::String(value) => { + let mut s = value.to_string(); + for item in diff.iter() { + let delta = item.as_text().unwrap(); + let mut index = 0; + for delta_item in delta.iter() { + match delta_item { + loro_delta::DeltaItem::Retain { len, attr: _ } => { + index += len; + } + loro_delta::DeltaItem::Replace { + value, + attr: _, + delete, + } => { + let (start, end) = if cfg!(feature = "wasm") { + ( + utf16_to_utf8_index(&s, index).unwrap(), + utf16_to_utf8_index(&s, index + *delete).unwrap(), + ) + } else { + ( + unicode_to_utf8_index(&s, index).unwrap(), + unicode_to_utf8_index(&s, index + *delete).unwrap(), + ) + }; + s.replace_range(start..end, value.as_str()); + index += value.len_bytes(); + } + } + } + } + *value = s.into() + } + LoroValue::List(seq) => { + let is_tree = matches!(diff.first(), Some(Diff::Tree(_))); + if !is_tree { + let seq = seq.make_mut(); + for item in diff.iter() { + let delta = item.as_list().unwrap(); + let mut index = 0; + for delta_item in delta.iter() { + match delta_item { + loro_delta::DeltaItem::Retain { len, attr: _ } => { + index += len; + } + loro_delta::DeltaItem::Replace { + value, + attr: _, + delete, + } => { + let len = value.len(); + seq.splice( + index..index + delete, + value.iter().map(|x| x.to_value()), + ); + index += len; + } + } + } + } + } else { + // let seq = Arc::make_mut(seq); + // for item in diff.iter() { + // match item { + // Diff::Tree(tree) => { + // let mut v = TreeValue(seq); + // v.apply_diff(tree); + // } + // _ => unreachable!(), + // } + // } + unimplemented!() + } + } + LoroValue::Map(map) => { + for item in diff.iter() { + match item { + Diff::Map(diff) => { + let map = map.make_mut(); + for (key, value) in diff.updated.iter() { + match &value.value { + Some(value) => { + map.insert(key.to_string(), value.to_value()); + } + None => { + map.remove(&key.to_string()); + } + } + } + } + _ => unreachable!(), + } + } + } + _ => unreachable!(), + } + } + + fn apply_diff(&mut self, diff: &[Diff]) { + match self { + LoroValue::String(value) => { + let mut s = value.to_string(); + for item in diff.iter() { + let delta = item.as_text().unwrap(); + let mut index = 0; + for delta_item in delta.iter() { + match delta_item { + loro_delta::DeltaItem::Retain { len, attr: _ } => { + index += len; + } + loro_delta::DeltaItem::Replace { + value, + attr: _, + delete, + } => { + s.replace_range(index..index + *delete, value.as_str()); + index += value.len_bytes(); + } + } + } + } + *value = s.into(); + } + LoroValue::List(seq) => { + let is_tree = matches!(diff.first(), Some(Diff::Tree(_))); + if !is_tree { + let seq = seq.make_mut(); + for item in diff.iter() { + let delta = item.as_list().unwrap(); + let mut index = 0; + for delta_item in delta.iter() { + match delta_item { + loro_delta::DeltaItem::Retain { len, .. } => { + index += len; + } + loro_delta::DeltaItem::Replace { + value, + attr: _, + delete, + } => { + let value_iter = value.iter().map(unresolved_to_collection); + seq.splice(index..index + *delete, value_iter); + index += value.len(); + } + } + } + } + } else { + // let seq = Arc::make_mut(seq); + // for item in diff.iter() { + // match item { + // Diff::Tree(tree) => { + // let mut v = TreeValue(seq); + // v.apply_diff(tree); + // } + // _ => unreachable!(), + // } + // } + unimplemented!() + } + } + LoroValue::Map(map) => { + for item in diff.iter() { + match item { + Diff::Map(diff) => { + let map = map.make_mut(); + for (key, value) in diff.updated.iter() { + match &value.value { + Some(value) => { + map.insert( + key.to_string(), + unresolved_to_collection(value), + ); + } + None => { + map.remove(&key.to_string()); + } + } + } + } + _ => unreachable!(), + } + } + } + _ => unreachable!(), + } + } + + fn apply(&mut self, path: &Path, diff: &[Diff]) { + if diff.is_empty() { + return; + } + + let hint = match diff[0] { + Diff::List(_) => TypeHint::List, + Diff::Text(_) => TypeHint::Text, + Diff::Map(_) => TypeHint::Map, + Diff::Tree(_) => TypeHint::Tree, + #[cfg(feature = "counter")] + Diff::Counter(_) => TypeHint::Counter, + Diff::Unknown => unreachable!(), + }; + let value = { + let mut hints = Vec::with_capacity(path.len()); + for item in path.iter().skip(1) { + match item { + Index::Key(_) => hints.push(TypeHint::Map), + Index::Seq(_) => hints.push(TypeHint::List), + Index::Node(_) => hints.push(TypeHint::Tree), + } + } + + hints.push(hint); + let mut value: &mut LoroValue = self; + for (item, hint) in path.iter().zip(hints.iter()) { + match item { + Index::Key(key) => { + let m = value.as_map_mut().unwrap(); + let map = m.make_mut(); + value = map.entry(key.to_string()).or_insert_with(|| match hint { + TypeHint::Map => LoroValue::Map(Default::default()), + TypeHint::Text => LoroValue::String(Default::default()), + TypeHint::List => LoroValue::List(Default::default()), + TypeHint::Tree => LoroValue::List(Default::default()), + #[cfg(feature = "counter")] + TypeHint::Counter => LoroValue::Double(0.), + }) + } + Index::Seq(index) => { + let l = value.as_list_mut().unwrap(); + let list = l.make_mut(); + value = list.get_mut(*index).unwrap(); + } + Index::Node(tree_id) => { + let l = value.as_list_mut().unwrap(); + let list = l.make_mut(); + let Some(map) = list.iter_mut().find(|x| { + let id = x.as_map().unwrap().get("id").unwrap().as_string().unwrap(); + id.as_ref() == tree_id.to_string() + }) else { + // delete node first + return; + }; + let map_mut = map.as_map_mut().unwrap().make_mut(); + let meta = map_mut.get_mut("meta").unwrap(); + if meta.is_container() { + *meta = ContainerType::Map.default_value(); + } + value = meta + } + } + } + value + }; + value.apply_diff(diff); + } +} + +pub(crate) fn unresolved_to_collection(v: &ValueOrHandler) -> LoroValue { + match v { + ValueOrHandler::Value(v) => v.clone(), + ValueOrHandler::Handler(c) => c.c_type().default_value(), + } +} + +#[cfg(test)] +mod tests { + use loro_common::IdLp; + use loro_delta::{array_vec::ArrayVec, DeltaItem as EventDeltaItem}; + use rustc_hash::FxHashMap; + + use crate::{ + delta::{Delta, DeltaItem}, + event::{ListDeltaMeta, ListDiff, ListDiffInsertItem}, + handler::{Handler, HandlerTrait}, + LoroDoc, + }; + + use super::*; + + fn text_diff(items: Vec) -> TextDiff { + let mut diff = TextDiff::new(); + for item in items { + diff.push(item); + } + diff + } + + fn list_values(values: Vec) -> ListDiffInsertItem { + let mut array = ArrayVec::new(); + for value in values { + array.push(ValueOrHandler::Value(value)).unwrap(); + } + array + } + + fn list_diff(items: Vec) -> ListDiff { + let mut diff = ListDiff::new(); + for item in items { + diff.push(item); + } + diff + } + + fn map_value(value: Option, lamport: u32) -> crate::delta::ResolvedMapValue { + crate::delta::ResolvedMapValue { + value: value.map(ValueOrHandler::Value), + idlp: IdLp::new(1, lamport), + } + } + + #[test] + fn text_delta_json_roundtrip_preserves_attributes_and_operation_order() { + let mut attributes = FxHashMap::default(); + attributes.insert("bold".to_string(), LoroValue::Bool(true)); + let meta = TextMeta(attributes); + + let item = DeltaItem::Insert { + insert: StringSlice::from("hi"), + attributes: meta.clone(), + }; + assert_eq!( + item.to_json_value(), + serde_json::json!({"insert": "hi", "attributes": {"bold": true}}) + ); + assert_eq!( + DeltaItem::::from_json( + r#"{"insert":"hi","attributes":{"bold":true}}"# + ), + item + ); + + let mut delta = Delta::::new(); + delta.push(DeltaItem::Retain { + retain: 2, + attributes: meta.clone(), + }); + delta.push(DeltaItem::Insert { + insert: StringSlice::from("!"), + attributes: TextMeta::default(), + }); + delta.push(DeltaItem::Delete { + delete: 1, + attributes: TextMeta::default(), + }); + + let json = delta.to_json(); + assert_eq!( + serde_json::from_str::(&json).unwrap(), + serde_json::json!([ + {"retain": 2, "attributes": {"bold": true}}, + {"insert": "!"}, + {"delete": 1} + ]) + ); + assert_eq!(Delta::::from_json(&json), delta); + } + + #[test] + fn text_diff_json_splits_replace_with_insert_and_delete_entries() { + let diff = text_diff(vec![TextDiffItem::Replace { + value: StringSlice::from("new"), + attr: TextMeta::default(), + delete: 2, + }]); + + assert_eq!( + diff.to_json_value(), + serde_json::json!([{"insert": "new"}, {"delete": 2}]) + ); + assert_eq!( + TextDiff::from_json(r#"[{"retain":1},{"insert":"x"},{"delete":2}]"#).to_json_value(), + serde_json::json!([{"retain": 1}, {"insert": "x"}, {"delete": 2}]) + ); + + let (empty_replace, follow_up) = diff_item_to_json_value(&TextDiffItem::Replace { + value: StringSlice::from(""), + attr: TextMeta::default(), + delete: 0, + }); + assert_eq!(empty_replace, serde_json::json!({"retain": 0})); + assert_eq!(follow_up, None); + } + + #[test] + fn apply_diff_updates_string_list_and_map_values() { + let mut text = LoroValue::String("aéz".into()); + text.apply_diff_shallow(&[Diff::Text(text_diff(vec![ + TextDiffItem::Retain { + len: 1, + attr: TextMeta::default(), + }, + TextDiffItem::Replace { + value: StringSlice::from("bc"), + attr: TextMeta::default(), + delete: 1, + }, + ]))]); + assert_eq!(text, LoroValue::String("abcz".into())); + + let mut list = + LoroValue::List(vec![LoroValue::I64(1), LoroValue::I64(2), LoroValue::I64(3)].into()); + list.apply_diff(&[Diff::List(list_diff(vec![ + EventDeltaItem::Retain { + len: 1, + attr: ListDeltaMeta::default(), + }, + EventDeltaItem::Replace { + value: list_values(vec![LoroValue::I64(20), LoroValue::I64(21)]), + attr: ListDeltaMeta::default(), + delete: 1, + }, + ]))]); + assert_eq!( + list, + LoroValue::List( + vec![ + LoroValue::I64(1), + LoroValue::I64(20), + LoroValue::I64(21), + LoroValue::I64(3), + ] + .into() + ) + ); + + let mut map = LoroValue::Map( + FxHashMap::from_iter([ + ("keep".to_string(), LoroValue::I64(1)), + ("remove".to_string(), LoroValue::I64(2)), + ]) + .into(), + ); + map.apply_diff(&[Diff::Map( + crate::delta::ResolvedMapDelta::new() + .with_entry("add".into(), map_value(Some(LoroValue::I64(3)), 1)) + .with_entry("remove".into(), map_value(None, 2)), + )]); + let map = map.as_map().unwrap(); + assert_eq!(map.get("keep"), Some(&LoroValue::I64(1))); + assert_eq!(map.get("add"), Some(&LoroValue::I64(3))); + assert_eq!(map.get("remove"), None); + } + + #[test] + fn apply_creates_missing_nested_values_from_path_and_diff_hint() { + let mut root = LoroValue::Map(Default::default()); + let mut title_path = Path::new(); + title_path.push(Index::Key("doc".into())); + title_path.push(Index::Key("title".into())); + root.apply( + &title_path, + &[Diff::Text(text_diff(vec![TextDiffItem::Replace { + value: StringSlice::from("Loro"), + attr: TextMeta::default(), + delete: 0, + }]))], + ); + + assert_eq!( + root.to_json_value(), + serde_json::json!({"doc": {"title": "Loro"}}) + ); + + let mut tags_path = Path::new(); + tags_path.push(Index::Key("doc".into())); + tags_path.push(Index::Key("tags".into())); + root.apply( + &tags_path, + &[Diff::List(list_diff(vec![EventDeltaItem::Replace { + value: list_values(vec![ + LoroValue::String("crdt".into()), + LoroValue::String("rust".into()), + ]), + attr: ListDeltaMeta::default(), + delete: 0, + }]))], + ); + assert_eq!( + root.to_json_value(), + serde_json::json!({"doc": {"title": "Loro", "tags": ["crdt", "rust"]}}) + ); + } + + #[test] + fn unresolved_container_handlers_apply_as_default_collection_values() { + let doc = LoroDoc::new_auto_commit(); + let map_handler = Handler::new_unattached(ContainerType::Map); + let text_handler = Handler::new_unattached(ContainerType::Text); + let attached_list = doc.get_list("list").to_handler(); + + assert_eq!( + unresolved_to_collection(&ValueOrHandler::Handler(map_handler)), + LoroValue::Map(Default::default()) + ); + assert_eq!( + unresolved_to_collection(&ValueOrHandler::Handler(text_handler)), + LoroValue::String(Default::default()) + ); + assert_eq!( + unresolved_to_collection(&ValueOrHandler::Handler(attached_list)), + LoroValue::List(Default::default()) + ); + assert_eq!( + unresolved_to_collection(&ValueOrHandler::Value(LoroValue::Bool(true))), + LoroValue::Bool(true) + ); + } +} + +#[cfg(feature = "wasm")] +pub mod wasm { + use crate::{ + delta::{Delta, DeltaItem, Meta, StyleMeta, TreeDiff, TreeDiffItem, TreeExternalDiff}, + event::{Index, TextDiff, TextDiffItem, TextMeta}, + utils::string_slice::StringSlice, + TreeParentId, + }; + use fractional_index::FractionalIndex; + use generic_btree::rle::HasLength; + use js_sys::{Array, Object}; + use loro_common::{LoroValue, TreeID}; + use wasm_bindgen::{__rt::IntoJsResult, JsCast, JsValue}; + + impl From for JsValue { + fn from(value: Index) -> Self { + match value { + Index::Key(key) => JsValue::from_str(&key), + Index::Seq(num) => JsValue::from_f64(num as f64), + Index::Node(node) => node.into(), + } + } + } + + impl From<&TreeDiff> for JsValue { + fn from(value: &TreeDiff) -> Self { + let array = Array::new(); + for diff in value.diff.iter() { + let obj = Object::new(); + js_sys::Reflect::set(&obj, &"target".into(), &diff.target.into()).unwrap(); + match &diff.action { + TreeExternalDiff::Create { + parent, + index, + position, + } => { + js_sys::Reflect::set(&obj, &"action".into(), &"create".into()).unwrap(); + js_sys::Reflect::set( + &obj, + &"parent".into(), + &JsValue::from(parent.tree_id()), + ) + .unwrap(); + js_sys::Reflect::set(&obj, &"index".into(), &(*index).into()).unwrap(); + js_sys::Reflect::set( + &obj, + &"fractionalIndex".into(), + &position.to_string().into(), + ) + .unwrap(); + } + TreeExternalDiff::Delete { + old_parent, + old_index, + } => { + js_sys::Reflect::set(&obj, &"action".into(), &"delete".into()).unwrap(); + js_sys::Reflect::set( + &obj, + &"oldParent".into(), + &JsValue::from(old_parent.tree_id()), + ) + .unwrap(); + js_sys::Reflect::set(&obj, &"oldIndex".into(), &(*old_index).into()) + .unwrap(); + } + TreeExternalDiff::Move { + parent, + index, + position, + old_parent, + old_index, + } => { + js_sys::Reflect::set(&obj, &"action".into(), &"move".into()).unwrap(); + js_sys::Reflect::set( + &obj, + &"parent".into(), + &JsValue::from(parent.tree_id()), + ) + .unwrap(); + js_sys::Reflect::set(&obj, &"index".into(), &(*index).into()).unwrap(); + js_sys::Reflect::set( + &obj, + &"fractionalIndex".into(), + &position.to_string().into(), + ) + .unwrap(); + js_sys::Reflect::set( + &obj, + &"oldParent".into(), + &JsValue::from(old_parent.tree_id()), + ) + .unwrap(); + js_sys::Reflect::set(&obj, &"oldIndex".into(), &(*old_index).into()) + .unwrap(); + } + } + array.push(&obj); + } + array.into_js_result().unwrap() + } + } + + impl TryFrom<&JsValue> for TreeDiff { + type Error = String; + + fn try_from(value: &JsValue) -> Result { + if !value.is_array() { + return Err("Expected an array".to_string()); + } + + let array = js_sys::Array::from(value); + let mut diff = Vec::new(); + + for i in 0..array.length() { + let item = array.get(i); + if !item.is_object() { + return Err(format!("Item at index {i} is not an object")); + } + + let obj = js_sys::Object::from(item); + let target = js_sys::Reflect::get(&obj, &"target".into()) + .map_err(|e| format!("Failed to get target: {:?}", e))?; + let target = TreeID::try_from(target) + .map_err(|e| format!("Failed to parse target: {:?}", e))?; + + let action = js_sys::Reflect::get(&obj, &"action".into()) + .map_err(|e| format!("Failed to get action: {:?}", e))?; + let action = action + .as_string() + .ok_or_else(|| "action is not a string".to_string())?; + + let action = match action.as_str() { + "create" => { + let parent = js_sys::Reflect::get(&obj, &"parent".into()) + .map_err(|e| format!("Failed to get parent: {:?}", e))?; + let parent_id = if parent.is_null() || parent.is_undefined() { + None + } else { + Some( + TreeID::try_from(parent) + .map_err(|e| format!("Failed to parse parent: {:?}", e))?, + ) + }; + let parent = TreeParentId::from(parent_id); + let index = js_sys::Reflect::get(&obj, &"index".into()) + .map_err(|e| format!("Failed to get index: {:?}", e))?; + let index = index + .as_f64() + .ok_or_else(|| "index is not a number".to_string())? + as usize; + + let position = js_sys::Reflect::get(&obj, &"fractionalIndex".into()) + .map_err(|e| format!("Failed to get fractionalIndex: {:?}", e))?; + let position = position + .as_string() + .ok_or_else(|| "fractionalIndex is not a string".to_string())?; + let position = FractionalIndex::from_hex_string(position); + + TreeExternalDiff::Create { + parent, + index, + position, + } + } + "move" => { + let parent = js_sys::Reflect::get(&obj, &"parent".into()) + .map_err(|e| format!("Failed to get parent: {:?}", e))?; + let parent_id = if parent.is_null() || parent.is_undefined() { + None + } else { + Some( + TreeID::try_from(parent) + .map_err(|e| format!("Failed to parse parent: {:?}", e))?, + ) + }; + let parent = TreeParentId::from(parent_id); + + let index = js_sys::Reflect::get(&obj, &"index".into()) + .map_err(|e| format!("Failed to get index: {:?}", e))?; + let index = index + .as_f64() + .ok_or_else(|| "index is not a number".to_string())? + as usize; + + let position = js_sys::Reflect::get(&obj, &"fractionalIndex".into()) + .map_err(|e| format!("Failed to get fractionalIndex: {:?}", e))?; + let position = position + .as_string() + .ok_or_else(|| "fractionalIndex is not a string".to_string())?; + let position = FractionalIndex::from_hex_string(position); + + let old_parent = js_sys::Reflect::get(&obj, &"oldParent".into()) + .map_err(|e| format!("Failed to get oldParent: {:?}", e))?; + let old_parent_id = if old_parent.is_null() || old_parent.is_undefined() { + None + } else { + Some( + TreeID::try_from(old_parent) + .map_err(|e| format!("Failed to parse oldParent: {:?}", e))?, + ) + }; + let old_parent = TreeParentId::from(old_parent_id); + + let old_index = js_sys::Reflect::get(&obj, &"oldIndex".into()) + .map_err(|e| format!("Failed to get oldIndex: {:?}", e))?; + let old_index = old_index + .as_f64() + .ok_or_else(|| "oldIndex is not a number".to_string())? + as usize; + + TreeExternalDiff::Move { + parent, + index, + position, + old_parent, + old_index, + } + } + "delete" => { + let old_parent = js_sys::Reflect::get(&obj, &"oldParent".into()) + .map_err(|e| format!("Failed to get oldParent: {:?}", e))?; + let old_parent_id = if old_parent.is_null() || old_parent.is_undefined() { + None + } else { + Some( + TreeID::try_from(old_parent) + .map_err(|e| format!("Failed to parse oldParent: {:?}", e))?, + ) + }; + let old_parent = TreeParentId::from(old_parent_id); + + let old_index = js_sys::Reflect::get(&obj, &"oldIndex".into()) + .map_err(|e| format!("Failed to get oldIndex: {:?}", e))?; + let old_index = old_index + .as_f64() + .ok_or_else(|| "oldIndex is not a number".to_string())? + as usize; + + TreeExternalDiff::Delete { + old_parent, + old_index, + } + } + action => Err(format!("Unknown tree diff action: {action}"))?, + }; + + diff.push(TreeDiffItem { target, action }); + } + + Ok(TreeDiff { diff }) + } + } + + impl From<&Delta> for JsValue { + fn from(value: &Delta) -> Self { + let arr = Array::new_with_length(value.len() as u32); + for (i, v) in value.iter().enumerate() { + arr.set(i as u32, JsValue::from(v.clone())); + } + + arr.into_js_result().unwrap() + } + } + + impl From> for JsValue { + fn from(value: DeltaItem) -> Self { + let obj = Object::new(); + match value { + DeltaItem::Retain { + retain: len, + attributes: meta, + } => { + js_sys::Reflect::set( + &obj, + &JsValue::from_str("retain"), + &JsValue::from_f64(len as f64), + ) + .unwrap(); + if !meta.is_empty() { + js_sys::Reflect::set( + &obj, + &JsValue::from_str("attributes"), + &JsValue::from(&meta), + ) + .unwrap(); + } + } + DeltaItem::Insert { + insert: value, + attributes: meta, + } => { + js_sys::Reflect::set( + &obj, + &JsValue::from_str("insert"), + &JsValue::from_str(value.as_str()), + ) + .unwrap(); + if !meta.is_empty() { + js_sys::Reflect::set( + &obj, + &JsValue::from_str("attributes"), + &JsValue::from(&meta), + ) + .unwrap(); + } + } + DeltaItem::Delete { + delete: len, + attributes: _, + } => { + js_sys::Reflect::set( + &obj, + &JsValue::from_str("delete"), + &JsValue::from_f64(len as f64), + ) + .unwrap(); + } + } + + obj.into_js_result().unwrap() + } + } + + pub fn text_diff_to_js_value(diff: &TextDiff) -> JsValue { + let arr = Array::new(); + let mut i = 0; + for v in diff.iter() { + let (a, b) = text_diff_item_to_js_value(v); + arr.set(i as u32, a); + i += 1; + if let Some(b) = b { + arr.set(i as u32, b); + i += 1; + } + } + + arr.into_js_result().unwrap() + } + + fn text_diff_item_to_js_value(value: &TextDiffItem) -> (JsValue, Option) { + match value { + loro_delta::DeltaItem::Retain { len, attr } => { + let obj = Object::new(); + js_sys::Reflect::set( + &obj, + &JsValue::from_str("retain"), + &JsValue::from_f64(*len as f64), + ) + .unwrap(); + if !attr.is_empty() { + js_sys::Reflect::set( + &obj, + &JsValue::from_str("attributes"), + &JsValue::from(attr), + ) + .unwrap(); + } + (obj.into_js_result().unwrap(), None) + } + loro_delta::DeltaItem::Replace { + value, + attr, + delete, + } => { + let mut a = None; + let mut b = None; + if value.rle_len() > 0 { + let obj = Object::new(); + js_sys::Reflect::set( + &obj, + &JsValue::from_str("insert"), + &JsValue::from_str(value.as_str()), + ) + .unwrap(); + if !attr.is_empty() { + js_sys::Reflect::set( + &obj, + &JsValue::from_str("attributes"), + &JsValue::from(attr), + ) + .unwrap(); + } + a = Some(obj.into_js_result().unwrap()); + } + + if *delete > 0 { + let obj = Object::new(); + js_sys::Reflect::set( + &obj, + &JsValue::from_str("delete"), + &JsValue::from_f64(*delete as f64), + ) + .unwrap(); + b = Some(obj.into_js_result().unwrap()); + } + + if a.is_none() { + a = std::mem::take(&mut b); + } + + (a.unwrap(), b) + } + } + } + + impl From<&StyleMeta> for JsValue { + fn from(value: &StyleMeta) -> Self { + // TODO: refactor: should we extract the common code of ToJson and ToJsValue + let obj = Object::new(); + for (key, style) in value.iter() { + let value = JsValue::from(style.data); + js_sys::Reflect::set(&obj, &JsValue::from_str(&key), &value).unwrap(); + } + + obj.into_js_result().unwrap() + } + } + + impl From<&TextMeta> for JsValue { + fn from(value: &TextMeta) -> Self { + let obj = Object::new(); + for (key, value) in value.0.iter() { + js_sys::Reflect::set(&obj, &JsValue::from_str(key), &JsValue::from(value.clone())) + .unwrap(); + } + + obj.into_js_result().unwrap() + } + } + + impl TryFrom<&JsValue> for TextMeta { + type Error = JsValue; + + fn try_from(value: &JsValue) -> Result { + if value.is_null() || value.is_undefined() { + return Ok(TextMeta::default()); + } + + let obj = value.dyn_ref::().ok_or("Expected an object")?; + let mut meta = TextMeta::default(); + + let entries = Object::entries(obj); + for i in 0..entries.length() { + let entry = entries.get(i); + let entry_arr = entry.dyn_ref::().ok_or("Expected an array")?; + let key = entry_arr + .get(0) + .as_string() + .ok_or("Expected a string key")?; + let value = entry_arr.get(1); + meta.0.insert(key, LoroValue::from(value)); + } + + Ok(meta) + } + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/version.rs b/3rdparty/loro/crates/loro-internal/src/version.rs new file mode 100644 index 0000000000..1c6ea17a4a --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/version.rs @@ -0,0 +1,1197 @@ +mod frontiers; +pub use frontiers::Frontiers; + +use crate::{ + id::{Counter, ID}, + oplog::AppDag, + span::{CounterSpan, IdSpan}, + LoroError, PeerID, +}; +use loro_common::{HasCounter, HasCounterSpan, HasIdSpan, HasLamportSpan, IdFull, IdSpanVector}; +use rustc_hash::FxHashMap; +use serde::{Deserialize, Serialize}; +use smallvec::SmallVec; +use std::{ + cmp::Ordering, + ops::{ControlFlow, Deref, DerefMut}, +}; + +/// [VersionVector](https://en.wikipedia.org/wiki/Version_vector) +/// is a map from [PeerID] to [Counter]. Its a right-open interval. +/// +/// i.e. a [VersionVector] of `{A: 1, B: 2}` means that A has 1 atomic op and B has 2 atomic ops, +/// thus ID of `{client: A, counter: 1}` is out of the range. +// +// NOTE: though normally it doesn't make sense to have an entry with counter = 0, but it can be 0 in certain cases. +// Like, when using start_vv and end_vv to mark the version range, start_vv may have an entry with counter = 0. +#[repr(transparent)] +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct VersionVector(FxHashMap); + +#[repr(transparent)] +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct VersionRange(pub(crate) FxHashMap); + +#[macro_export] +macro_rules! version_range { + ($($peer:expr => ($start:expr, $end:expr)),* $(,)?) => {{ + let mut map = ::rustc_hash::FxHashMap::default(); + $( + map.insert($peer, ($start, $end)); + )* + $crate::version::VersionRange::from_map(map) + }}; +} + +impl VersionRange { + pub fn new() -> Self { + Self(Default::default()) + } + + pub fn from_map(map: FxHashMap) -> Self { + Self(map) + } + + pub fn iter(&self) -> impl Iterator + '_ { + self.0.iter() + } + + pub fn iter_mut(&mut self) -> impl Iterator + '_ { + self.0.iter_mut() + } + + pub fn clear(&mut self) { + self.0.clear() + } + + pub fn get(&self, peer: &PeerID) -> Option<&(Counter, Counter)> { + self.0.get(peer) + } + + pub fn insert(&mut self, peer: PeerID, start: Counter, end: Counter) { + self.0.insert(peer, (start, end)); + } + + pub fn from_vv(vv: &VersionVector) -> Self { + let mut ans = Self::new(); + for (peer, counter) in vv.iter() { + let counter = normalize_vv_counter(*counter); + if counter > 0 { + ans.insert(*peer, 0, counter); + } + } + ans + } + + pub fn contains_ops_between(&self, vv_a: &VersionVector, vv_b: &VersionVector) -> bool { + for span in vv_a.sub_iter(vv_b) { + if !self.contains_id_span(IdSpan::new( + span.peer, + span.counter.start.saturating_sub(1), + span.counter.end, + )) { + return false; + } + } + + for span in vv_b.sub_iter(vv_a) { + if !self.contains_id_span(IdSpan::new( + span.peer, + span.counter.start.saturating_sub(1), + span.counter.end, + )) { + return false; + } + } + + true + } + + pub fn has_overlap_with(&self, mut span: IdSpan) -> bool { + span.normalize_(); + if let Some((start, end)) = self.get(&span.peer) { + start < &span.counter.end && end > &span.counter.start + } else { + false + } + } + + pub fn contains_id(&self, id: ID) -> bool { + if let Some((start, end)) = self.get(&id.peer) { + start <= &id.counter && end > &id.counter + } else { + false + } + } + + pub fn contains_id_span(&self, mut span: IdSpan) -> bool { + span.normalize_(); + if let Some((start, end)) = self.get(&span.peer) { + start <= &span.counter.start && end >= &span.counter.end + } else { + false + } + } + + pub fn extends_to_include_id_span(&mut self, mut span: IdSpan) { + span.normalize_(); + if let Some((start, end)) = self.0.get_mut(&span.peer) { + *start = (*start).min(span.counter.start); + *end = (*end).max(span.counter.end); + } else { + self.insert(span.peer, span.counter.start, span.counter.end); + } + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + pub fn inner(&self) -> &FxHashMap { + &self.0 + } +} + +/// Immutable version vector +/// +/// It has O(1) clone time and O(logN) insert/delete/lookup time. +/// +/// It's more memory efficient than [VersionVector] when the version vector +/// can be created from cloning and modifying other similar version vectors. +#[repr(transparent)] +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct ImVersionVector(im::HashMap); + +#[inline] +fn normalize_vv_counter(counter: Counter) -> Counter { + counter.max(0) +} + +#[inline] +fn last_id_to_vv_end(id: ID) -> Counter { + id.counter.saturating_add(1).max(0) +} + +impl ImVersionVector { + pub fn new() -> Self { + Self(Default::default()) + } + + pub fn clear(&mut self) { + self.0.clear() + } + + pub fn get(&self, key: &PeerID) -> Option<&Counter> { + self.0.get(key) + } + + pub fn get_mut(&mut self, key: &PeerID) -> Option<&mut Counter> { + self.0.get_mut(key) + } + + pub fn insert(&mut self, k: PeerID, v: Counter) { + self.0.insert(k, v); + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + pub fn iter(&self) -> im::hashmap::Iter<'_, PeerID, Counter> { + self.0.iter() + } + + pub fn remove(&mut self, k: &PeerID) -> Option { + self.0.remove(k) + } + + pub fn len(&self) -> usize { + self.0.len() + } + + pub fn contains_key(&self, k: &PeerID) -> bool { + self.0.contains_key(k) + } + + pub fn encode(&self) -> Vec { + postcard::to_allocvec(self).unwrap() + } + + pub fn decode(bytes: &[u8]) -> Result { + let vv = VersionVector::decode(bytes)?; + Ok(Self::from_vv(&vv)) + } + + pub fn to_vv(&self) -> VersionVector { + VersionVector(self.0.iter().map(|(&k, &v)| (k, v)).collect()) + } + + pub fn from_vv(vv: &VersionVector) -> Self { + ImVersionVector(vv.0.iter().map(|(&k, &v)| (k, v)).collect()) + } + + pub fn extend_to_include_vv<'a>( + &mut self, + vv: impl Iterator, + ) { + for (&client_id, &counter) in vv { + let counter = normalize_vv_counter(counter); + if counter == 0 { + continue; + } + + if let Some(my_counter) = self.0.get_mut(&client_id) { + *my_counter = normalize_vv_counter(*my_counter); + if *my_counter < counter { + *my_counter = counter; + } + } else { + self.0.insert(client_id, counter); + } + } + } + + #[inline] + pub fn merge(&mut self, other: &Self) { + self.extend_to_include_vv(other.0.iter()); + } + + #[inline] + pub fn merge_vv(&mut self, other: &VersionVector) { + self.extend_to_include_vv(other.0.iter()); + } + + #[inline] + pub fn set_last(&mut self, id: ID) { + let end = last_id_to_vv_end(id); + if end == 0 { + self.0.remove(&id.peer); + } else { + self.0.insert(id.peer, end); + } + } + + pub fn extend_to_include_last_id(&mut self, id: ID) { + let end = last_id_to_vv_end(id); + if end == 0 { + return; + } + + if let Some(counter) = self.0.get_mut(&id.peer) { + *counter = normalize_vv_counter(*counter); + if *counter < end { + *counter = end; + } + } else { + self.0.insert(id.peer, end); + } + } + + pub(crate) fn includes_id(&self, x: ID) -> bool { + if self.is_empty() || x.counter < 0 { + return false; + } + + normalize_vv_counter(self.get(&x.peer).copied().unwrap_or(0)) > x.counter + } +} + +impl PartialEq for VersionVector { + fn eq(&self, other: &Self) -> bool { + self.iter().all(|(client, counter)| { + normalize_vv_counter(*other.get(client).unwrap_or(&0)) == normalize_vv_counter(*counter) + }) && other.iter().all(|(client, counter)| { + normalize_vv_counter(*self.get(client).unwrap_or(&0)) == normalize_vv_counter(*counter) + }) + } +} + +impl Eq for VersionVector {} + +impl PartialEq for ImVersionVector { + fn eq(&self, other: &Self) -> bool { + self.0.iter().all(|(client, counter)| { + normalize_vv_counter(*other.0.get(client).unwrap_or(&0)) + == normalize_vv_counter(*counter) + }) && other.0.iter().all(|(client, counter)| { + normalize_vv_counter(*self.0.get(client).unwrap_or(&0)) + == normalize_vv_counter(*counter) + }) + } +} + +impl Eq for ImVersionVector {} + +impl Deref for VersionVector { + type Target = FxHashMap; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +#[derive(Default, Debug, PartialEq, Eq)] +pub struct VersionVectorDiff { + /// The spans that the `left` side needs to retreat to reach the `right` side + /// + /// these spans are included in the left, but not in the right + pub retreat: IdSpanVector, + /// The spans that the `left` side needs to forward to reach the `right` side + /// + /// these spans are included in the right, but not in the left + pub forward: IdSpanVector, +} + +impl VersionVectorDiff { + #[inline] + pub fn merge_left(&mut self, span: IdSpan) { + merge(&mut self.retreat, span); + } + + #[inline] + pub fn merge_right(&mut self, span: IdSpan) { + merge(&mut self.forward, span); + } + + #[inline] + pub fn subtract_start_left(&mut self, span: IdSpan) { + subtract_start(&mut self.retreat, span); + } + + #[inline] + pub fn subtract_start_right(&mut self, span: IdSpan) { + subtract_start(&mut self.forward, span); + } + + pub fn get_id_spans_left(&self) -> impl Iterator + '_ { + self.retreat.iter().map(|(peer, span)| IdSpan { + peer: *peer, + counter: *span, + }) + } + + pub fn get_id_spans_right(&self) -> impl Iterator + '_ { + self.forward.iter().map(|(peer, span)| IdSpan { + peer: *peer, + counter: *span, + }) + } +} + +fn subtract_start(m: &mut FxHashMap, target: IdSpan) { + if let Some(span) = m.get_mut(&target.peer) { + if span.start < target.counter.end { + span.start = target.counter.end; + } + } +} + +fn merge(m: &mut FxHashMap, mut target: IdSpan) { + target.normalize_(); + if let Some(span) = m.get_mut(&target.peer) { + span.start = span.start.min(target.counter.start); + span.end = span.end.max(target.counter.end); + } else { + m.insert(target.peer, target.counter); + } +} + +impl PartialOrd for VersionVector { + fn partial_cmp(&self, other: &Self) -> Option { + let mut self_greater = true; + let mut other_greater = true; + let mut eq = true; + for (client_id, other_end) in other.iter() { + let other_end = normalize_vv_counter(*other_end); + if let Some(self_end) = self.get(client_id) { + let self_end = normalize_vv_counter(*self_end); + if self_end < other_end { + self_greater = false; + eq = false; + } + if self_end > other_end { + other_greater = false; + eq = false; + } + } else if other_end > 0 { + self_greater = false; + eq = false; + } + } + + for (client_id, self_end) in self.iter() { + if other.contains_key(client_id) { + continue; + } else if normalize_vv_counter(*self_end) > 0 { + other_greater = false; + eq = false; + } + } + + if eq { + Some(Ordering::Equal) + } else if self_greater { + Some(Ordering::Greater) + } else if other_greater { + Some(Ordering::Less) + } else { + None + } + } +} + +impl PartialOrd for ImVersionVector { + fn partial_cmp(&self, other: &Self) -> Option { + let mut self_greater = true; + let mut other_greater = true; + let mut eq = true; + for (client_id, other_end) in other.iter() { + let other_end = normalize_vv_counter(*other_end); + if let Some(self_end) = self.get(client_id) { + let self_end = normalize_vv_counter(*self_end); + if self_end < other_end { + self_greater = false; + eq = false; + } + if self_end > other_end { + other_greater = false; + eq = false; + } + } else if other_end > 0 { + self_greater = false; + eq = false; + } + } + + for (client_id, self_end) in self.iter() { + if other.contains_key(client_id) { + continue; + } else if normalize_vv_counter(*self_end) > 0 { + other_greater = false; + eq = false; + } + } + + if eq { + Some(Ordering::Equal) + } else if self_greater { + Some(Ordering::Greater) + } else if other_greater { + Some(Ordering::Less) + } else { + None + } + } +} + +impl DerefMut for VersionVector { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl VersionVector { + pub fn diff(&self, rhs: &Self) -> VersionVectorDiff { + let mut ans: VersionVectorDiff = Default::default(); + for (client_id, &counter) in self.iter() { + let counter = normalize_vv_counter(counter); + if let Some(&rhs_counter) = rhs.get(client_id) { + let rhs_counter = normalize_vv_counter(rhs_counter); + match counter.cmp(&rhs_counter) { + Ordering::Less => { + ans.forward.insert( + *client_id, + CounterSpan { + start: counter, + end: rhs_counter, + }, + ); + } + Ordering::Greater => { + ans.retreat.insert( + *client_id, + CounterSpan { + start: rhs_counter, + end: counter, + }, + ); + } + Ordering::Equal => {} + } + } else if counter > 0 { + ans.retreat.insert( + *client_id, + CounterSpan { + start: 0, + end: counter, + }, + ); + } + } + for (client_id, &rhs_counter) in rhs.iter() { + let rhs_counter = normalize_vv_counter(rhs_counter); + if rhs_counter > 0 && !self.contains_key(client_id) { + ans.forward.insert( + *client_id, + CounterSpan { + start: 0, + end: rhs_counter, + }, + ); + } + } + + ans + } + + /// Returns two iterators that cover the differences between two version vectors. + /// + /// - The first iterator contains the spans that are in `self` but not in `rhs` + /// - The second iterator contains the spans that are in `rhs` but not in `self` + pub fn diff_iter<'a>( + &'a self, + rhs: &'a Self, + ) -> ( + impl Iterator + 'a, + impl Iterator + 'a, + ) { + (self.sub_iter(rhs), rhs.sub_iter(self)) + } + + /// Returns the spans that are in `self` but not in `rhs` + pub fn sub_iter<'a>(&'a self, rhs: &'a Self) -> impl Iterator + 'a { + self.iter().filter_map(move |(peer, &counter)| { + let counter = normalize_vv_counter(counter); + if let Some(&rhs_counter) = rhs.get(peer) { + let rhs_counter = normalize_vv_counter(rhs_counter); + if counter > rhs_counter { + Some(IdSpan { + peer: *peer, + counter: CounterSpan { + start: rhs_counter, + end: counter, + }, + }) + } else { + None + } + } else if counter > 0 { + Some(IdSpan { + peer: *peer, + counter: CounterSpan { + start: 0, + end: counter, + }, + }) + } else { + None + } + }) + } + + /// Returns the spans that are in `self` but not in `rhs` + pub fn sub_iter_im<'a>( + &'a self, + rhs: &'a ImVersionVector, + ) -> impl Iterator + 'a { + self.iter().filter_map(move |(peer, &counter)| { + let counter = normalize_vv_counter(counter); + if let Some(&rhs_counter) = rhs.get(peer) { + let rhs_counter = normalize_vv_counter(rhs_counter); + if counter > rhs_counter { + Some(IdSpan { + peer: *peer, + counter: CounterSpan { + start: rhs_counter, + end: counter, + }, + }) + } else { + None + } + } else if counter > 0 { + Some(IdSpan { + peer: *peer, + counter: CounterSpan { + start: 0, + end: counter, + }, + }) + } else { + None + } + }) + } + + /// Iter all span from a -> b and b -> a + pub fn iter_between<'a>(&'a self, other: &'a Self) -> impl Iterator + 'a { + // PERF: can be optimized a little + self.sub_iter(other).chain(other.sub_iter(self)) + } + + pub fn sub_vec(&self, rhs: &Self) -> IdSpanVector { + self.sub_iter(rhs).map(|x| (x.peer, x.counter)).collect() + } + + pub fn distance_between(&self, other: &Self) -> usize { + let mut ans = 0usize; + for (client_id, &counter) in self.iter() { + let counter = counter.max(0) as i64; + if let Some(&other_counter) = other.get(client_id) { + let other_counter = other_counter.max(0) as i64; + ans += counter.abs_diff(other_counter) as usize; + } else { + ans += counter as usize; + } + } + + for (client_id, &counter) in other.iter() { + if !self.contains_key(client_id) { + ans += counter.max(0) as usize; + } + } + + ans + } + + pub fn to_spans(&self) -> IdSpanVector { + self.iter() + .filter_map(|(client_id, &counter)| { + let counter = normalize_vv_counter(counter); + (counter > 0).then_some(( + *client_id, + CounterSpan { + start: 0, + end: counter, + }, + )) + }) + .collect() + } + + #[inline] + pub fn get_frontiers(&self) -> Frontiers { + self.iter() + .filter_map(|(client_id, &counter)| { + if counter > 0 { + Some(ID { + peer: *client_id, + counter: counter - 1, + }) + } else { + None + } + }) + .collect() + } + + #[inline] + pub fn new() -> Self { + Self(Default::default()) + } + + /// set the inclusive ending point. target id will be included by self + #[inline] + pub fn set_last(&mut self, id: ID) { + let end = last_id_to_vv_end(id); + if end == 0 { + self.0.remove(&id.peer); + } else { + self.0.insert(id.peer, end); + } + } + + #[inline] + pub fn get_last(&self, client_id: PeerID) -> Option { + self.0.get(&client_id).and_then(|&x| { + let x = normalize_vv_counter(x); + if x == 0 { + None + } else { + Some(x - 1) + } + }) + } + + /// set the exclusive ending point. target id will NOT be included by self + #[inline] + pub fn set_end(&mut self, id: ID) { + if id.counter <= 0 { + self.0.remove(&id.peer); + } else { + self.0.insert(id.peer, id.counter); + } + } + + /// Update the end counter of the given client if the end is greater. + /// Return whether updated + #[inline] + pub fn try_update_last(&mut self, id: ID) -> bool { + let new_end = last_id_to_vv_end(id); + if new_end == 0 { + if self.0.get(&id.peer).is_some_and(|counter| *counter < 0) { + self.0.remove(&id.peer); + } + return false; + } + + if let Some(end) = self.0.get_mut(&id.peer) { + *end = normalize_vv_counter(*end); + if *end < new_end { + *end = new_end; + true + } else { + false + } + } else { + self.0.insert(id.peer, new_end); + true + } + } + + pub fn get_missing_span(&self, target: &Self) -> Vec { + let mut ans = vec![]; + for (client_id, other_end) in target.iter() { + let other_end = normalize_vv_counter(*other_end); + if other_end == 0 { + continue; + } + + let my_end = self + .get(client_id) + .map(|counter| normalize_vv_counter(*counter)) + .unwrap_or(0); + if my_end < other_end { + ans.push(IdSpan::new(*client_id, my_end, other_end)); + } + } + + ans + } + + pub fn merge(&mut self, other: &Self) { + for (&client_id, &other_end) in other.iter() { + let other_end = normalize_vv_counter(other_end); + if other_end == 0 { + continue; + } + + if let Some(my_end) = self.get_mut(&client_id) { + *my_end = normalize_vv_counter(*my_end); + if *my_end < other_end { + *my_end = other_end; + } + } else { + self.0.insert(client_id, other_end); + } + } + } + + pub fn includes_vv(&self, other: &VersionVector) -> bool { + match self.partial_cmp(other) { + Some(ord) => match ord { + Ordering::Less => false, + Ordering::Equal => true, + Ordering::Greater => true, + }, + None => false, + } + } + + pub fn includes_id(&self, id: ID) -> bool { + if id.counter < 0 { + return false; + } + + if let Some(end) = self.get(&id.peer) { + if normalize_vv_counter(*end) > id.counter { + return true; + } + } + false + } + + pub fn intersect_span(&self, target: IdSpan) -> Option { + if let Some(&end) = self.get(&target.peer) { + let end = normalize_vv_counter(end); + let count_start = target.ctr_start().max(0); + let count_end = target.ctr_end().max(0); + if end > count_start && count_end > count_start { + return Some(CounterSpan { + start: count_start, + end: end.min(count_end), + }); + } + } + + None + } + + pub fn extend_to_include_vv<'a>( + &mut self, + vv: impl Iterator, + ) { + for (&client_id, &counter) in vv { + let counter = normalize_vv_counter(counter); + if counter == 0 { + continue; + } + + if let Some(my_counter) = self.get_mut(&client_id) { + *my_counter = normalize_vv_counter(*my_counter); + if *my_counter < counter { + *my_counter = counter; + } + } else { + self.0.insert(client_id, counter); + } + } + } + + pub fn extend_to_include_last_id(&mut self, id: ID) { + let end = last_id_to_vv_end(id); + if end == 0 { + return; + } + + if let Some(counter) = self.get_mut(&id.peer) { + *counter = normalize_vv_counter(*counter); + if *counter < end { + *counter = end; + } + } else { + self.0.insert(id.peer, end); + } + } + + pub fn extend_to_include_end_id(&mut self, id: ID) { + let end = normalize_vv_counter(id.counter); + if end == 0 { + return; + } + + if let Some(counter) = self.get_mut(&id.peer) { + *counter = normalize_vv_counter(*counter); + if *counter < end { + *counter = end; + } + } else { + self.0.insert(id.peer, end); + } + } + + pub fn extend_to_include(&mut self, span: IdSpan) { + let end = normalize_vv_counter(span.counter.norm_end()); + if end == 0 { + if self.0.get(&span.peer).is_some_and(|counter| *counter < 0) { + self.0.remove(&span.peer); + } + return; + } + + if let Some(counter) = self.get_mut(&span.peer) { + *counter = normalize_vv_counter(*counter); + if *counter < end { + *counter = end; + } + } else { + self.insert(span.peer, end); + } + } + + pub fn shrink_to_exclude(&mut self, span: IdSpan) { + let start = normalize_vv_counter(span.counter.min()); + let end = normalize_vv_counter(span.counter.norm_end()); + if end <= start { + return; + } + + if start == 0 { + self.remove(&span.peer); + return; + } + + if let Some(counter) = self.get_mut(&span.peer) { + *counter = normalize_vv_counter(*counter); + if *counter > start { + *counter = start; + } + if *counter == 0 { + self.remove(&span.peer); + } + } + } + + pub fn forward(&mut self, spans: &IdSpanVector) { + for span in spans.iter() { + self.extend_to_include(IdSpan { + peer: *span.0, + counter: *span.1, + }); + } + } + + pub fn retreat(&mut self, spans: &IdSpanVector) { + for span in spans.iter() { + self.shrink_to_exclude(IdSpan { + peer: *span.0, + counter: *span.1, + }); + } + } + + pub fn intersection(&self, other: &VersionVector) -> VersionVector { + let mut ans = VersionVector::new(); + for (client_id, &counter) in self.iter() { + let counter = normalize_vv_counter(counter); + if let Some(&other_counter) = other.get(client_id) { + let other_counter = normalize_vv_counter(other_counter); + if counter < other_counter { + if counter != 0 { + ans.insert(*client_id, counter); + } + } else if other_counter != 0 { + ans.insert(*client_id, other_counter); + } + } + } + ans + } + + #[inline(always)] + pub fn encode(&self) -> Vec { + postcard::to_allocvec(self).unwrap() + } + + #[inline(always)] + pub fn decode(bytes: &[u8]) -> Result { + postcard::from_bytes(bytes).map_err(|_| LoroError::DecodeVersionVectorError) + } + + pub fn to_im_vv(&self) -> ImVersionVector { + ImVersionVector(self.0.iter().map(|(&k, &v)| (k, v)).collect()) + } + + pub fn from_im_vv(im_vv: &ImVersionVector) -> Self { + VersionVector(im_vv.0.iter().map(|(&k, &v)| (k, v)).collect()) + } +} + +/// Use minimal set of ids to represent the frontiers +#[tracing::instrument(skip(dag))] +pub fn shrink_frontiers(last_ids: &Frontiers, dag: &AppDag) -> Result { + // it only keep the ids of ops that are concurrent to each other + + if last_ids.len() <= 1 { + return Ok(last_ids.clone()); + } + + let mut last_ids = { + let ids = filter_duplicated_peer_id(last_ids); + if last_ids.len() == 1 { + let mut frontiers = Frontiers::default(); + frontiers.push(last_ids.as_single().unwrap()); + return Ok(frontiers); + } + + let mut last_ids = Vec::with_capacity(ids.len()); + for id in ids { + let Some(lamport) = dag.get_lamport(&id) else { + return Err(id); + }; + last_ids.push(IdFull::new(id.peer, id.counter, lamport)) + } + + last_ids + }; + + let mut frontiers = Vec::new(); + // Iterate from the greatest lamport to the smallest + last_ids.sort_by_key(|x| x.lamport); + for id in last_ids.iter().rev() { + let mut should_insert = true; + let mut len = 0; + // travel backward because they have more similar lamport + for f_id in frontiers.iter().rev() { + dag.travel_ancestors(*f_id, &mut |x| { + len += 1; + if x.contains_id(id.id()) { + should_insert = false; + ControlFlow::Break(()) + } else if x.lamport_last() < id.lamport { + // Already travel to a node with smaller lamport, no need to continue, we are sure two ops are concurrent now + ControlFlow::Break(()) + } else { + ControlFlow::Continue(()) + } + }); + } + + if should_insert { + frontiers.push(id.id()); + } + } + + Ok(frontiers.into()) +} + +fn filter_duplicated_peer_id(last_ids: &Frontiers) -> Vec { + let mut peer_max_counters = FxHashMap::default(); + for id in last_ids.iter() { + let counter = peer_max_counters.entry(id.peer).or_insert(id.counter); + if id.counter > *counter { + *counter = id.counter; + } + } + + peer_max_counters + .into_iter() + .map(|(peer, counter)| ID::new(peer, counter)) + .collect() +} + +impl Default for VersionVector { + fn default() -> Self { + Self::new() + } +} + +impl From> for VersionVector { + fn from(map: FxHashMap) -> Self { + let mut im_map = FxHashMap::default(); + for (client_id, counter) in map { + im_map.insert(client_id, counter); + } + Self(im_map) + } +} + +impl From> for VersionVector { + fn from(vec: Vec) -> Self { + let mut vv = VersionVector::new(); + for id in vec { + vv.set_last(id); + } + + vv + } +} + +impl FromIterator for VersionVector { + fn from_iter>(iter: T) -> Self { + let iter = iter.into_iter(); + let mut vv = VersionVector(FxHashMap::with_capacity_and_hasher( + iter.size_hint().0, + Default::default(), + )); + for id in iter { + vv.set_last(id); + } + + vv + } +} + +impl FromIterator<(PeerID, Counter)> for VersionVector { + fn from_iter>(iter: T) -> Self { + VersionVector(FxHashMap::from_iter(iter)) + } +} + +// Note: It will be encoded into binary format, so the order of its fields should not be changed. + +pub fn are_frontiers_eq(a: &[ID], b: &[ID]) -> bool { + if a.len() != b.len() { + return false; + } + + let mut a: SmallVec<[ID; 10]> = a.into(); + let mut b: SmallVec<[ID; 10]> = b.into(); + + a.sort(); + b.sort(); + + a == b +} + +#[cfg(test)] +mod tests { + #![allow(clippy::neg_cmp_op_on_partial_ord)] + use super::*; + mod cmp { + use super::*; + #[test] + fn test() { + let a: VersionVector = vec![ID::new(1, 1), ID::new(2, 2)].into(); + let b: VersionVector = vec![ID::new(1, 1), ID::new(2, 2)].into(); + assert_eq!(a.partial_cmp(&b), Some(Ordering::Equal)); + assert!(a == b); + + let a: VersionVector = vec![ID::new(1, 2), ID::new(2, 1)].into(); + let b: VersionVector = vec![ID::new(1, 1), ID::new(2, 2)].into(); + assert_eq!(a.partial_cmp(&b), None); + + assert!(!(a > b)); + assert!(!(b > a)); + assert!(!(b == a)); + + let a: VersionVector = vec![ID::new(1, 2), ID::new(2, 3)].into(); + let b: VersionVector = vec![ID::new(1, 1), ID::new(2, 2)].into(); + assert_eq!(a.partial_cmp(&b), Some(Ordering::Greater)); + assert!(a > b); + assert!(a >= b); + + let a: VersionVector = vec![ID::new(1, 0), ID::new(2, 2)].into(); + let b: VersionVector = vec![ID::new(1, 1), ID::new(2, 2)].into(); + assert_eq!(a.partial_cmp(&b), Some(Ordering::Less)); + assert!(a < b); + assert!(a <= b); + } + } + + #[test] + fn im() { + let mut a = VersionVector::new(); + a.set_last(ID::new(1, 1)); + a.set_last(ID::new(2, 1)); + let mut b = a.clone(); + b.merge(&vec![ID::new(1, 2), ID::new(2, 2)].into()); + assert!(a != b); + assert_eq!(a.get(&1), Some(&2)); + assert_eq!(a.get(&2), Some(&2)); + assert_eq!(b.get(&1), Some(&3)); + assert_eq!(b.get(&2), Some(&3)); + } + + #[test] + fn test_encode_decode_im_version_vector() { + let vv = VersionVector::from_iter([(1, 1), (2, 2), (3, 3)]); + let im_vv = vv.to_im_vv(); + let decoded_vv = VersionVector::from_im_vv(&im_vv); + assert_eq!(vv, decoded_vv); + } + + #[test] + fn test_version_vector_encoding_decoding() { + let mut vv = VersionVector::new(); + vv.insert(1, 10); + vv.insert(2, 20); + vv.insert(3, 30); + + // Encode VersionVector + let encoded = vv.encode(); + + // Decode to ImVersionVector + let decoded_im_vv = ImVersionVector::decode(&encoded).unwrap(); + + // Convert VersionVector to ImVersionVector for comparison + let im_vv = vv.to_im_vv(); + + // Compare the original ImVersionVector with the decoded one + assert_eq!(im_vv, decoded_im_vv); + + // Convert back to VersionVector and compare + let decoded_vv = VersionVector::from_im_vv(&decoded_im_vv); + assert_eq!(vv, decoded_vv); + } +} diff --git a/3rdparty/loro/crates/loro-internal/src/version/frontiers.rs b/3rdparty/loro/crates/loro-internal/src/version/frontiers.rs new file mode 100644 index 0000000000..9c26268fad --- /dev/null +++ b/3rdparty/loro/crates/loro-internal/src/version/frontiers.rs @@ -0,0 +1,574 @@ +use super::*; +use either::Either; + +/// Frontiers representation. +// +// Internal Invariance: +// - Frontiers::Map(map) always have at least 2 elements. +#[derive(Clone, Default)] +pub enum Frontiers { + #[default] + None, + ID(ID), + // We use internal map to avoid the module outside accidentally create or modify the map + // to make it empty or only contain 1 element + Map(InternalMap), +} + +use std::{fmt, sync::Arc}; + +impl fmt::Debug for Frontiers { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("Frontiers") + .field(&FrontiersDebugHelper(self)) + .finish() + } +} + +struct FrontiersDebugHelper<'a>(&'a Frontiers); + +impl fmt::Debug for FrontiersDebugHelper<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut list = f.debug_list(); + match self.0 { + Frontiers::None => {} + Frontiers::ID(id) => { + list.entry(id); + } + Frontiers::Map(map) => { + for id in map.iter() { + list.entry(&id); + } + } + } + list.finish() + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct InternalMap(Arc>); + +impl InternalMap { + fn new() -> Self { + Self(Arc::new(FxHashMap::default())) + } + + fn len(&self) -> usize { + self.0.len() + } + + fn iter(&self) -> impl Iterator + '_ { + self.0 + .iter() + .map(|(&peer, &counter)| ID::new(peer, counter)) + } + + fn contains(&self, id: &ID) -> bool { + self.0 + .get(&id.peer) + .is_some_and(|&counter| counter == id.counter) + } + + fn insert(&mut self, id: ID) { + Arc::make_mut(&mut self.0) + .entry(id.peer) + .and_modify(|e| *e = (*e).max(id.counter)) + .or_insert(id.counter); + } + + fn remove(&mut self, id: &ID) -> bool { + let map = Arc::make_mut(&mut self.0); + if let Some(counter) = map.get_mut(&id.peer) { + if *counter == id.counter { + map.remove(&id.peer); + return true; + } + } + false + } + + fn retain(&mut self, mut f: F) + where + F: FnMut(&ID) -> bool, + { + let map = Arc::make_mut(&mut self.0); + map.retain(|&peer, &mut counter| f(&ID::new(peer, counter))); + } +} + +impl Frontiers { + pub fn len(&self) -> usize { + match self { + Frontiers::None => 0, + Frontiers::ID(_) => 1, + Frontiers::Map(map) => map.len(), + } + } + + pub fn iter(&self) -> impl Iterator + '_ { + match self { + Frontiers::None => Either::Left(Either::Left(std::iter::empty())), + Frontiers::ID(id) => Either::Left(Either::Right(std::iter::once(*id))), + Frontiers::Map(map) => Either::Right(map.iter()), + } + } + + pub fn contains(&self, id: &ID) -> bool { + match self { + Frontiers::None => false, + Frontiers::ID(inner_id) => inner_id == id, + Frontiers::Map(map) => map.contains(id), + } + } + + pub fn push(&mut self, id: ID) { + match self { + Frontiers::None => *self = Frontiers::ID(id), + Frontiers::ID(existing_id) => { + if existing_id.peer != id.peer { + let mut map = InternalMap::new(); + map.insert(*existing_id); + map.insert(id); + *self = Frontiers::Map(map); + } else if existing_id.counter < id.counter { + *existing_id = id; + } + } + Frontiers::Map(map) => map.insert(id), + } + } + + pub fn retain(&mut self, mut f: F) + where + F: FnMut(&ID) -> bool, + { + match self { + Frontiers::None => {} + Frontiers::ID(id) => { + if !f(id) { + *self = Frontiers::None; + } + } + Frontiers::Map(map) => { + map.retain(|id| f(id)); + match map.len() { + 0 => *self = Frontiers::None, + 1 => { + let id = map.iter().next().unwrap(); + *self = Frontiers::ID(id); + } + _ => {} + } + } + } + } + + pub fn remove(&mut self, id: &ID) { + match self { + Frontiers::None => {} + Frontiers::ID(existing_id) => { + if existing_id == id { + *self = Frontiers::None; + } + } + Frontiers::Map(map) => { + if map.remove(id) { + match map.len() { + 0 => *self = Frontiers::None, + 1 => { + let id = map.iter().next().unwrap(); + *self = Frontiers::ID(id); + } + _ => {} + } + } + } + } + } +} + +impl PartialEq for Frontiers { + fn eq(&self, other: &Self) -> bool { + let len = self.len(); + if len != other.len() { + return false; + } + + match (self, other) { + (Frontiers::None, Frontiers::None) => true, + (Frontiers::ID(id1), Frontiers::ID(id2)) => id1 == id2, + (Frontiers::Map(map1), Frontiers::Map(map2)) => map1 == map2, + _ => unreachable!(), + } + } +} + +impl Eq for Frontiers {} + +impl Frontiers { + pub fn new() -> Self { + Self::None + } + + #[inline] + pub fn from_id(id: ID) -> Self { + Self::ID(id) + } + + #[inline] + pub fn encode(&self) -> Vec { + let mut vec: Vec = self.iter().collect(); + vec.sort(); + postcard::to_allocvec(&vec).unwrap() + } + + #[inline] + pub fn decode(bytes: &[u8]) -> Result { + let vec: Vec = postcard::from_bytes(bytes).map_err(|_| { + LoroError::DecodeError("Decode Frontiers error".to_string().into_boxed_str()) + })?; + Ok(Self::from(vec)) + } + + pub fn update_frontiers_on_new_change(&mut self, id: ID, deps: &Frontiers) { + if self.len() <= 8 && self == deps { + *self = Frontiers::from_id(id); + return; + } + + // Remove all IDs in deps from self + for dep in deps.iter() { + self.remove(&dep); + } + + // Add the new ID + self.push(id); + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Returns the single ID if the Frontiers contains exactly one ID, otherwise returns None. + pub fn as_single(&self) -> Option { + match self { + Frontiers::ID(id) => Some(*id), + _ => None, + } + } + + /// Returns a reference to the internal map if the Frontiers contains multiple IDs, + /// otherwise returns None. + pub fn as_map(&self) -> Option<&InternalMap> { + match self { + Frontiers::Map(map) => Some(map), + _ => None, + } + } + + /// Merges another Frontiers into this one. + /// + /// Id from other will override the id with the same peer from self. + pub fn merge_with_greater(&mut self, other: &Frontiers) { + if self.is_empty() { + *self = other.clone(); + return; + } + + if let Some(id) = self.as_single() { + match other { + Frontiers::None => {} + Frontiers::ID(other_id) => { + if id.peer == other_id.peer { + *self = Frontiers::ID(ID::new(id.peer, id.counter.max(other_id.counter))); + } else { + self.push(*other_id); + } + return; + } + Frontiers::Map(internal_map) => { + let mut map = internal_map.clone(); + Arc::make_mut(&mut map.0) + .entry(id.peer) + .and_modify(|c| *c = (*c).max(id.counter)) + .or_insert(id.counter); + *self = Frontiers::Map(map); + } + } + + return; + } + + let Frontiers::Map(map) = self else { + unreachable!() + }; + let map = Arc::make_mut(&mut map.0); + for id in other.iter() { + map.entry(id.peer) + .and_modify(|c| *c = (*c).max(id.counter)) + .or_insert(id.counter); + } + } + + pub fn to_vec(&self) -> Vec { + match self { + Frontiers::None => Vec::new(), + Frontiers::ID(id) => vec![*id], + Frontiers::Map(map) => map.iter().collect(), + } + } + + /// Keeps only one element in the Frontiers, deleting all others. + /// If the Frontiers is empty, it remains empty. + /// If it contains multiple elements, it keeps the first one encountered. + pub fn keep_one(&mut self) { + match self { + Frontiers::None => {} + Frontiers::ID(_) => {} + Frontiers::Map(map) => { + if let Some((&peer, &counter)) = map.0.iter().next() { + *self = Frontiers::ID(ID::new(peer, counter)); + } + } + } + } +} +impl From<&[ID]> for Frontiers { + fn from(ids: &[ID]) -> Self { + match ids.len() { + 0 => Frontiers::None, + 1 => Frontiers::ID(ids[0]), + _ => { + let mut map = InternalMap::new(); + for &id in ids { + map.insert(id); + } + Frontiers::Map(map) + } + } + } +} + +impl From> for Frontiers { + fn from(ids: Vec) -> Self { + match ids.len() { + 0 => Frontiers::None, + 1 => Frontiers::ID(ids[0]), + _ => { + let mut map = InternalMap::new(); + for id in ids { + map.insert(id); + } + Frontiers::Map(map) + } + } + } +} + +impl From for Frontiers { + fn from(value: ID) -> Self { + Self::ID(value) + } +} + +impl FromIterator for Frontiers { + fn from_iter>(iter: I) -> Self { + let mut new = Self::new(); + for id in iter { + new.push(id); + } + new + } +} + +impl From> for Frontiers { + fn from(value: Option) -> Self { + match value { + Some(id) => Frontiers::ID(id), + None => Frontiers::None, + } + } +} + +impl From<[ID; N]> for Frontiers { + fn from(value: [ID; N]) -> Self { + match N { + 0 => Frontiers::None, + 1 => Frontiers::ID(value[0]), + _ => { + let mut map = InternalMap::new(); + for id in value { + map.insert(id); + } + Frontiers::Map(map) + } + } + } +} + +impl From<&Vec> for Frontiers { + fn from(ids: &Vec) -> Self { + match ids.len() { + 0 => Frontiers::None, + 1 => Frontiers::ID(ids[0]), + _ => { + let mut map = InternalMap::new(); + for id in ids { + map.insert(*id); + } + Frontiers::Map(map) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_frontiers_push_insert_remove() { + let mut frontiers = Frontiers::None; + + // Test push + frontiers.push(ID::new(1, 1)); + assert_eq!(frontiers, Frontiers::ID(ID::new(1, 1))); + + frontiers.push(ID::new(2, 1)); + assert!(matches!(frontiers, Frontiers::Map(_))); + assert_eq!(frontiers.len(), 2); + + frontiers.push(ID::new(1, 2)); + assert_eq!(frontiers.len(), 2); + assert!(frontiers.contains(&ID::new(1, 2))); + assert!(!frontiers.contains(&ID::new(1, 1))); + + // Test insert (via InternalMap) + if let Frontiers::Map(ref mut map) = frontiers { + map.insert(ID::new(3, 1)); + } + assert_eq!(frontiers.len(), 3); + assert!(frontiers.contains(&ID::new(3, 1))); + + // Test remove + frontiers.remove(&ID::new(2, 1)); + assert_eq!(frontiers.len(), 2); + assert!(!frontiers.contains(&ID::new(2, 1))); + + frontiers.remove(&ID::new(1, 2)); + assert_eq!(frontiers, Frontiers::ID(ID::new(3, 1))); + + frontiers.remove(&ID::new(3, 1)); + assert_eq!(frontiers, Frontiers::None); + } + + #[test] + fn test_frontiers_edge_cases() { + let mut frontiers = Frontiers::None; + + // Push to empty + frontiers.push(ID::new(1, 1)); + assert_eq!(frontiers, Frontiers::ID(ID::new(1, 1))); + + // Push same peer, higher counter + frontiers.push(ID::new(1, 2)); + assert_eq!(frontiers, Frontiers::ID(ID::new(1, 2))); + + // Push same peer, lower counter (should not change) + frontiers.push(ID::new(1, 1)); + assert_eq!(frontiers, Frontiers::ID(ID::new(1, 2))); + + // Push different peer + frontiers.push(ID::new(2, 1)); + assert!(matches!(frontiers, Frontiers::Map(_))); + assert_eq!(frontiers.len(), 2); + + // Remove non-existent + frontiers.remove(&ID::new(3, 1)); + assert_eq!(frontiers.len(), 2); + + // Remove until only one left + frontiers.remove(&ID::new(2, 1)); + assert_eq!(frontiers, Frontiers::ID(ID::new(1, 2))); + + // Remove last + frontiers.remove(&ID::new(1, 2)); + assert_eq!(frontiers, Frontiers::None); + } + + #[test] + fn test_frontiers_retain() { + let mut frontiers = Frontiers::None; + + // Test retain on empty frontiers + frontiers.retain(|_| true); + assert_eq!(frontiers, Frontiers::None); + + // Test retain on single ID + frontiers.push(ID::new(1, 1)); + frontiers.retain(|id| id.peer == 1); + assert_eq!(frontiers, Frontiers::ID(ID::new(1, 1))); + + frontiers.retain(|id| id.peer == 2); + assert_eq!(frontiers, Frontiers::None); + + // Test retain on multiple IDs + frontiers.push(ID::new(1, 1)); + frontiers.push(ID::new(2, 2)); + frontiers.push(ID::new(3, 3)); + + // Retain only even peer IDs + frontiers.retain(|id| id.peer % 2 == 0); + assert_eq!(frontiers, Frontiers::ID(ID::new(2, 2))); + + // Add more IDs and test retaining multiple + frontiers.push(ID::new(1, 1)); + frontiers.push(ID::new(3, 3)); + frontiers.push(ID::new(4, 4)); + + frontiers.retain(|id| id.peer > 2); + assert!(matches!(frontiers, Frontiers::Map(_))); + assert_eq!(frontiers.len(), 2); + assert!(frontiers.contains(&ID::new(3, 3))); + assert!(frontiers.contains(&ID::new(4, 4))); + + // Retain none + frontiers.retain(|_| false); + assert_eq!(frontiers, Frontiers::None); + } + + #[test] + fn test_frontiers_encode_decode() { + let mut frontiers = Frontiers::None; + + // Test encode/decode for empty frontiers + let encoded = frontiers.encode(); + let decoded = Frontiers::decode(&encoded).unwrap(); + assert_eq!(frontiers, decoded); + + // Test encode/decode for single ID + frontiers.push(ID::new(1, 100)); + let encoded = frontiers.encode(); + let decoded = Frontiers::decode(&encoded).unwrap(); + assert_eq!(frontiers, decoded); + + // Test encode/decode for multiple IDs + frontiers.push(ID::new(2, 200)); + frontiers.push(ID::new(3, 300)); + let encoded = frontiers.encode(); + let decoded = Frontiers::decode(&encoded).unwrap(); + assert_eq!(frontiers, decoded); + + // Test encode/decode for many IDs + for i in 4..20 { + frontiers.push(ID::new(i, i as Counter * 100)); + } + let encoded = frontiers.encode(); + let decoded = Frontiers::decode(&encoded).unwrap(); + assert_eq!(frontiers, decoded); + + // Test decode with invalid input + assert!(Frontiers::decode(&[0xFF]).is_err()); + assert!(Frontiers::decode(&[]).is_err()); + } +} diff --git a/3rdparty/loro/crates/loro/Cargo.toml b/3rdparty/loro/crates/loro/Cargo.toml new file mode 100644 index 0000000000..54cfa0f88e --- /dev/null +++ b/3rdparty/loro/crates/loro/Cargo.toml @@ -0,0 +1,45 @@ +[package] +name = "loro" +version = "1.13.7" +edition = "2021" +license = "MIT" +documentation = "https://docs.rs/loro/" +description = "Loro is a high-performance CRDTs framework. Make your app collaborative efforlessly." +homepage = "https://loro.dev" +repository = "https://github.com/loro-dev/loro/" +authors = ["Zixuan Chen", "Liang Zhao"] +categories = [] +keywords = ["crdt", "local-first"] + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +loro-internal = { path = "../loro-internal", version = "1.13.7" } +loro-common = { path = "../loro-common", version = "1.13.1", features = [ + "serde_json", +] } +loro-kv-store = { path = "../kv-store", version = "1.13.7" } +delta = { path = "../delta", package = "loro-delta", version = "1.13.0" } +generic-btree = { version = "^0.10.7" } +enum-as-inner = { workspace = true } +tracing = { workspace = true } +rustc-hash = { workspace = true } + +[dev-dependencies] +serde_json = { version = "1.0.87", features = ["float_roundtrip"] } +anyhow = "1.0.83" +ctor = "0.2" +rand = "0.8.5" +pretty_assertions = "1.4.0" +loom = "0.7" +base64 = "0.22.1" +serial_test = "3" + +[features] +default = ["counter"] +counter = ["loro-internal/counter"] +jsonpath = ["loro-internal/jsonpath"] +logging = ["loro-internal/logging"] + +[lints.rust] +unexpected_cfgs = { level = "warn", check-cfg = ['cfg(loom)'] } diff --git a/3rdparty/loro/crates/loro/LICENSE b/3rdparty/loro/crates/loro/LICENSE new file mode 100644 index 0000000000..3cf271d30f --- /dev/null +++ b/3rdparty/loro/crates/loro/LICENSE @@ -0,0 +1,21 @@ +MIT License Copyright (c) 2023 Zixuan Chen + +Permission is hereby granted, free of +charge, to any person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the Software without +restriction, including without limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to the +following conditions: + +The above copyright notice and this permission notice +(including the next paragraph) shall be included in all copies or substantial +portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO +EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. \ No newline at end of file diff --git a/3rdparty/loro/crates/loro/README.md b/3rdparty/loro/crates/loro/README.md new file mode 100644 index 0000000000..fe9a21763a --- /dev/null +++ b/3rdparty/loro/crates/loro/README.md @@ -0,0 +1,136 @@ +# Loro + +Loro is a high‑performance CRDT framework for local‑first apps that keeps state consistent across devices and users, works offline and in real time, automatically merges conflicts, and enables undo/redo and time travel. + +[Loro](https://loro.dev) is a high-performance CRDTs library offering Rust, JavaScript and Swift APIs. + +## Common Tasks & Examples + +### Getting Started + +- Create a document: [`LoroDoc::new()`](struct.LoroDoc.html#method.new) - Initialize a new collaborative document +- Add containers: [`get_text`](struct.LoroDoc.html#method.get_text), [`get_list`](struct.LoroDoc.html#method.get_list), [`get_map`](struct.LoroDoc.html#method.get_map), [`get_tree`](struct.LoroDoc.html#method.get_tree), [`get_movable_list`](struct.LoroDoc.html#method.get_movable_list), [`get_counter` (feature "counter")](struct.LoroDoc.html#method.get_counter) +- Listen to changes: [`subscribe`](struct.LoroDoc.html#method.subscribe) or [`subscribe_root`](struct.LoroDoc.html#method.subscribe_root) - React to document/container modifications +- Export/Import state: [`export`](struct.LoroDoc.html#method.export) and [`import`](struct.LoroDoc.html#method.import) - Save and load documents + +### Real-time Collaboration + +- Sync between peers: [`export(ExportMode::updates(&vv))`](struct.LoroDoc.html#method.export) + [`import`](struct.LoroDoc.html#method.import) - Exchange incremental updates (see [`ExportMode::updates`](enum.ExportMode.html#method.updates)) +- Stream updates: [`subscribe_local_update`](struct.LoroDoc.html#method.subscribe_local_update) - Send changes over WebSocket/WebRTC +- Set unique peer ID: [`set_peer_id`](struct.LoroDoc.html#method.set_peer_id) - Ensure each client has a unique identifier +- Handle conflicts: Automatic - All Loro data types are CRDTs that merge concurrent edits + +### Rich Text Editing + +- Create rich text: [`get_text`](struct.LoroDoc.html#method.get_text) - Initialize a collaborative text container +- Edit text: [`LoroText::insert`](struct.LoroText.html#method.insert), [`LoroText::delete`](struct.LoroText.html#method.delete), [`LoroText::apply_delta`](struct.LoroText.html#method.apply_delta) +- Apply formatting: [`LoroText::mark`](struct.LoroText.html#method.mark) - Add bold, italic, links, custom styles +- Track cursor positions: [`LoroText::get_cursor`](struct.LoroText.html#method.get_cursor) + [`LoroDoc::get_cursor_pos`](struct.LoroDoc.html#method.get_cursor_pos) - Stable positions across edits +- Configure styles: [`config_text_style`](struct.LoroDoc.html#method.config_text_style) / [`config_default_text_style`](struct.LoroDoc.html#method.config_default_text_style) - Define expand behavior for marks + +### Data Structures + +- Ordered lists: [`get_list`](struct.LoroDoc.html#method.get_list) - Arrays with [`push`](struct.LoroList.html#method.push), [`insert`](struct.LoroList.html#method.insert), [`delete`](struct.LoroList.html#method.delete) +- Key-value maps: [`get_map`](struct.LoroDoc.html#method.get_map) - Objects with [`insert`](struct.LoroMap.html#method.insert), [`get`](struct.LoroMap.html#method.get), [`delete`](struct.LoroMap.html#method.delete) +- Mergeable map-key children: [`ensure_mergeable_text`](struct.LoroMap.html#method.ensure_mergeable_text), [`ensure_mergeable_map`](struct.LoroMap.html#method.ensure_mergeable_map), and related `ensure_mergeable_*` methods - Lazy child containers that converge when peers create the same map-key child concurrently +- Hierarchical trees: [`get_tree`](struct.LoroDoc.html#method.get_tree) - Trees with [`create`](struct.LoroTree.html#method.create), [`mov`](struct.LoroTree.html#method.mov), [`mov_to`](struct.LoroTree.html#method.mov_to) +- Reorderable lists: [`get_movable_list`](struct.LoroDoc.html#method.get_movable_list) - Drag-and-drop with [`mov`](struct.LoroMovableList.html#method.mov), [`set`](struct.LoroMovableList.html#method.set) +- Counters: [`get_counter` (feature "counter")](struct.LoroDoc.html#method.get_counter) - Distributed counters with [`increment`](struct.LoroCounter.html#method.increment) + +### Ephemeral State & Presence + +- Not currently provided in the Rust crate. Model presence in your app layer alongside CRDT updates (e.g., via your network transport). Cursors can be shared using [`get_cursor`](struct.LoroText.html#method.get_cursor) data if needed. + +### Version Control & History + +- Undo/redo: [`UndoManager`](struct.UndoManager.html) - Local undo of user’s own edits +- Time travel: [`checkout`](struct.LoroDoc.html#method.checkout) to any [`Frontiers`](struct.Frontiers.html) - Debug or review history +- Version tracking: [`oplog_vv`](struct.LoroDoc.html#method.oplog_vv), [`state_frontiers`](struct.LoroDoc.html#method.state_frontiers), [`VersionVector`](struct.VersionVector.html) +- Fork documents: [`fork`](struct.LoroDoc.html#method.fork) or [`fork_at`](struct.LoroDoc.html#method.fork_at) - Create branches for experimentation +- Merge branches: [`import`](struct.LoroDoc.html#method.import) - Combine changes from forked documents + +### Performance & Storage + +- Incremental updates: [`export(ExportMode::updates(&their_vv))`](struct.LoroDoc.html#method.export) - Send only changes (see [`ExportMode::updates`](enum.ExportMode.html#method.updates)) +- Compact history: [`export(ExportMode::Snapshot)`](struct.LoroDoc.html#method.export) - Full state with compressed history (see [`ExportMode::Snapshot`](enum.ExportMode.html#variant.Snapshot)) +- Shallow snapshots: [`export(ExportMode::shallow_snapshot(&frontiers))`](struct.LoroDoc.html#method.export) - State without partial history (see [`ExportMode::shallow_snapshot`](enum.ExportMode.html#method.shallow_snapshot)) + +## Documentation + +- Start with the [Rust API docs for `LoroDoc`](struct.LoroDoc.html) (container management, versioning, import/export, events) + That page hosts examples and details for most important methods you’ll use day-to-day. +- [Loro Website](https://loro.dev) for more details and guides +- [Loro Examples](https://github.com/loro-dev/loro-examples) for more examples and guides + +## Getting Started + +Add to your `Cargo.toml`: + +```toml +[dependencies] +loro = "^1" +``` + +### LoroDoc quick tour + +- Containers: [`get_text`](struct.LoroDoc.html#method.get_text), [`get_map`](struct.LoroDoc.html#method.get_map), [`get_list`](struct.LoroDoc.html#method.get_list), [`get_movable_list`](struct.LoroDoc.html#method.get_movable_list), [`get_tree`](struct.LoroDoc.html#method.get_tree) +- Import/Export: [`export(ExportMode::…)`](struct.LoroDoc.html#method.export), [`import`](struct.LoroDoc.html#method.import), [`from_snapshot`](struct.LoroDoc.html#method.from_snapshot) +- Versioning: [`oplog_vv`](struct.LoroDoc.html#method.oplog_vv), [`state_frontiers`](struct.LoroDoc.html#method.state_frontiers), [`checkout`/`checkout_to_latest`](struct.LoroDoc.html#method.checkout), [`revert_to`](struct.LoroDoc.html#method.revert_to), [`fork`](struct.LoroDoc.html#method.fork) +- Events: [`subscribe`](struct.LoroDoc.html#method.subscribe), [`subscribe_root`](struct.LoroDoc.html#method.subscribe_root), [`subscribe_local_update`](struct.LoroDoc.html#method.subscribe_local_update) (send deltas to peers) +- Paths/JSON: [`get_path_to_container`](struct.LoroDoc.html#method.get_path_to_container), [`get_deep_value`](struct.LoroDoc.html#method.get_deep_value) / [`ToJson`](trait.ToJson.html) (`to_json_value()`), optional [`jsonpath` (feature)](struct.LoroDoc.html#method.jsonpath) + +Optional cargo features: + +```toml +[dependencies] +loro = { version = "^1", features = ["jsonpath"] } +``` + +## Quick Examples + +1) Local edits, change events, and two-peer sync + +```rust +use loro::{LoroDoc, ExportMode}; +use std::sync::Arc; + +let a = LoroDoc::new(); +let b = LoroDoc::new(); + +// Listen for container diffs on `a` +let _changes = a.subscribe_root(Arc::new(|e| { + println!("changed containers: {}", e.events.len()); +})); + +a.get_text("text").insert(0, "Hello, Loro!").unwrap(); +a.commit(); // events fire on commit/export/import/checkout + +// Sync via export/import (send `updates` via your transport) +let updates = a.export(ExportMode::all_updates()).unwrap(); +b.import(&updates).unwrap(); + +assert_eq!(a.get_deep_value(), b.get_deep_value()); +``` + +2) Time travel and revert + +```rust +use loro::LoroDoc; + +let doc = LoroDoc::new(); +let text = doc.get_text("text"); +text.insert(0, "Hello").unwrap(); +let v0 = doc.state_frontiers(); + +text.insert(5, ", world").unwrap(); +assert_eq!(text.to_string(), "Hello, world"); + +// Time travel to v0 (read-only) +doc.checkout(&v0).unwrap(); +assert_eq!(text.to_string(), "Hello"); + +// Return to latest and revert +doc.checkout_to_latest(); +doc.revert_to(&v0).unwrap(); +assert_eq!(text.to_string(), "Hello"); +``` diff --git a/3rdparty/loro/crates/loro/src/counter.rs b/3rdparty/loro/crates/loro/src/counter.rs new file mode 100644 index 0000000000..85bc6e2190 --- /dev/null +++ b/3rdparty/loro/crates/loro/src/counter.rs @@ -0,0 +1,87 @@ +use loro_internal::{ + container::ContainerID, handler::counter::CounterHandler, HandlerTrait, LoroResult, +}; + +use crate::{Container, ContainerTrait, LoroDoc, SealedTrait}; + +/// A counter that can be incremented or decremented. +#[derive(Debug, Clone)] +pub struct LoroCounter { + pub(crate) handler: CounterHandler, +} + +impl Default for LoroCounter { + fn default() -> Self { + Self::new() + } +} + +impl LoroCounter { + /// Create a new Counter. + pub fn new() -> Self { + Self { + handler: CounterHandler::new_detached(), + } + } + + /// Increment the counter by the given value. + pub fn increment(&self, value: f64) -> LoroResult<()> { + self.handler.increment(value) + } + + /// Decrement the counter by the given value. + pub fn decrement(&self, value: f64) -> LoroResult<()> { + self.handler.decrement(value) + } + + /// Get the current value of the counter. + pub fn get_value(&self) -> f64 { + self.handler.get_value().into_double().unwrap() + } + + /// Get the current value of the counter + pub fn get(&self) -> f64 { + self.handler.get_value().into_double().unwrap() + } +} + +impl SealedTrait for LoroCounter {} +impl ContainerTrait for LoroCounter { + type Handler = CounterHandler; + + fn id(&self) -> ContainerID { + self.handler.id() + } + + fn to_container(&self) -> Container { + Container::Counter(self.clone()) + } + + fn to_handler(&self) -> Self::Handler { + self.handler.clone() + } + + fn from_handler(handler: Self::Handler) -> Self { + Self { handler } + } + + fn is_attached(&self) -> bool { + self.handler.is_attached() + } + + fn get_attached(&self) -> Option { + self.handler.get_attached().map(Self::from_handler) + } + + fn try_from_container(container: Container) -> Option { + container.into_counter().ok() + } + + fn is_deleted(&self) -> bool { + self.handler.is_deleted() + } + + fn doc(&self) -> Option { + self.handler.doc().map(LoroDoc::_new) + } +} diff --git a/3rdparty/loro/crates/loro/src/event.rs b/3rdparty/loro/crates/loro/src/event.rs new file mode 100644 index 0000000000..4273b27703 --- /dev/null +++ b/3rdparty/loro/crates/loro/src/event.rs @@ -0,0 +1,431 @@ +//! Loro event handling. +use delta::array_vec::ArrayVec; +use delta::DeltaRope; +use enum_as_inner::EnumAsInner; +use loro_common::IdLp; +use loro_internal::container::ContainerID; +pub use loro_internal::delta::TreeDiff; +use loro_internal::delta::{ResolvedMapDelta, ResolvedMapValue}; +use loro_internal::event::{EventTriggerKind, ListDeltaMeta}; +use loro_internal::handler::{TextDelta, ValueOrHandler}; +use loro_internal::undo::DiffBatch as InnerDiffBatch; +use loro_internal::{ + event::{Diff as DiffInner, Index}, + ContainerDiff as ContainerDiffInner, DiffEvent as DiffEventInner, +}; +use loro_internal::{FxHashMap, ListDiffInsertItem}; +use std::borrow::Cow; +use std::ops::Deref; +use std::sync::Arc; + +use crate::ValueOrContainer; +use crate::{LoroError, LoroResult}; + +/// A subscriber to the event. +#[allow(clippy::unused_unit)] +pub type Subscriber = Arc Fn(DiffEvent<'a>) + Send + Sync>; + +/// An event that is triggered by a change in the state of a [super::LoroDoc]. +#[derive(Debug)] +pub struct DiffEvent<'a> { + /// How the event is triggered. + pub triggered_by: EventTriggerKind, + /// The origin of the event. + pub origin: &'a str, + /// The current receiver of the event. + pub current_target: Option, + /// The diffs of the event. + pub events: Vec>, +} + +/// A diff of a container. +#[derive(Debug)] +pub struct ContainerDiff<'a> { + /// The target container id of the diff. + pub target: &'a ContainerID, + /// The path of the diff. + pub path: &'a [(ContainerID, Index)], + /// Whether the diff is from unknown container. + pub is_unknown: bool, + /// The diff + pub diff: Diff<'a>, +} + +/// A concrete diff. +#[derive(Debug, EnumAsInner, Clone)] +pub enum Diff<'a> { + /// A list diff. + List(Vec), + /// A text diff. + Text(Vec), + /// A map diff. + Map(MapDelta<'a>), + /// A tree diff. + Tree(Cow<'a, TreeDiff>), + #[cfg(feature = "counter")] + /// A counter diff. + Counter(f64), + /// An unknown diff. + Unknown, +} + +/// A list diff item. +/// +/// We use a `Vec` to represent a list diff. +/// +/// Each item can be either an insert, delete, or retain. +/// +/// ## Example +/// +/// `[Retain(3), Delete(1), Insert{insert: [Value(1), Value(2)], is_move: false}]` +/// +/// It means that the list has 3 elements that are not changed, 1 element is deleted, and 2 elements are inserted. +/// +/// If the original list is [1, 2, 3, 4, 5], the list after the diff is [1, 2, 3, 1, 2, 5]. +#[derive(Debug, Clone)] +pub enum ListDiffItem { + /// Insert a new element into the list. + Insert { + /// The new elements to insert. + insert: Vec, + /// Whether the new elements are created by moving + is_move: bool, + }, + /// Delete n elements from the list at the current index. + Delete { + /// The number of elements to delete. + delete: usize, + }, + /// Retain n elements in the list. + /// + /// This is used to keep the current index unchanged. + Retain { + /// The number of elements to retain. + retain: usize, + }, +} + +/// A map delta. +#[derive(Debug, Clone)] +pub struct MapDelta<'a> { + /// All the updated keys and their new values. + pub updated: FxHashMap, Option>, +} + +impl<'a> From> for DiffEvent<'a> { + fn from(value: DiffEventInner<'a>) -> Self { + DiffEvent { + triggered_by: value.event_meta.by, + origin: &value.event_meta.origin, + current_target: value.current_target, + events: value.events.iter().map(|&diff| diff.into()).collect(), + } + } +} + +impl<'a> From<&'a ContainerDiffInner> for ContainerDiff<'a> { + fn from(value: &'a ContainerDiffInner) -> Self { + ContainerDiff { + target: &value.id, + path: &value.path, + is_unknown: value.is_unknown, + diff: (&value.diff).into(), + } + } +} + +impl<'a> From<&'a DiffInner> for Diff<'a> { + fn from(value: &'a DiffInner) -> Self { + match value { + DiffInner::List(l) => { + let mut ans = Vec::new(); + for item in l.iter() { + match item { + delta::DeltaItem::Retain { len, .. } => { + ans.push(ListDiffItem::Retain { retain: *len }); + } + delta::DeltaItem::Replace { + value, + delete, + attr, + } => { + if !value.is_empty() { + ans.push(ListDiffItem::Insert { + insert: value + .iter() + .map(|v| ValueOrContainer::from(v.clone())) + .collect(), + is_move: attr.from_move, + }); + } + if *delete > 0 { + ans.push(ListDiffItem::Delete { delete: *delete }); + } + } + } + } + + Diff::List(ans) + } + DiffInner::Map(m) => Diff::Map(MapDelta { + updated: m + .updated + .iter() + .map(|(k, v)| (Cow::Borrowed(k.as_str()), v.value.clone().map(|v| v.into()))) + .collect(), + }), + DiffInner::Text(t) => { + let text = TextDelta::from_text_diff(t.iter()); + Diff::Text(text) + } + DiffInner::Tree(t) => Diff::Tree(Cow::Borrowed(t)), + #[cfg(feature = "counter")] + DiffInner::Counter(c) => Diff::Counter(*c), + DiffInner::Unknown => Diff::Unknown, + _ => todo!(), + } + } +} + +impl From for Diff<'static> { + fn from(value: DiffInner) -> Self { + match value { + DiffInner::List(l) => { + let mut ans = Vec::new(); + for item in l.iter() { + match item { + delta::DeltaItem::Retain { len, .. } => { + ans.push(ListDiffItem::Retain { retain: *len }); + } + delta::DeltaItem::Replace { + value, + delete, + attr, + } => { + if !value.is_empty() { + ans.push(ListDiffItem::Insert { + insert: value + .iter() + .map(|v| ValueOrContainer::from(v.clone())) + .collect(), + is_move: attr.from_move, + }); + } + if *delete > 0 { + ans.push(ListDiffItem::Delete { delete: *delete }); + } + } + } + } + + Diff::List(ans) + } + DiffInner::Map(m) => Diff::Map(MapDelta { + updated: m + .updated + .iter() + .map(|(k, v)| (Cow::Owned(k.to_string()), v.value.clone().map(|v| v.into()))) + .collect(), + }), + DiffInner::Text(t) => { + let text = TextDelta::from_text_diff(t.iter()); + Diff::Text(text) + } + DiffInner::Tree(t) => Diff::Tree(Cow::Owned(t.clone())), + #[cfg(feature = "counter")] + DiffInner::Counter(c) => Diff::Counter(c), + DiffInner::Unknown => Diff::Unknown, + _ => todo!(), + } + } +} + +impl From for ValueOrContainer { + fn from(value: ValueOrHandler) -> Self { + match value { + ValueOrHandler::Value(v) => ValueOrContainer::Value(v), + ValueOrHandler::Handler(h) => ValueOrContainer::Container(h.into()), + } + } +} + +/// A batch of diffs. +#[derive(Default, Clone)] +pub struct DiffBatch { + cid_to_events: FxHashMap>, + order: Vec, +} + +impl std::fmt::Debug for DiffBatch { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let v: Vec<(&ContainerID, &Diff<'static>)> = self.iter().collect(); + write!(f, "{:#?}", v) + } +} + +impl DiffBatch { + /// Returns an iterator over the diffs in this batch, in the order they were added. + /// + /// The iterator yields tuples of `(&ContainerID, &Diff)` where: + /// - `ContainerID` is the ID of the container that was modified + /// - `Diff` contains the actual changes made to that container + /// + /// The order of the diffs is preserved from when they were originally added to the batch. + pub fn iter(&self) -> impl Iterator)> { + self.order.iter().map(|id| (id, &self.cid_to_events[id])) + } + + /// Push a new event to the batch. + /// + /// If the cid already exists in the batch, return Err + pub fn push(&mut self, cid: ContainerID, diff: Diff<'static>) -> Result<(), Diff<'static>> { + match self.cid_to_events.entry(cid.clone()) { + std::collections::hash_map::Entry::Occupied(_) => Err(diff), + std::collections::hash_map::Entry::Vacant(vacant_entry) => { + vacant_entry.insert(diff); + self.order.push(cid); + Ok(()) + } + } + } + + pub(crate) fn validate_for_apply(&self) -> LoroResult<()> { + for (_, diff) in self.iter() { + validate_diff_for_apply(diff)?; + } + + Ok(()) + } +} + +fn validate_diff_for_apply(diff: &Diff<'_>) -> LoroResult<()> { + let mut index = 0usize; + match diff { + Diff::Text(delta) => { + for item in delta { + match item { + TextDelta::Retain { retain, .. } => { + index = checked_apply_diff_index_end(index, *retain)?; + } + TextDelta::Delete { delete } => { + index = checked_apply_diff_index_end(index, *delete)?; + } + TextDelta::Insert { .. } => {} + } + } + } + Diff::List(delta) => { + for item in delta { + match item { + ListDiffItem::Retain { retain } => { + index = checked_apply_diff_index_end(index, *retain)?; + } + ListDiffItem::Delete { delete } => { + index = checked_apply_diff_index_end(index, *delete)?; + } + ListDiffItem::Insert { .. } => {} + } + } + } + Diff::Map(_) | Diff::Tree(_) | Diff::Unknown => {} + #[cfg(feature = "counter")] + Diff::Counter(_) => {} + } + + Ok(()) +} + +fn checked_apply_diff_index_end(pos: usize, len: usize) -> LoroResult { + pos.checked_add(len).ok_or_else(|| LoroError::OutOfBound { + pos: usize::MAX, + len: usize::MAX, + info: "apply_diff".into(), + }) +} + +impl From for DiffBatch { + fn from(value: InnerDiffBatch) -> Self { + let mut map = + FxHashMap::with_capacity_and_hasher(value.cid_to_events.len(), Default::default()); + for (id, diff) in value.cid_to_events.into_iter() { + map.insert(id.clone(), diff.into()); + } + + DiffBatch { + cid_to_events: map, + order: value.order, + } + } +} + +impl From> for DiffInner { + fn from(value: Diff<'static>) -> Self { + match value { + Diff::List(vec) => { + let mut ans: DeltaRope = DeltaRope::new(); + for item in vec.iter() { + match item { + ListDiffItem::Insert { insert, is_move } => { + for item in ArrayVec::from_many( + insert.iter().map(|v| v.clone().into_value_or_handler()), + ) { + ans.push_insert( + item, + ListDeltaMeta { + from_move: *is_move, + }, + ); + } + } + ListDiffItem::Delete { delete } => { + ans.push_delete(*delete); + } + ListDiffItem::Retain { retain } => { + ans.push_retain(*retain, ListDeltaMeta { from_move: false }); + } + } + } + + DiffInner::List(ans) + } + Diff::Text(t) => { + let text = TextDelta::into_text_diff(t.into_iter()); + DiffInner::Text(text) + } + Diff::Map(map_delta) => DiffInner::Map(ResolvedMapDelta { + updated: map_delta + .updated + .into_iter() + .map(|(k, v)| { + ( + k.deref().into(), + ResolvedMapValue { + value: v.map(|v| v.into_value_or_handler()), + idlp: IdLp::new(0, 0), + }, + ) + }) + .collect(), + }), + Diff::Tree(cow) => DiffInner::Tree(cow.into_owned()), + #[cfg(feature = "counter")] + Diff::Counter(c) => DiffInner::Counter(c), + Diff::Unknown => DiffInner::Unknown, + } + } +} + +impl From for InnerDiffBatch { + fn from(value: DiffBatch) -> Self { + let mut map = + FxHashMap::with_capacity_and_hasher(value.cid_to_events.len(), Default::default()); + for (id, diff) in value.cid_to_events.into_iter() { + map.insert(id.clone(), diff.into()); + } + + InnerDiffBatch { + cid_to_events: map, + order: value.order, + } + } +} diff --git a/3rdparty/loro/crates/loro/src/lib.rs b/3rdparty/loro/crates/loro/src/lib.rs new file mode 100644 index 0000000000..62f8051a02 --- /dev/null +++ b/3rdparty/loro/crates/loro/src/lib.rs @@ -0,0 +1,3983 @@ +#![doc = include_str!("../README.md")] +#![allow(clippy::uninlined_format_args)] +#![warn(missing_docs)] +#![warn(missing_debug_implementations)] +use event::DiffBatch; +use event::{DiffEvent, Subscriber}; +pub use loro_common::InternalString; +pub use loro_internal::cursor::CannotFindRelativePosition; +use loro_internal::cursor::Cursor; +use loro_internal::cursor::PosQueryResult; +use loro_internal::cursor::Side; +pub use loro_internal::encoding::ImportStatus; +use loro_internal::handler::{HandlerTrait, ValueOrHandler}; +pub use loro_internal::loro::ChangeTravelError; +pub use loro_internal::pre_commit::{ + ChangeModifier, FirstCommitFromPeerCallback, FirstCommitFromPeerPayload, PreCommitCallback, + PreCommitCallbackPayload, +}; +pub use loro_internal::sync; +pub use loro_internal::undo::{OnPop, UndoItemMeta, UndoOrRedo}; +use loro_internal::version::shrink_frontiers; +pub use loro_internal::version::ImVersionVector; +use loro_internal::DocState; +use loro_internal::LoroDoc as InnerLoroDoc; +use loro_internal::OpLog; +use loro_internal::{ + handler::Handler as InnerHandler, ListHandler as InnerListHandler, + MapHandler as InnerMapHandler, MovableListHandler as InnerMovableListHandler, + TextHandler as InnerTextHandler, TreeHandler as InnerTreeHandler, + UnknownHandler as InnerUnknownHandler, +}; +use rustc_hash::FxHashSet; +use std::cmp::Ordering; +use std::ops::ControlFlow; +use std::ops::Deref; +use std::ops::Range; +use std::sync::Arc; +use tracing::info; + +pub use loro_internal::diff::diff_impl::UpdateOptions; +pub use loro_internal::diff::diff_impl::UpdateTimeoutError; +pub use loro_internal::subscription::LocalUpdateCallback; +pub use loro_internal::subscription::PeerIdUpdateCallback; +pub use loro_internal::ChangeMeta; +pub use loro_internal::LORO_VERSION; +pub mod event; +pub use loro_internal::awareness; +pub use loro_internal::change::Timestamp; +pub use loro_internal::configure::Configure; +pub use loro_internal::configure::{StyleConfig, StyleConfigMap}; +pub use loro_internal::container::richtext::ExpandType; +pub use loro_internal::container::{ContainerID, ContainerType, IntoContainerId}; +pub use loro_internal::cursor; +pub use loro_internal::delta::{TreeDeltaItem, TreeDiff, TreeDiffItem, TreeExternalDiff}; +pub use loro_internal::encoding::ImportBlobMetadata; +pub use loro_internal::encoding::{EncodedBlobMode, ExportMode}; +pub use loro_internal::event::{EventTriggerKind, Index}; +pub use loro_internal::handler::TextDelta; +pub use loro_internal::json; +pub use loro_internal::json::{ + FutureOp as JsonFutureOp, FutureOpWrapper as JsonFutureOpWrapper, JsonChange, JsonOp, + JsonOpContent, JsonSchema, ListOp as JsonListOp, MapOp as JsonMapOp, + MovableListOp as JsonMovableListOp, TextOp as JsonTextOp, TreeOp as JsonTreeOp, +}; +pub use loro_internal::kv_store::{KvStore, MemKvStore}; +pub use loro_internal::loro::CommitOptions; +pub use loro_internal::loro::DocAnalysis; +pub use loro_internal::oplog::FrontiersNotIncluded; +pub use loro_internal::undo; +pub use loro_internal::version::{Frontiers, VersionRange, VersionVector, VersionVectorDiff}; +pub use loro_internal::ApplyDiff; +pub use loro_internal::Subscription; +pub use loro_internal::UndoManager as InnerUndoManager; +pub use loro_internal::{loro_value, to_value}; +pub use loro_internal::{ + Counter, CounterSpan, FractionalIndex, IdLp, IdSpan, Lamport, PeerID, TreeID, TreeParentId, ID, +}; +pub use loro_internal::{ + LoroBinaryValue, LoroEncodeError, LoroError, LoroListValue, LoroMapValue, LoroResult, + LoroStringValue, LoroTreeError, LoroValue, ToJson, +}; +pub use loro_kv_store as kv_store; + +#[cfg(feature = "jsonpath")] +pub use loro_internal::jsonpath; +#[cfg(feature = "jsonpath")] +pub use loro_internal::jsonpath::SubscribeJsonPathCallback; + +#[cfg(feature = "counter")] +mod counter; +#[cfg(feature = "counter")] +pub use counter::LoroCounter; + +/// `LoroDoc` is the entry for the whole document. +/// When it's dropped, all the associated [`Container`]s will be invalidated. +/// +/// **Important:** Loro is a pure library and does not handle network protocols. +/// It is the responsibility of the user to manage the storage, loading, and synchronization +/// of the bytes exported by Loro in a manner suitable for their specific environment. +#[derive(Debug)] +pub struct LoroDoc { + doc: InnerLoroDoc, + // This field is here to prevent some weird issues in debug mode + #[cfg(debug_assertions)] + _temp: u8, +} + +impl Default for LoroDoc { + fn default() -> Self { + Self::new() + } +} + +impl Clone for LoroDoc { + /// This creates a reference clone, not a deep clone. The cloned doc will share the same + /// underlying doc as the original one. + /// + /// For deep clone, please use the `.fork()` method. + fn clone(&self) -> Self { + let doc = self.doc.clone(); + LoroDoc::_new(doc) + } +} + +impl LoroDoc { + #[inline(always)] + fn _new(doc: InnerLoroDoc) -> Self { + Self { + doc, + #[cfg(debug_assertions)] + _temp: 0, + } + } + + /// Create a new `LoroDoc` instance. + #[inline] + pub fn new() -> Self { + let doc = InnerLoroDoc::default(); + doc.start_auto_commit(); + + LoroDoc::_new(doc) + } + + /// Duplicate the document with a different PeerID + /// + /// The time complexity and space complexity of this operation are both O(n), + /// + /// When called in detached mode, it will fork at the current state frontiers. + /// It will have the same effect as `fork_at(&self.state_frontiers())`. + #[inline] + pub fn fork(&self) -> Self { + let doc = self.doc.fork(); + LoroDoc::_new(doc) + } + + /// Fork the document at the given frontiers. + /// + /// The created doc will only contain the history before the specified frontiers. + pub fn fork_at(&self, frontiers: &Frontiers) -> LoroResult { + let new_doc = self.doc.fork_at(frontiers)?; + new_doc.start_auto_commit(); + Ok(LoroDoc::_new(new_doc)) + } + + /// Get the configurations of the document. + #[inline] + pub fn config(&self) -> &Configure { + self.doc.config() + } + + /// Get `Change` at the given id. + /// + /// `Change` is a grouped continuous operations that share the same id, timestamp, commit message. + /// + /// - The id of the `Change` is the id of its first op. + /// - The second op's id is `{ peer: change.id.peer, counter: change.id.counter + 1 }` + /// + /// The same applies on `Lamport`: + /// + /// - The lamport of the `Change` is the lamport of its first op. + /// - The second op's lamport is `change.lamport + 1` + /// + /// The length of the `Change` is how many operations it contains + pub fn get_change(&self, id: ID) -> Option { + let change = self.doc.oplog().lock().get_change_at(id)?; + Some(ChangeMeta::from_change(&change)) + } + + /// Decodes the metadata for an imported blob from the provided bytes. + /// + /// # Example + /// ``` + /// use loro::{LoroDoc, ExportMode}; + /// + /// let doc = LoroDoc::new(); + /// doc.get_text("t").insert(0, "Hello").unwrap(); + /// let updates = doc.export(ExportMode::all_updates()).unwrap(); + /// let meta = LoroDoc::decode_import_blob_meta(&updates, true).unwrap(); + /// assert!(meta.change_num >= 1); + /// ``` + #[inline] + pub fn decode_import_blob_meta( + bytes: &[u8], + check_checksum: bool, + ) -> LoroResult { + InnerLoroDoc::decode_import_blob_meta(bytes, check_checksum) + } + + /// Set whether to record the timestamp of each change. Default is `false`. + /// + /// If enabled, the Unix timestamp will be recorded for each change automatically. + /// You can also set a timestamp explicitly via [`set_next_commit_timestamp`]. + /// + /// Important: this is a runtime configuration. It is not serialized into updates or + /// snapshots. You must reapply it for each new `LoroDoc` you create or load. + /// + /// NOTE: Timestamps are forced to be in ascending order. If you commit a new change with + /// a timestamp earlier than the latest, the largest existing timestamp will be used instead. + /// + /// # Example + /// ``` + /// use loro::LoroDoc; + /// let doc = LoroDoc::new(); + /// doc.set_record_timestamp(true); + /// doc.get_text("t").insert(0, "hi").unwrap(); + /// doc.commit(); + /// ``` + #[inline] + pub fn set_record_timestamp(&self, record: bool) { + self.doc.set_record_timestamp(record); + } + + /// Enables editing in detached mode, which is disabled by default. + /// + /// The doc enter detached mode after calling `detach` or checking out a non-latest version. + /// + /// # Important Notes: + /// + /// - This mode uses a different PeerID for each checkout. + /// - Ensure no concurrent operations share the same PeerID if set manually. + /// - Importing does not affect the document's state or version; changes are + /// recorded in the [OpLog] only. Call `checkout` to apply changes. + /// + /// # Example + /// ``` + /// use loro::LoroDoc; + /// + /// let doc = LoroDoc::new(); + /// let v0 = doc.state_frontiers(); + /// // Make some edits… + /// doc.get_text("t").insert(0, "Hello").unwrap(); + /// doc.commit(); + /// + /// // Travel back and enable detached editing + /// doc.checkout(&v0).unwrap(); + /// assert!(doc.is_detached()); + /// doc.set_detached_editing(true); + /// doc.get_text("t").insert(0, "old").unwrap(); + /// // Later, re-attach to see latest again + /// doc.attach(); + /// ``` + #[inline] + pub fn set_detached_editing(&self, enable: bool) { + self.doc.set_detached_editing(enable); + } + + /// Whether editing the doc in detached mode is allowed, which is disabled by + /// default. + /// + /// The doc enter detached mode after calling `detach` or checking out a non-latest version. + /// + /// # Important Notes: + /// + /// - This mode uses a different PeerID for each checkout. + /// - Ensure no concurrent operations share the same PeerID if set manually. + /// - Importing does not affect the document's state or version; changes are + /// recorded in the [OpLog] only. Call `checkout` to apply changes. + #[inline] + pub fn is_detached_editing_enabled(&self) -> bool { + self.doc.is_detached_editing_enabled() + } + + /// Set the interval of mergeable changes, **in seconds**. + /// + /// If two continuous local changes are within the interval, they will be merged into one change. + /// The default value is 1000 seconds. + /// + /// By default, we record timestamps in seconds for each change. So if the merge interval is 1, and changes A and B + /// have timestamps of 3 and 4 respectively, then they will be merged into one change. + #[inline] + pub fn set_change_merge_interval(&self, interval: i64) { + self.doc.set_change_merge_interval(interval); + } + + /// Set the rich text format configuration of the document. + /// + /// Configure the `expand` behavior for marks used by [`LoroText::mark`]/[`LoroText::unmark`]. + /// This controls how marks grow when text is inserted at their boundaries. + /// + /// - `after` (default): inserts just after the range expand the mark + /// - `before`: inserts just before the range expand the mark + /// - `both`: inserts on either side expand the mark + /// - `none`: do not expand at boundaries + /// + /// # Example + /// ``` + /// use loro::{LoroDoc, StyleConfigMap, StyleConfig, ExpandType}; + /// let doc = LoroDoc::new(); + /// let mut styles = StyleConfigMap::new(); + /// styles.insert("bold".into(), StyleConfig { expand: ExpandType::After }); + /// doc.config_text_style(styles); + /// ``` + #[inline] + pub fn config_text_style(&self, text_style: StyleConfigMap) { + self.doc.config_text_style(text_style) + } + + /// Configures the default text style for the document. + /// + /// This method sets the default text style configuration for the document when using LoroText. + /// If `None` is provided, the default style is reset. + /// + /// # Parameters + /// + /// - `text_style`: The style configuration to set as the default. `None` to reset. + /// + /// # Example + /// ``` + /// use loro::{LoroDoc, StyleConfig, ExpandType}; + /// let doc = LoroDoc::new(); + /// doc.config_default_text_style(Some(StyleConfig { expand: ExpandType::After })); + /// ``` + pub fn config_default_text_style(&self, text_style: Option) { + self.doc.config_default_text_style(text_style); + } + + /// Attach the document state to the latest known version. + /// + /// > The document becomes detached during a `checkout` operation. + /// > Being `detached` implies that the `DocState` is not synchronized with the latest version of the `OpLog`. + /// > In a detached state, the document is not editable, and any `import` operations will be + /// > recorded in the `OpLog` without being applied to the `DocState`. + #[inline] + pub fn attach(&self) { + self.doc.attach() + } + + /// Checkout the `DocState` to a specific version. + /// + /// The document becomes detached during a `checkout` operation. + /// Being `detached` implies that the `DocState` is not synchronized with the latest version of the `OpLog`. + /// In a detached state, the document is not editable, and any `import` operations will be + /// recorded in the `OpLog` without being applied to the `DocState`. + /// + /// You should call `attach` (or `checkout_to_latest`) to reattach the `DocState` to the latest version of `OpLog`. + /// If you need to edit while detached, enable [`set_detached_editing(true)`], but note it uses a different + /// PeerID per checkout. + #[inline] + pub fn checkout(&self, frontiers: &Frontiers) -> LoroResult<()> { + self.doc.checkout(frontiers) + } + + /// Checkout the `DocState` to the latest version. + /// + /// > The document becomes detached during a `checkout` operation. + /// > Being `detached` implies that the `DocState` is not synchronized with the latest version of the `OpLog`. + /// > In a detached state, the document is not editable, and any `import` operations will be + /// > recorded in the `OpLog` without being applied to the `DocState`. + /// + /// This has the same effect as `attach`. + #[inline] + pub fn checkout_to_latest(&self) { + self.doc.checkout_to_latest() + } + + /// Compare the frontiers with the current OpLog's version. + /// + /// If `other` contains any version that's not contained in the current OpLog, return [Ordering::Less]. + #[inline] + pub fn cmp_with_frontiers(&self, other: &Frontiers) -> Ordering { + self.doc.cmp_with_frontiers(other) + } + + /// Compare two frontiers. + /// + /// If the frontiers are not included in the document, return [`FrontiersNotIncluded`]. + #[inline] + pub fn cmp_frontiers( + &self, + a: &Frontiers, + b: &Frontiers, + ) -> Result, FrontiersNotIncluded> { + self.doc.cmp_frontiers(a, b) + } + + /// Force the document enter the detached mode. + /// + /// In this mode, importing new updates only records them in the OpLog; the [loro_internal::DocState] is not updated until you reattach. + /// + /// Learn more at https://loro.dev/docs/advanced/doc_state_and_oplog#attacheddetached-status + #[inline] + pub fn detach(&self) { + self.doc.detach() + } + + /// Import a batch of updates/snapshot. + /// + /// The data can be in arbitrary order. The import result will be the same. + /// Auto-commit: same as [`import`], this finalizes the current transaction first. + /// + /// # Example + /// ``` + /// use loro::{LoroDoc, ExportMode}; + /// let a = LoroDoc::new(); + /// a.get_text("t").insert(0, "A").unwrap(); + /// let u1 = a.export(ExportMode::all_updates()).unwrap(); + /// a.get_text("t").insert(1, "B").unwrap(); + /// let u2 = a.export(ExportMode::all_updates()).unwrap(); + /// + /// let b = LoroDoc::new(); + /// let status = b.import_batch(&[u2, u1]).unwrap(); // arbitrary order + /// assert!(status.pending.is_none()); + /// ``` + #[inline] + pub fn import_batch(&self, bytes: &[Vec]) -> LoroResult { + self.doc.import_batch(bytes) + } + + /// Get a [Container] by container id. + #[inline] + pub fn get_container(&self, id: ContainerID) -> Option { + self.doc.get_handler(id).map(Container::from_handler) + } + + /// Get a [LoroMovableList] by container id. + /// + /// If the provided id is string, it will be converted into a root container id with the name of the string. + /// + /// Panics if the container does not exist. Use [`try_get_movable_list`] for a safe alternative. + #[inline] + pub fn get_movable_list(&self, id: I) -> LoroMovableList { + LoroMovableList { + handler: self.doc.get_movable_list(id), + } + } + + /// Try to get a [LoroMovableList] by container id. + /// + /// Returns `None` if the container does not exist in the document. + #[inline] + pub fn try_get_movable_list(&self, id: I) -> Option { + self.doc + .try_get_movable_list(id) + .map(|handler| LoroMovableList { handler }) + } + + /// Get a [LoroList] by container id. + /// + /// If the provided id is string, it will be converted into a root container id with the name of the string. + /// Note: creating/accessing a root container does not record history; creating nested + /// containers (e.g., `Map::insert_container`) does. + /// + /// Panics if the container does not exist. Use [`try_get_list`] for a safe alternative. + #[inline] + pub fn get_list(&self, id: I) -> LoroList { + LoroList { + handler: self.doc.get_list(id), + } + } + + /// Try to get a [LoroList] by container id. + /// + /// Returns `None` if the container does not exist in the document. + #[inline] + pub fn try_get_list(&self, id: I) -> Option { + self.doc + .try_get_list(id) + .map(|handler| LoroList { handler }) + } + + /// Get a [LoroMap] by container id. + /// + /// If the provided id is string, it will be converted into a root container id with the name of the string. + /// Note: creating/accessing a root container does not record history; creating nested + /// containers (e.g., `Map::insert_container`) does. + /// + /// Panics if the container does not exist. Use [`try_get_map`] for a safe alternative. + #[inline] + pub fn get_map(&self, id: I) -> LoroMap { + LoroMap { + handler: self.doc.get_map(id), + } + } + + /// Try to get a [LoroMap] by container id. + /// + /// Returns `None` if the container does not exist in the document. + #[inline] + pub fn try_get_map(&self, id: I) -> Option { + self.doc.try_get_map(id).map(|handler| LoroMap { handler }) + } + + /// Get a [LoroText] by container id. + /// + /// If the provided id is string, it will be converted into a root container id with the name of the string. + /// Note: creating/accessing a root container does not record history; creating nested + /// containers (e.g., `Map::insert_container`) does. + /// + /// Panics if the container does not exist. Use [`try_get_text`] for a safe alternative. + #[inline] + pub fn get_text(&self, id: I) -> LoroText { + LoroText { + handler: self.doc.get_text(id), + } + } + + /// Try to get a [LoroText] by container id. + /// + /// Returns `None` if the container does not exist in the document. + #[inline] + pub fn try_get_text(&self, id: I) -> Option { + self.doc + .try_get_text(id) + .map(|handler| LoroText { handler }) + } + + /// Get a [LoroTree] by container id. + /// + /// If the provided id is string, it will be converted into a root container id with the name of the string. + /// Note: creating/accessing a root container does not record history; creating nested + /// containers (e.g., `Map::insert_container`) does. + /// + /// Panics if the container does not exist. Use [`try_get_tree`] for a safe alternative. + #[inline] + pub fn get_tree(&self, id: I) -> LoroTree { + LoroTree { + handler: self.doc.get_tree(id), + } + } + + /// Try to get a [LoroTree] by container id. + /// + /// Returns `None` if the container does not exist in the document. + #[inline] + pub fn try_get_tree(&self, id: I) -> Option { + self.doc + .try_get_tree(id) + .map(|handler| LoroTree { handler }) + } + + #[cfg(feature = "counter")] + /// Get a [LoroCounter] by container id. + /// + /// If the provided id is string, it will be converted into a root container id with the name of the string. + /// + /// Panics if the container does not exist. Use [`try_get_counter`] for a safe alternative. + #[inline] + pub fn get_counter(&self, id: I) -> LoroCounter { + LoroCounter { + handler: self.doc.get_counter(id), + } + } + + #[cfg(feature = "counter")] + /// Try to get a [LoroCounter] by container id. + /// + /// Returns `None` if the container does not exist in the document. + #[inline] + pub fn try_get_counter(&self, id: I) -> Option { + self.doc + .try_get_counter(id) + .map(|handler| LoroCounter { handler }) + } + + /// Commit the cumulative auto commit transaction. + /// + /// There is a transaction behind every operation. + /// The events will be emitted after a transaction is committed. A transaction is committed when: + /// + /// - `doc.commit()` is called. + /// - `doc.export(mode)` is called. + /// - `doc.import(data)` is called. + /// - `doc.checkout(version)` is called. + /// + /// Note: Loro transactions are not ACID database transactions. There is no rollback or + /// isolation; they are a grouping mechanism for events/history. For interactive undo/redo, + /// use [`UndoManager`]. + /// + /// Empty-commit behavior: this method is an explicit commit. If the pending + /// transaction is empty, any previously set next-commit options (message/timestamp/origin) + /// are swallowed and will not carry over. + #[inline] + pub fn commit(&self) { + self.doc.commit_then_renew(); + } + + /// Commit the cumulative auto commit transaction with custom options. + /// + /// There is a transaction behind every operation. + /// It will automatically commit when users invoke export or import. + /// The event will be sent after a transaction is committed + /// + /// See also: [`set_next_commit_message`], [`set_next_commit_origin`], + /// [`set_next_commit_timestamp`]. Commit messages are persisted and replicate to peers; + /// origins are local-only metadata. + /// + /// Empty-commit behavior: this method is an explicit commit. If the pending + /// transaction is empty, the provided options are swallowed and will not carry over. + /// For implicit commits triggered by `export`/`checkout` (commit barriers), + /// message/timestamp/origin from an empty transaction are preserved for the next commit. + #[inline] + pub fn commit_with(&self, options: CommitOptions) { + self.doc.commit_with(options); + } + + /// Set commit message for the current uncommitted changes + /// + /// It will be persisted. + pub fn set_next_commit_message(&self, msg: &str) { + self.doc.set_next_commit_message(msg) + } + + /// Set `origin` for the current uncommitted changes, it can be used to track the source of changes in an event. + /// + /// It will NOT be persisted. + pub fn set_next_commit_origin(&self, origin: &str) { + self.doc.set_next_commit_origin(origin) + } + + /// Set the timestamp of the next commit. + /// + /// It will be persisted and stored in the `OpLog`. + /// You can get the timestamp from the [`Change`] type. + pub fn set_next_commit_timestamp(&self, timestamp: Timestamp) { + self.doc.set_next_commit_timestamp(timestamp) + } + + /// Set the options of the next commit. + /// + /// It will be used when the next commit is performed. + /// + /// # Example + /// ``` + /// use loro::{LoroDoc, CommitOptions}; + /// let doc = LoroDoc::new(); + /// doc.set_next_commit_options(CommitOptions::new().origin("ui").commit_msg("tagged")); + /// doc.get_text("t").insert(0, "x").unwrap(); + /// doc.commit(); + /// ``` + pub fn set_next_commit_options(&self, options: CommitOptions) { + self.doc.set_next_commit_options(options); + } + + /// Clear the options of the next commit. + pub fn clear_next_commit_options(&self) { + self.doc.clear_next_commit_options(); + } + + /// Whether the document is in detached mode, where the [loro_internal::DocState] is not + /// synchronized with the latest version of the [loro_internal::OpLog]. + #[inline] + pub fn is_detached(&self) -> bool { + self.doc.is_detached() + } + + /// Create a new `LoroDoc` from a snapshot. + /// + /// The snapshot is created via [`LoroDoc::export`] with [`ExportMode::Snapshot`]. + /// + /// # Example + /// ``` + /// use loro::{LoroDoc, ExportMode}; + /// + /// let doc = LoroDoc::new(); + /// let text = doc.get_text("text"); + /// text.insert(0, "Hello").unwrap(); + /// let snapshot = doc.export(ExportMode::Snapshot).unwrap(); + /// + /// let restored = LoroDoc::from_snapshot(&snapshot).unwrap(); + /// assert_eq!(restored.get_deep_value(), doc.get_deep_value()); + /// ``` + pub fn from_snapshot(bytes: &[u8]) -> LoroResult { + let inner = InnerLoroDoc::from_snapshot(bytes)?; + inner.start_auto_commit(); + Ok(Self::_new(inner)) + } + + /// Import data exported by [`LoroDoc::export`]. + /// + /// Use [`ExportMode::Snapshot`] for full-state snapshots, or + /// [`ExportMode::all_updates`] / [`ExportMode::updates`] for updates. + /// + /// # Example + /// ``` + /// use loro::{LoroDoc, ExportMode}; + /// + /// let a = LoroDoc::new(); + /// a.get_text("text").insert(0, "Hello").unwrap(); + /// let updates = a.export(ExportMode::all_updates()).unwrap(); + /// + /// let b = LoroDoc::new(); + /// b.import(&updates).unwrap(); + /// assert_eq!(a.get_deep_value(), b.get_deep_value()); + /// ``` + /// Pitfalls: + /// - Missing dependencies: check the returned [`ImportStatus`]. If `pending` is non-empty, + /// fetch those missing ranges (e.g., using `export(ExportMode::updates(&doc.oplog_vv()))`) and re-import. + /// - Auto-commit: `import` finalizes the current transaction before applying incoming data. + #[inline] + pub fn import(&self, bytes: &[u8]) -> Result { + self.doc.import_with(bytes, "".into()) + } + + /// Import data exported by [`LoroDoc::export`] and mark it with a custom origin. + /// + /// The `origin` string will be attached to the ensuing change event, which is handy + /// for telemetry or filtering. + /// Pitfalls: + /// - Same as [`import`]: verify `ImportStatus.pending` and fetch dependencies if needed. + #[inline] + pub fn import_with(&self, bytes: &[u8], origin: &str) -> Result { + self.doc.import_with(bytes, origin.into()) + } + + /// Import the json schema updates. + /// + /// # Example + /// ``` + /// use loro::{LoroDoc, VersionVector}; + /// let a = LoroDoc::new(); + /// a.get_text("t").insert(0, "hi").unwrap(); + /// a.commit(); + /// let json = a.export_json_updates(&VersionVector::default(), &a.oplog_vv()); + /// + /// let b = LoroDoc::new(); + /// b.import_json_updates(json).unwrap(); + /// assert_eq!(a.get_deep_value(), b.get_deep_value()); + /// ``` + #[inline] + pub fn import_json_updates>( + &self, + json: T, + ) -> Result { + self.doc.import_json_updates(json) + } + + /// Export the current state with json-string format of the document. + /// + /// # Example + /// ``` + /// use loro::{LoroDoc, VersionVector}; + /// let doc = LoroDoc::new(); + /// let start = VersionVector::default(); + /// let end = doc.oplog_vv(); + /// let json = doc.export_json_updates(&start, &end); + /// ``` + #[inline] + pub fn export_json_updates( + &self, + start_vv: &VersionVector, + end_vv: &VersionVector, + ) -> JsonSchema { + self.doc.export_json_updates(start_vv, end_vv, true) + } + + /// Export the current state with json-string format of the document, without peer compression. + /// + /// Compared to [`export_json_updates`], this method does not compress the peer IDs in the updates. + /// So the operations are easier to be processed by application code. + /// + /// # Example + /// ``` + /// use loro::{LoroDoc, VersionVector}; + /// let doc = LoroDoc::new(); + /// let start = VersionVector::default(); + /// let end = doc.oplog_vv(); + /// let json = doc.export_json_updates_without_peer_compression(&start, &end); + /// ``` + #[inline] + pub fn export_json_updates_without_peer_compression( + &self, + start_vv: &VersionVector, + end_vv: &VersionVector, + ) -> JsonSchema { + self.doc.export_json_updates(start_vv, end_vv, false) + } + + /// Exports changes within the specified ID span to JSON schema format. + /// + /// The JSON schema format produced by this method is identical to the one generated by `export_json_updates`. + /// It ensures deterministic output, making it ideal for hash calculations and integrity checks. + /// + /// This method can also export pending changes from the uncommitted transaction that have not yet been applied to the OpLog. + /// + /// This method will NOT trigger a new commit implicitly. + /// + /// # Example + /// ``` + /// use loro::{LoroDoc, IdSpan}; + /// + /// let doc = LoroDoc::new(); + /// doc.set_peer_id(0).unwrap(); + /// doc.get_text("text").insert(0, "a").unwrap(); + /// doc.commit(); + /// let doc_clone = doc.clone(); + /// let _sub = doc.subscribe_pre_commit(Box::new(move |e| { + /// let changes = doc_clone.export_json_in_id_span(IdSpan::new( + /// 0, + /// 0, + /// e.change_meta.id.counter + e.change_meta.len as i32, + /// )); + /// // 2 because commit one and the uncommit one + /// assert_eq!(changes.len(), 2); + /// true + /// })); + /// doc.get_text("text").insert(0, "b").unwrap(); + /// let changes = doc.export_json_in_id_span(IdSpan::new(0, 0, 2)); + /// assert_eq!(changes.len(), 1); + /// doc.commit(); + /// // change merged + /// assert_eq!(changes.len(), 1); + /// ``` + pub fn export_json_in_id_span(&self, id_span: IdSpan) -> Vec { + self.doc.export_json_in_id_span(id_span) + } + + /// Convert `Frontiers` into `VersionVector` + /// + /// Returns `None` if the frontiers are not included by this doc's OpLog. + /// + /// # Example + /// ``` + /// use loro::LoroDoc; + /// let doc = LoroDoc::new(); + /// let f = doc.state_frontiers(); + /// let vv = doc.frontiers_to_vv(&f); + /// assert!(vv.is_some()); + /// ``` + #[inline] + pub fn frontiers_to_vv(&self, frontiers: &Frontiers) -> Option { + self.doc.frontiers_to_vv(frontiers) + } + + /// Minimize the frontiers by removing the unnecessary entries. + /// + /// Returns `Err(ID)` if any frontier is not included by this doc's history. + /// + /// # Example + /// ``` + /// use loro::LoroDoc; + /// let doc = LoroDoc::new(); + /// let f = doc.state_frontiers(); + /// let _minimized = doc.minimize_frontiers(&f).unwrap(); + /// ``` + pub fn minimize_frontiers(&self, frontiers: &Frontiers) -> Result { + self.with_oplog(|oplog| shrink_frontiers(frontiers, oplog.dag())) + } + + /// Convert `VersionVector` into `Frontiers` + #[inline] + pub fn vv_to_frontiers(&self, vv: &VersionVector) -> Frontiers { + self.doc.vv_to_frontiers(vv) + } + + /// Access the `OpLog`. + /// + /// NOTE: The API in `OpLog` is unstable. Keep the closure short; avoid calling methods + /// that might re-enter the document while holding the lock. + #[inline] + pub fn with_oplog(&self, f: impl FnOnce(&OpLog) -> R) -> R { + let oplog = self.doc.oplog().lock(); + f(&oplog) + } + + /// Access the `DocState`. + /// + /// NOTE: The API in `DocState` is unstable. Keep the closure short; avoid calling methods + /// that might re-enter the document while holding the lock. + #[inline] + pub fn with_state(&self, f: impl FnOnce(&mut DocState) -> R) -> R { + let mut state = self.doc.app_state().lock(); + f(&mut state) + } + + /// Get the `VersionVector` version of `OpLog` + #[inline] + pub fn oplog_vv(&self) -> VersionVector { + self.doc.oplog_vv() + } + + /// Get the `VersionVector` version of `DocState` + #[inline] + pub fn state_vv(&self) -> VersionVector { + self.doc.state_vv() + } + + /// The doc only contains the history since this version + /// + /// This is empty if the doc is not shallow. + /// + /// The ops included by the shallow history start version vector are not in the doc. + #[inline] + pub fn shallow_since_vv(&self) -> ImVersionVector { + self.doc.shallow_since_vv() + } + + /// The doc only contains the history since this version + /// + /// This is empty if the doc is not shallow. + /// + /// The ops included by the shallow history start frontiers are not in the doc. + #[inline] + pub fn shallow_since_frontiers(&self) -> Frontiers { + self.doc.shallow_since_frontiers() + } + + /// Get the total number of operations in the `OpLog` + #[inline] + pub fn len_ops(&self) -> usize { + self.doc.len_ops() + } + + /// Get the total number of changes in the `OpLog` + #[inline] + pub fn len_changes(&self) -> usize { + self.doc.len_changes() + } + + /// Get the shallow value of the document. + #[inline] + pub fn get_value(&self) -> LoroValue { + self.doc.get_value() + } + + /// Get the entire state of the current DocState + #[inline] + pub fn get_deep_value(&self) -> LoroValue { + self.doc.get_deep_value() + } + + /// Get the entire state of the current DocState with container id + pub fn get_deep_value_with_id(&self) -> LoroValue { + self.doc.app_state().lock().get_deep_value_with_id() + } + + /// Get the `Frontiers` version of `OpLog`. + #[inline] + pub fn oplog_frontiers(&self) -> Frontiers { + self.doc.oplog_frontiers() + } + + /// Get the `Frontiers` version of `DocState`. + /// + /// When detached or during checkout, `state_frontiers()` may differ from `oplog_frontiers()`. + /// Learn more about [`Frontiers`](https://loro.dev/docs/advanced/version_deep_dive). + /// + /// # Example + /// ``` + /// use loro::LoroDoc; + /// let doc = LoroDoc::new(); + /// let before = doc.state_frontiers(); + /// doc.get_text("t").insert(0, "x").unwrap(); + /// let after = doc.state_frontiers(); + /// assert_ne!(before, after); + /// ``` + #[inline] + pub fn state_frontiers(&self) -> Frontiers { + self.doc.state_frontiers() + } + + /// Get the PeerID + #[inline] + pub fn peer_id(&self) -> PeerID { + self.doc.peer_id() + } + + /// Change the PeerID + /// + /// Pitfalls: + /// - Never reuse the same PeerID across concurrent writers (multiple tabs/devices). Duplicate + /// PeerIDs can produce conflicting OpIDs and corrupt the document. + /// - Do not assign a fixed PeerID to a user or device without strict single-ownership locking. + /// Prefer the default random PeerID per process/session. + #[inline] + pub fn set_peer_id(&self, peer: PeerID) -> LoroResult<()> { + self.doc.set_peer_id(peer) + } + + /// Subscribe the events of a container. + /// + /// The callback will be invoked after a transaction that change the container. + /// Returns a subscription that can be used to unsubscribe. + /// + /// The events will be emitted after a transaction is committed. A transaction is committed when: + /// + /// - `doc.commit()` is called. + /// - `doc.export(mode)` is called. + /// - `doc.import(data)` is called. + /// - `doc.checkout(version)` is called. + /// + /// # Example + /// + /// ``` + /// # use loro::LoroDoc; + /// # use loro::ContainerTrait; + /// # use std::sync::{atomic::AtomicBool, Arc}; + /// # use loro::{event::DiffEvent, LoroResult, TextDelta}; + /// # + /// let doc = LoroDoc::new(); + /// let text = doc.get_text("text"); + /// let ran = Arc::new(AtomicBool::new(false)); + /// let ran2 = ran.clone(); + /// let sub = doc.subscribe( + /// &text.id(), + /// Arc::new(move |event| { + /// assert!(event.triggered_by.is_local()); + /// for event in event.events { + /// let delta = event.diff.as_text().unwrap(); + /// let d = TextDelta::Insert { + /// insert: "123".into(), + /// attributes: Default::default(), + /// }; + /// assert_eq!(delta, &vec![d]); + /// ran2.store(true, std::sync::atomic::Ordering::Relaxed); + /// } + /// }), + /// ); + /// text.insert(0, "123").unwrap(); + /// doc.commit(); + /// assert!(ran.load(std::sync::atomic::Ordering::Relaxed)); + /// // unsubscribe + /// sub.unsubscribe(); + /// ``` + #[inline] + pub fn subscribe(&self, container_id: &ContainerID, callback: Subscriber) -> Subscription { + self.doc.subscribe( + container_id, + Arc::new(move |e| { + callback(DiffEvent::from(e)); + }), + ) + } + + /// Subscribe all the events. + /// + /// The callback will be invoked when any part of the [loro_internal::DocState] is changed. + /// Returns a subscription that can be used to unsubscribe. + /// + /// The events will be emitted after a transaction is committed. A transaction is committed when: + /// + /// - `doc.commit()` is called. + /// - `doc.export(mode)` is called. + /// - `doc.import(data)` is called. + /// - `doc.checkout(version)` is called. + #[inline] + pub fn subscribe_root(&self, callback: Subscriber) -> Subscription { + // self.doc.subscribe_root(callback) + self.doc.subscribe_root(Arc::new(move |e| { + callback(DiffEvent::from(e)); + })) + } + + /// Subscribe to local document updates. + /// + /// The callback receives encoded update bytes whenever local changes are committed. + /// This is useful for syncing changes to other document instances or persisting updates. + /// + /// **Auto-unsubscription**: If the callback returns `false`, the subscription will be + /// automatically removed, providing a convenient way to implement one-time or conditional + /// subscriptions in Rust. + /// + /// # Parameters + /// - `callback`: Function that receives `&Vec` (encoded updates) and returns `bool` + /// - Return `true` to keep the subscription active + /// - Return `false` to automatically unsubscribe + /// + /// # Example + /// ```rust + /// use loro::LoroDoc; + /// use std::sync::{Arc, Mutex}; + /// + /// let doc = LoroDoc::new(); + /// let updates = Arc::new(Mutex::new(Vec::new())); + /// let updates_clone = updates.clone(); + /// let count = Arc::new(Mutex::new(0)); + /// let count_clone = count.clone(); + /// + /// // Subscribe and collect first 3 updates, then auto-unsubscribe + /// let sub = doc.subscribe_local_update(Box::new(move |bytes| { + /// updates_clone.lock().unwrap().push(bytes.clone()); + /// let mut c = count_clone.lock().unwrap(); + /// *c += 1; + /// *c < 3 // Auto-unsubscribe after 3 updates + /// })); + /// + /// doc.get_text("text").insert(0, "hello").unwrap(); + /// doc.commit(); + /// ``` + pub fn subscribe_local_update(&self, callback: LocalUpdateCallback) -> Subscription { + self.doc.subscribe_local_update(callback) + } + + /// Subscribe to peer ID changes in the document. + /// + /// The callback is triggered whenever the document's peer ID is modified. + /// This is useful for tracking identity changes and updating related state accordingly. + /// + /// **Auto-unsubscription**: If the callback returns `false`, the subscription will be + /// automatically removed, providing a convenient way to implement one-time or conditional + /// subscriptions in Rust. + /// + /// # Parameters + /// - `callback`: Function that receives `&ID` (the new peer ID) and returns `bool` + /// - Return `true` to keep the subscription active + /// - Return `false` to automatically unsubscribe + /// + /// # Example + /// ```rust + /// use loro::LoroDoc; + /// use std::sync::{Arc, Mutex}; + /// + /// let doc = LoroDoc::new(); + /// let peer_changes = Arc::new(Mutex::new(Vec::new())); + /// let changes_clone = peer_changes.clone(); + /// + /// let sub = doc.subscribe_peer_id_change(Box::new(move |new_peer_id| { + /// changes_clone.lock().unwrap().push(*new_peer_id); + /// true // Keep subscription active + /// })); + /// + /// doc.set_peer_id(42).unwrap(); + /// doc.set_peer_id(100).unwrap(); + /// ``` + pub fn subscribe_peer_id_change(&self, callback: PeerIdUpdateCallback) -> Subscription { + self.doc.subscribe_peer_id_change(callback) + } + + /// Check the correctness of the document state by comparing it with the state + /// calculated by applying all the history. + #[inline] + pub fn check_state_correctness_slow(&self) { + self.doc.check_state_diff_calc_consistency_slow() + } + + /// Get the value or container by the logical path. + /// + /// Mergeable child containers are addressed through their visible map keys in + /// the logical path, not through the binary marker stored in the parent map slot. + #[inline] + pub fn get_by_path(&self, path: &[Index]) -> Option { + self.doc.get_by_path(path).map(ValueOrContainer::from) + } + + /// Get the handler by the string path. + /// + /// The path can be specified in different ways depending on the container type: + /// + /// For Tree: + /// 1. Using node IDs: `tree/{node_id}/property` + /// 2. Using indices: `tree/0/1/property` + /// + /// For List and MovableList: + /// - Using indices: `list/0` or `list/1/property` + /// + /// For Map: + /// - Using keys: `map/key` or `map/nested/property` + /// - Mergeable child containers also use their logical map keys in paths. + /// + /// For tree structures, index-based paths follow depth-first traversal order. + /// The indices start from 0 and represent the position of a node among its siblings. + /// + /// # Examples + /// ``` + /// # use loro::{LoroDoc, LoroValue}; + /// let doc = LoroDoc::new(); + /// + /// // Tree example + /// let tree = doc.get_tree("tree"); + /// let root = tree.create(None).unwrap(); + /// tree.get_meta(root).unwrap().insert("name", "root").unwrap(); + /// // Access tree by ID or index + /// let name1 = doc.get_by_str_path(&format!("tree/{}/name", root)).unwrap().into_value().unwrap(); + /// let name2 = doc.get_by_str_path("tree/0/name").unwrap().into_value().unwrap(); + /// assert_eq!(name1, name2); + /// + /// // List example + /// let list = doc.get_list("list"); + /// list.insert(0, "first").unwrap(); + /// list.insert(1, "second").unwrap(); + /// // Access list by index + /// let item = doc.get_by_str_path("list/0"); + /// assert_eq!(item.unwrap().into_value().unwrap().into_string().unwrap(), "first".into()); + /// + /// // Map example + /// let map = doc.get_map("map"); + /// map.insert("key", "value").unwrap(); + /// // Access map by key + /// let value = doc.get_by_str_path("map/key"); + /// assert_eq!(value.unwrap().into_value().unwrap().into_string().unwrap(), "value".into()); + /// + /// // MovableList example + /// let mlist = doc.get_movable_list("mlist"); + /// mlist.insert(0, "item").unwrap(); + /// // Access movable list by index + /// let item = doc.get_by_str_path("mlist/0"); + /// assert_eq!(item.unwrap().into_value().unwrap().into_string().unwrap(), "item".into()); + /// ``` + #[inline] + pub fn get_by_str_path(&self, path: &str) -> Option { + self.doc.get_by_str_path(path).map(ValueOrContainer::from) + } + + /// Get the absolute position of the given cursor. + /// + /// # Example + /// + /// ``` + /// # use loro::{LoroDoc, ToJson}; + /// let doc = LoroDoc::new(); + /// let text = &doc.get_text("text"); + /// text.insert(0, "01234").unwrap(); + /// let pos = text.get_cursor(5, Default::default()).unwrap(); + /// assert_eq!(doc.get_cursor_pos(&pos).unwrap().current.pos, 5); + /// text.insert(0, "01234").unwrap(); + /// assert_eq!(doc.get_cursor_pos(&pos).unwrap().current.pos, 10); + /// text.delete(0, 10).unwrap(); + /// assert_eq!(doc.get_cursor_pos(&pos).unwrap().current.pos, 0); + /// text.insert(0, "01234").unwrap(); + /// assert_eq!(doc.get_cursor_pos(&pos).unwrap().current.pos, 5); + /// ``` + #[inline] + pub fn get_cursor_pos( + &self, + cursor: &Cursor, + ) -> Result { + self.doc.query_pos(cursor) + } + + /// Get the inner LoroDoc ref. + #[inline] + pub fn inner(&self) -> &InnerLoroDoc { + &self.doc + } + + /// Whether the history cache is built. + #[inline] + pub fn has_history_cache(&self) -> bool { + self.doc.has_history_cache() + } + + /// Free the history cache that is used for making checkout faster. + /// + /// If you use checkout that switching to an old/concurrent version, the history cache will be built. + /// You can free it by calling this method. + #[inline] + pub fn free_history_cache(&self) { + self.doc.free_history_cache() + } + + /// Free the cached diff calculator that is used for checkout. + #[inline] + pub fn free_diff_calculator(&self) { + self.doc.free_diff_calculator() + } + + /// Encoded all ops and history cache to bytes and store them in the kv store. + /// + /// This will free up the memory that used by parsed ops + #[inline] + pub fn compact_change_store(&self) { + self.doc.compact_change_store() + } + + /// Export the document in the given mode. + /// + /// Common modes: + /// - [`ExportMode::Snapshot`]: full state + history + /// - [`ExportMode::all_updates()`]: all known ops + /// - [`ExportMode::updates(&VersionVector)`]: ops since a specific version + /// - [`ExportMode::shallow_snapshot(..)`]: GC’d snapshot starting at frontiers + /// - [`ExportMode::updates_in_range(..)`]: ops in specific ID spans + /// + /// Important notes: + /// - Auto-commit: `export` finalizes the current transaction before producing bytes. + /// - Shallow snapshots: peers cannot import updates from before the shallow start. + /// - Performance: exporting fresh snapshots periodically can reduce import time for new peers. + /// + /// # Examples + /// ``` + /// use loro::{ExportMode, LoroDoc}; + /// + /// let doc = LoroDoc::new(); + /// doc.get_text("text").insert(0, "Hello").unwrap(); + /// + /// // 1) Full snapshot + /// let snapshot = doc.export(ExportMode::Snapshot).unwrap(); + /// + /// // 2) All updates + /// let all = doc.export(ExportMode::all_updates()).unwrap(); + /// + /// // 3) Updates from another peer’s version vector + /// let vv = doc.oplog_vv(); + /// let delta = doc.export(ExportMode::updates(&vv)).unwrap(); + /// assert!(!delta.is_empty()); + /// ``` + pub fn export(&self, mode: ExportMode) -> Result, LoroEncodeError> { + self.doc.export(mode) + } + + /// Analyze the container info of the doc + /// + /// This is used for development and debugging. It can be slow. + pub fn analyze(&self) -> DocAnalysis { + self.doc.analyze() + } + + /// Get the logical path from the root to the container. + /// + /// For mergeable child containers, the path resolves through the parent map + /// key that activates the child. It does not expose the binary marker value + /// stored in the map slot. + pub fn get_path_to_container(&self, id: &ContainerID) -> Option> { + self.doc.get_path_to_container(id) + } + + /// Evaluate a JSONPath expression on the document and return matching values or handlers. + /// + /// This method allows querying the document structure using JSONPath syntax. + /// It returns a vector of `ValueOrHandler` which can represent either primitive values + /// or container handlers, depending on what the JSONPath expression matches. + /// + /// # Arguments + /// + /// * `path` - A string slice containing the JSONPath expression to evaluate. + /// + /// # Returns + /// + /// A `Result` containing either: + /// - `Ok(Vec)`: A vector of matching values or handlers. + /// - `Err(String)`: An error message if the JSONPath expression is invalid or evaluation fails. + /// + /// # Example + /// + /// ``` + /// # use loro::{LoroDoc, ToJson}; + /// + /// let doc = LoroDoc::new(); + /// let map = doc.get_map("users"); + /// map.insert("alice", 30).unwrap(); + /// map.insert("bob", 25).unwrap(); + /// + /// let result = doc.jsonpath("$.users.alice").unwrap(); + /// assert_eq!(result.len(), 1); + /// assert_eq!(result[0].as_value().unwrap().to_json_value(), serde_json::json!(30)); + /// ``` + #[inline] + #[cfg(feature = "jsonpath")] + pub fn jsonpath(&self, path: &str) -> Result, JsonPathError> { + self.doc + .jsonpath(path) + .map(|vec| vec.into_iter().map(ValueOrContainer::from).collect()) + } + + /// Subscribe to updates that might affect the given JSONPath query. + /// + /// The callback: + /// - may fire **false positives** (never false negatives) to stay lightweight; + /// - does **not** include the query result so the caller can debounce/throttle + /// and run JSONPath themselves if desired; + /// - can be debounced/throttled before executing an expensive JSONPath read. + #[cfg(feature = "jsonpath")] + pub fn subscribe_jsonpath( + &self, + jsonpath: &str, + callback: SubscribeJsonPathCallback, + ) -> LoroResult { + self.doc.subscribe_jsonpath(jsonpath, callback) + } + + /// Get the number of operations in the pending transaction. + /// + /// The pending transaction is the one that is not committed yet. It will be committed + /// after calling `doc.commit()`, `doc.export(mode)` or `doc.checkout(version)`. + pub fn get_pending_txn_len(&self) -> usize { + self.doc.get_pending_txn_len() + } + + /// Traverses the ancestors of the Change containing the given ID, including itself. + /// + /// This method visits all ancestors in causal order, from the latest to the oldest, + /// based on their Lamport timestamps. + /// + /// # Arguments + /// + /// * `ids` - The IDs of the Change to start the traversal from. + /// * `f` - A mutable function that is called for each ancestor. It can return `ControlFlow::Break(())` to stop the traversal. + pub fn travel_change_ancestors( + &self, + ids: &[ID], + f: &mut dyn FnMut(ChangeMeta) -> ControlFlow<()>, + ) -> Result<(), ChangeTravelError> { + self.doc.travel_change_ancestors(ids, f) + } + + /// Check if the doc contains the full history. + pub fn is_shallow(&self) -> bool { + self.doc.is_shallow() + } + + /// Gets container IDs modified in the given ID range. + /// + /// Pitfalls: + /// - This method will implicitly commit the current transaction to ensure the change range is finalized. + /// + /// This method can be used in conjunction with `doc.travel_change_ancestors()` to traverse + /// the history and identify all changes that affected specific containers. + /// + /// # Arguments + /// + /// * `id` - The starting ID of the change range + /// * `len` - The length of the change range to check + pub fn get_changed_containers_in(&self, id: ID, len: usize) -> FxHashSet { + self.doc.get_changed_containers_in(id, len) + } + + /// Find the operation id spans that between the `from` version and the `to` version. + /// + /// Useful for exporting just the changes in a range, e.g., in response to a subscription. + /// + /// # Example + /// ``` + /// use loro::LoroDoc; + /// let doc = LoroDoc::new(); + /// let a = doc.state_frontiers(); + /// doc.get_text("t").insert(0, "x").unwrap(); + /// doc.commit(); + /// let b = doc.state_frontiers(); + /// let spans = doc.find_id_spans_between(&a, &b); + /// assert!(!spans.forward.is_empty()); + /// ``` + #[inline] + pub fn find_id_spans_between(&self, from: &Frontiers, to: &Frontiers) -> VersionVectorDiff { + self.doc.find_id_spans_between(from, to) + } + + /// Revert the current document state back to the target version + /// + /// Internally, it will generate a series of local operations that can revert the + /// current doc to the target version. It will calculate the diff between the current + /// state and the target state, and apply the diff to the current state. + /// + /// Pitfalls: + /// - The target frontiers must be included by the document's history. If the document + /// is shallow and the target is before the shallow start, revert will fail. + /// + /// # Example + /// ``` + /// use loro::LoroDoc; + /// let doc = LoroDoc::new(); + /// let t = doc.get_text("text"); + /// t.insert(0, "Hello").unwrap(); + /// let v0 = doc.state_frontiers(); + /// t.insert(5, ", world").unwrap(); + /// doc.commit(); + /// doc.revert_to(&v0).unwrap(); + /// assert_eq!(t.to_string(), "Hello"); + /// ``` + #[inline] + pub fn revert_to(&self, version: &Frontiers) -> LoroResult<()> { + self.doc.revert_to(version) + } + + /// Apply a diff to the current document state. + /// + /// Internally, it will apply the diff to the current state. + #[inline] + pub fn apply_diff(&self, diff: DiffBatch) -> LoroResult<()> { + diff.validate_for_apply()?; + self.doc.apply_diff(diff.into()) + } + + /// Calculate the diff between two versions. + /// + /// # Example + /// ``` + /// use loro::{LoroDoc}; + /// let doc = LoroDoc::new(); + /// let t = doc.get_text("text"); + /// let a = doc.state_frontiers(); + /// t.insert(0, "a").unwrap(); + /// let b = doc.state_frontiers(); + /// let diff = doc.diff(&a, &b).unwrap(); + /// assert!(diff.iter().next().is_some()); + /// ``` + #[inline] + pub fn diff(&self, a: &Frontiers, b: &Frontiers) -> LoroResult { + self.doc.diff(a, b).map(|x| x.into()) + } + + /// Check if the doc contains the target container. + /// + /// A root container always exists, while a normal container exists if it has + /// ever been created on the doc. Mergeable child ids are encoded as synthetic + /// root-shaped ids, but they are not ordinary roots: they exist only after + /// the mergeable child has materialized state. Parent map marker visibility + /// and child state existence are tracked separately. + /// + /// A mergeable container (created via `LoroMap::ensure_mergeable_*`) exists once it + /// has been ensured — even before any op is written into it — for as long as the + /// parent map's child ref is alive, or once it has state of its own. A mergeable cid + /// that was never ensured on this doc does not exist. + /// + /// # Examples + /// ``` + /// use loro::{LoroDoc, LoroText, LoroList, ExportMode}; + /// + /// let doc = LoroDoc::new(); + /// doc.set_peer_id(1); + /// let map = doc.get_map("map"); + /// map.insert_container("text", LoroText::new()).unwrap(); + /// map.insert_container("list", LoroList::new()).unwrap(); + /// + /// // Root map container exists + /// assert!(doc.has_container(&"cid:root-map:Map".try_into().unwrap())); + /// // Text container exists + /// assert!(doc.has_container(&"cid:0@1:Text".try_into().unwrap())); + /// // List container exists + /// assert!(doc.has_container(&"cid:1@1:List".try_into().unwrap())); + /// + /// let doc2 = LoroDoc::new(); + /// // Containers exist as long as the history or doc state includes them + /// doc.detach(); + /// doc2.import(&doc.export(ExportMode::all_updates()).unwrap()).unwrap(); + /// assert!(doc2.has_container(&"cid:root-map:Map".try_into().unwrap())); + /// assert!(doc2.has_container(&"cid:0@1:Text".try_into().unwrap())); + /// assert!(doc2.has_container(&"cid:1@1:List".try_into().unwrap())); + /// ``` + pub fn has_container(&self, container_id: &ContainerID) -> bool { + self.doc.has_container(container_id) + } + + /// Subscribe to the first commit from a peer. Operations performed on the `LoroDoc` within this callback + /// will be merged into the current commit. + /// + /// Subscribe to the first commit event from each peer. + /// + /// The callback is triggered only once per peer when they make their first commit to the document locally. + /// This is particularly useful for managing peer-to-user mappings or initialization logic. + /// + /// **Auto-unsubscription**: If the callback returns `false`, the subscription will be + /// automatically removed, providing a convenient way to implement one-time or conditional + /// subscriptions in Rust. + /// + /// # Parameters + /// - `callback`: Function that receives `&FirstCommitFromPeerPayload` and returns `bool` + /// - Return `true` to keep the subscription active + /// - Return `false` to automatically unsubscribe + /// + /// # Use Cases + /// - Initialize peer-specific data structures + /// - Map peer IDs to user information + /// + /// # Example + /// ```rust + /// use loro::LoroDoc; + /// use std::sync::{Arc, Mutex}; + /// + /// let doc = LoroDoc::new(); + /// doc.set_peer_id(0).unwrap(); + /// + /// let new_peers = Arc::new(Mutex::new(Vec::new())); + /// let peers_clone = new_peers.clone(); + /// let peer_count = Arc::new(Mutex::new(0)); + /// let count_clone = peer_count.clone(); + /// + /// // Track first 5 new peers, then auto-unsubscribe + /// let sub = doc.subscribe_first_commit_from_peer(Box::new(move |payload| { + /// peers_clone.lock().unwrap().push(payload.peer); + /// let mut count = count_clone.lock().unwrap(); + /// *count += 1; + /// *count < 5 // Auto-unsubscribe after tracking 5 peers + /// })); + /// + /// // This will trigger the callback for peer 0 + /// doc.get_text("text").insert(0, "hello").unwrap(); + /// doc.commit(); + /// + /// // Switch to a new peer and commit - triggers callback again + /// doc.set_peer_id(1).unwrap(); + /// doc.get_text("text").insert(0, "world").unwrap(); + /// doc.commit(); + /// ``` + pub fn subscribe_first_commit_from_peer( + &self, + callback: FirstCommitFromPeerCallback, + ) -> Subscription { + self.doc.subscribe_first_commit_from_peer(callback) + } + + /// Subscribe to pre-commit events. + /// + /// The callback is triggered when changes are about to be committed but before they're + /// applied to the OpLog. This allows you to modify commit metadata such as timestamps + /// and messages, or perform validation before changes are finalized. + /// + /// **Auto-unsubscription**: If the callback returns `false`, the subscription will be + /// automatically removed, providing a convenient way to implement one-time or conditional + /// subscriptions in Rust. + /// + /// Pitfall: `commit()` can be triggered implicitly by `import`, `export`, and `checkout`. + /// This hook still runs for those commits, which is helpful for annotating metadata + /// even for implicit commits. + /// + /// # Parameters + /// - `callback`: Function that receives `&PreCommitCallbackPayload` and returns `bool` + /// - Return `true` to keep the subscription active + /// - Return `false` to automatically unsubscribe + /// - The payload contains: + /// - `change_meta`: Metadata about the commit + /// - `modifier`: Interface to modify commit properties + /// + /// # Use Cases + /// - Add commit message prefixes or formatting + /// - Adjust timestamps for consistent ordering + /// - Log or audit commit operations + /// - Implement commit validation or approval workflows + /// + /// # Example + /// ```rust + /// use loro::LoroDoc; + /// use std::sync::{Arc, Mutex}; + /// + /// let doc = LoroDoc::new(); + /// let commit_count = Arc::new(Mutex::new(0)); + /// let count_clone = commit_count.clone(); + /// + /// // Add timestamps and auto-unsubscribe after 5 commits + /// let sub = doc.subscribe_pre_commit(Box::new(move |payload| { + /// // Add a prefix to commit messages + /// let new_message = format!("Auto: {}", payload.change_meta.message()); + /// payload.modifier.set_message(&new_message); + /// + /// let mut count = count_clone.lock().unwrap(); + /// *count += 1; + /// *count < 5 // Auto-unsubscribe after 5 commits + /// })); + /// + /// doc.get_text("text").insert(0, "hello").unwrap(); + /// doc.commit(); + /// ``` + pub fn subscribe_pre_commit(&self, callback: PreCommitCallback) -> Subscription { + self.doc.subscribe_pre_commit(callback) + } + + /// Delete all content from a root container and hide it from the document. + /// + /// When a root container is empty and hidden: + /// - It won't show up in `get_deep_value()` results + /// - It won't be included in document snapshots + /// + /// Only works on root containers (containers without parents). + pub fn delete_root_container(&self, cid: ContainerID) { + self.doc.delete_root_container(cid); + } + + /// Set whether to hide empty root containers. + /// + /// # Example + /// ``` + /// use loro::LoroDoc; + /// + /// let doc = LoroDoc::new(); + /// let map = doc.get_map("map"); + /// dbg!(doc.get_deep_value()); // {"map": {}} + /// doc.set_hide_empty_root_containers(true); + /// dbg!(doc.get_deep_value()); // {} + /// ``` + pub fn set_hide_empty_root_containers(&self, hide: bool) { + self.doc.set_hide_empty_root_containers(hide); + } +} + +/// It's used to prevent the user from implementing the trait directly. +#[allow(private_bounds)] +trait SealedTrait {} + +/// The common trait for all the containers. +/// It's used internally, you can't implement it directly. +#[allow(private_bounds)] +pub trait ContainerTrait: SealedTrait { + /// The handler of the container. + type Handler: HandlerTrait; + /// Get the ID of the container. + fn id(&self) -> ContainerID; + /// Convert the container to a [Container]. + fn to_container(&self) -> Container; + /// Convert the container to a handler. + fn to_handler(&self) -> Self::Handler; + /// Convert the handler to a container. + fn from_handler(handler: Self::Handler) -> Self; + /// Try to convert the container to the handler. + fn try_from_container(container: Container) -> Option + where + Self: Sized; + /// Whether the container is attached to a document. + fn is_attached(&self) -> bool; + /// If a detached container is attached, this method will return its corresponding attached handler. + fn get_attached(&self) -> Option + where + Self: Sized; + /// Whether the container is deleted. + fn is_deleted(&self) -> bool; + /// Get the doc of the container. + fn doc(&self) -> Option; + /// Subscribe to the container. + /// + /// If the Container is detached, this method will return `None`. + fn subscribe(&self, callback: Subscriber) -> Option { + self.doc().map(|doc| doc.subscribe(&self.id(), callback)) + } +} + +/// LoroList container. It's used to model arrays. +/// +/// It can have sub containers. +/// +/// Important: choose the right structure. +/// - Use `LoroList` for ordered collections where elements are appended/inserted/deleted. +/// - Use `LoroMovableList` when frequent reordering (drag-and-drop) is needed. +/// - Use `LoroMap` for keyed records and coordinates. +/// +/// ```no_run +/// // Bad: coordinates in a list can diverge under concurrency +/// // let coord = doc.get_list("coord"); +/// // coord.insert(0, 10).unwrap(); // x +/// // coord.insert(1, 20).unwrap(); // y +/// +/// // Good: use a map for labeled fields +/// // let coord = doc.get_map("coord"); +/// // coord.insert("x", 10).unwrap(); +/// // coord.insert("y", 20).unwrap(); +/// ``` +/// +/// ``` +/// # use loro::{LoroDoc, ContainerType, ToJson}; +/// # use serde_json::json; +/// let doc = LoroDoc::new(); +/// let list = doc.get_list("list"); +/// list.insert(0, 123).unwrap(); +/// list.insert(1, "h").unwrap(); +/// assert_eq!( +/// doc.get_deep_value().to_json_value(), +/// json!({ +/// "list": [123, "h"] +/// }) +/// ); +/// ``` +#[derive(Clone, Debug)] +pub struct LoroList { + handler: InnerListHandler, +} + +impl SealedTrait for LoroList {} +impl ContainerTrait for LoroList { + type Handler = InnerListHandler; + + fn id(&self) -> ContainerID { + self.handler.id() + } + + fn to_container(&self) -> Container { + Container::List(self.clone()) + } + + fn to_handler(&self) -> Self::Handler { + self.handler.clone() + } + + fn from_handler(handler: Self::Handler) -> Self { + Self { handler } + } + + fn is_attached(&self) -> bool { + self.handler.is_attached() + } + + fn get_attached(&self) -> Option { + self.handler.get_attached().map(Self::from_handler) + } + + fn try_from_container(container: Container) -> Option { + container.into_list().ok() + } + + fn is_deleted(&self) -> bool { + self.handler.is_deleted() + } + + fn doc(&self) -> Option { + self.handler.doc().map(|doc| { + doc.start_auto_commit(); + LoroDoc::_new(doc) + }) + } +} + +impl LoroList { + /// Create a new container that is detached from the document. + /// + /// The edits on a detached container will not be persisted. + /// To attach the container to the document, please insert it into an attached container. + pub fn new() -> Self { + Self { + handler: InnerListHandler::new_detached(), + } + } + + /// Whether the container is attached to a document + /// + /// The edits on a detached container will not be persisted. + /// To attach the container to the document, please insert it into an attached container. + pub fn is_attached(&self) -> bool { + self.handler.is_attached() + } + + /// Insert a value at the given position. + pub fn insert(&self, pos: usize, v: impl Into) -> LoroResult<()> { + self.handler.insert(pos, v) + } + + /// Delete values at the given position. + #[inline] + pub fn delete(&self, pos: usize, len: usize) -> LoroResult<()> { + self.handler.delete(pos, len) + } + + /// Get the value at the given position. + #[inline] + pub fn get(&self, index: usize) -> Option { + self.handler.get_(index).map(ValueOrContainer::from) + } + + /// Get the deep value of the container. + #[inline] + pub fn get_deep_value(&self) -> LoroValue { + self.handler.get_deep_value() + } + + /// Get the shallow value of the container. + /// + /// This does not convert the state of sub-containers; instead, it represents them as [LoroValue::Container]. + #[inline] + pub fn get_value(&self) -> LoroValue { + self.handler.get_value() + } + + /// Pop the last element of the list. + #[inline] + pub fn pop(&self) -> LoroResult> { + self.handler.pop() + } + + /// Push a value to the list. + #[inline] + pub fn push(&self, v: impl Into) -> LoroResult<()> { + self.handler.push(v.into()) + } + + /// Push a container to the list. + #[inline] + pub fn push_container(&self, child: C) -> LoroResult { + let pos = self.handler.len(); + Ok(C::from_handler( + self.handler.insert_container(pos, child.to_handler())?, + )) + } + + /// Iterate over the elements of the list. + pub fn for_each(&self, mut f: I) + where + I: FnMut(ValueOrContainer), + { + self.handler.for_each(&mut |v| { + f(ValueOrContainer::from(v)); + }) + } + + /// Get the length of the list. + #[inline] + pub fn len(&self) -> usize { + self.handler.len() + } + + /// Whether the list is empty. + #[inline] + pub fn is_empty(&self) -> bool { + self.handler.is_empty() + } + + /// Insert a container with the given type at the given index. + /// + /// # Example + /// + /// ``` + /// # use loro::{LoroDoc, ContainerType, LoroText, ToJson}; + /// # use serde_json::json; + /// let doc = LoroDoc::new(); + /// let list = doc.get_list("m"); + /// let text = list.insert_container(0, LoroText::new()).unwrap(); + /// text.insert(0, "12"); + /// text.insert(0, "0"); + /// assert_eq!(doc.get_deep_value().to_json_value(), json!({"m": ["012"]})); + /// ``` + #[inline] + pub fn insert_container(&self, pos: usize, child: C) -> LoroResult { + Ok(C::from_handler( + self.handler.insert_container(pos, child.to_handler())?, + )) + } + + /// Get the cursor at the given position. + /// + /// Using "index" to denote cursor positions can be unstable, as positions may + /// shift with document edits. To reliably represent a position or range within + /// a document, it is more effective to leverage the unique ID of each item/character + /// in a List CRDT or Text CRDT. + /// + /// Loro optimizes State metadata by not storing the IDs of deleted elements. This + /// approach complicates tracking cursors since they rely on these IDs. The solution + /// recalculates position by replaying relevant history to update stable positions + /// accurately. To minimize the performance impact of history replay, the system + /// updates cursor info to reference only the IDs of currently present elements, + /// thereby reducing the need for replay. + /// + /// # Example + /// + /// ``` + /// use loro::LoroDoc; + /// use loro_internal::cursor::Side; + /// + /// let doc = LoroDoc::new(); + /// let list = doc.get_list("list"); + /// list.insert(0, 0).unwrap(); + /// let cursor = list.get_cursor(0, Side::Middle).unwrap(); + /// assert_eq!(doc.get_cursor_pos(&cursor).unwrap().current.pos, 0); + /// list.insert(0, 0).unwrap(); + /// assert_eq!(doc.get_cursor_pos(&cursor).unwrap().current.pos, 1); + /// list.insert(0, 0).unwrap(); + /// list.insert(0, 0).unwrap(); + /// assert_eq!(doc.get_cursor_pos(&cursor).unwrap().current.pos, 3); + /// list.insert(4, 0).unwrap(); + /// assert_eq!(doc.get_cursor_pos(&cursor).unwrap().current.pos, 3); + /// ``` + pub fn get_cursor(&self, pos: usize, side: Side) -> Option { + self.handler.get_cursor(pos, side) + } + + /// Converts the LoroList to a Vec of LoroValue. + /// + /// This method unwraps the internal Arc and clones the data if necessary, + /// returning a Vec containing all the elements of the LoroList as LoroValue. + /// + /// # Returns + /// + /// A Vec containing all elements of the LoroList. + /// + /// # Example + /// + /// ``` + /// use loro::{LoroDoc, LoroValue}; + /// + /// let doc = LoroDoc::new(); + /// let list = doc.get_list("my_list"); + /// list.insert(0, 1).unwrap(); + /// list.insert(1, "hello").unwrap(); + /// list.insert(2, true).unwrap(); + /// + /// let vec = list.to_vec(); + /// ``` + pub fn to_vec(&self) -> Vec { + self.get_value().into_list().unwrap().unwrap() + } + + /// Delete all elements in the list. + pub fn clear(&self) -> LoroResult<()> { + self.handler.clear() + } + + /// Get the ID of the list item at the given position. + pub fn get_id_at(&self, pos: usize) -> Option { + self.handler.get_id_at(pos) + } +} + +impl Default for LoroList { + fn default() -> Self { + Self::new() + } +} + +/// LoroMap container. +/// +/// It's LWW(Last-Write-Win) Map. It can support Multi-Value Map in the future. +/// +/// # Mergeable child containers +/// +/// The `ensure_mergeable_*` methods create deterministic child containers under a +/// map key. If two peers independently call the same method with the same +/// `(parent map, key, container type)`, they get the same child container id and +/// their subsequent edits merge in that child. +/// +/// Internally, activation writes a specially constructed `LoroValue::Binary` +/// marker into the parent map slot. String values do not activate mergeable +/// children. If the key already holds a non-mergeable value, `ensure_mergeable_*` +/// returns an error and leaves the value unchanged. +/// Calling `ensure_mergeable_*` with a different container type at an existing +/// mergeable key is a deliberate kind change: the map slot activates the new +/// type, while the state of the other mergeable child remains preserved under +/// its deterministic id. +/// +/// Deleting the map key clears the ref and hides the mergeable child, but the +/// child's state remains addressable by its deterministic id. Calling the same +/// `ensure_mergeable_*` method again writes the ref back and resurfaces the +/// preserved child state. +/// +/// # Example +/// ``` +/// # use loro::{LoroDoc, ToJson, ExpandType, LoroValue}; +/// # use serde_json::json; +/// let doc = LoroDoc::new(); +/// let map = doc.get_map("map"); +/// map.insert("key", "value").unwrap(); +/// map.insert("true", true).unwrap(); +/// map.insert("null", LoroValue::Null).unwrap(); +/// map.insert("deleted", LoroValue::Null).unwrap(); +/// map.delete("deleted").unwrap(); +/// let text = map.ensure_mergeable_text("text").unwrap(); +/// text.insert(0, "Hello world!").unwrap(); +/// assert_eq!( +/// doc.get_deep_value().to_json_value(), +/// json!({ +/// "map": { +/// "key": "value", +/// "true": true, +/// "null": null, +/// "text": "Hello world!" +/// } +/// }) +/// ); +/// ``` +#[derive(Clone, Debug)] +pub struct LoroMap { + handler: InnerMapHandler, +} + +impl SealedTrait for LoroMap {} +impl ContainerTrait for LoroMap { + type Handler = InnerMapHandler; + + fn id(&self) -> ContainerID { + self.handler.id() + } + + fn to_container(&self) -> Container { + Container::Map(self.clone()) + } + + fn to_handler(&self) -> Self::Handler { + self.handler.clone() + } + + fn from_handler(handler: Self::Handler) -> Self { + Self { handler } + } + + fn is_attached(&self) -> bool { + self.handler.is_attached() + } + + fn get_attached(&self) -> Option { + self.handler.get_attached().map(Self::from_handler) + } + + fn try_from_container(container: Container) -> Option { + container.into_map().ok() + } + + fn is_deleted(&self) -> bool { + self.handler.is_deleted() + } + + fn doc(&self) -> Option { + self.handler.doc().map(|doc| { + doc.start_auto_commit(); + LoroDoc::_new(doc) + }) + } +} + +impl LoroMap { + /// Create a new container that is detached from the document. + /// + /// The edits on a detached container will not be persisted. + /// To attach the container to the document, please insert it into an attached container. + pub fn new() -> Self { + Self { + handler: InnerMapHandler::new_detached(), + } + } + + /// Whether the container is attached to a document. + pub fn is_attached(&self) -> bool { + self.handler.is_attached() + } + + /// Delete a key-value pair from the map. + pub fn delete(&self, key: &str) -> LoroResult<()> { + self.handler.delete(key) + } + + /// Iterate over the key-value pairs of the map. + pub fn for_each(&self, mut f: I) + where + I: FnMut(&str, ValueOrContainer), + { + self.handler.for_each(|k, v| { + f(k, ValueOrContainer::from(v)); + }) + } + + /// Insert a key-value pair into the map. + /// + /// > **Note**: When calling `map.set(key, value)` on a LoroMap, if `map.get(key)` already returns `value`, + /// > the operation will be a no-op (no operation recorded) to avoid unnecessary updates. + pub fn insert(&self, key: &str, value: impl Into) -> LoroResult<()> { + self.handler.insert(key, value) + } + + /// Get the length of the map. + pub fn len(&self) -> usize { + self.handler.len() + } + + /// Whether the map is empty. + pub fn is_empty(&self) -> bool { + self.handler.is_empty() + } + + /// Get the value of the map with the given key. + pub fn get(&self, key: &str) -> Option { + self.handler.get_(key).map(ValueOrContainer::from) + } + + /// Insert a regular child container with the given type at the given key. + /// + /// The inserted child receives a regular op-created container id. Use this + /// for replacement semantics or when each creation should produce a distinct + /// child. Use [`ensure_mergeable_map`](Self::ensure_mergeable_map), + /// [`ensure_mergeable_list`](Self::ensure_mergeable_list), + /// [`ensure_mergeable_movable_list`](Self::ensure_mergeable_movable_list), + /// [`ensure_mergeable_text`](Self::ensure_mergeable_text), + /// [`ensure_mergeable_tree`](Self::ensure_mergeable_tree), or + /// [`ensure_mergeable_counter`](Self::ensure_mergeable_counter) when peers + /// may lazily initialize the same logical child under a map key. + /// + /// # Example + /// + /// ``` + /// # use loro::{LoroDoc, LoroText, ContainerType, ToJson}; + /// # use serde_json::json; + /// let doc = LoroDoc::new(); + /// let map = doc.get_map("m"); + /// let text = map.insert_container("t", LoroText::new()).unwrap(); + /// text.insert(0, "12"); + /// text.insert(0, "0"); + /// assert_eq!(doc.get_deep_value().to_json_value(), json!({"m": {"t": "012"}})); + /// ``` + /// + /// Pitfalls: + /// - Concurrently inserting different containers at the same map key on different peers + /// can result in one overwriting the other rather than merging. Prefer + /// `ensure_mergeable_*` when the child should be identified by `(parent map, key, type)`. + pub fn insert_container(&self, key: &str, child: C) -> LoroResult { + Ok(C::from_handler( + self.handler.insert_container(key, child.to_handler())?, + )) + } + + /// Get the shallow value of the map. + /// + /// It will not convert the state of sub-containers, but represent them as [LoroValue::Container]. + pub fn get_value(&self) -> LoroValue { + self.handler.get_value() + } + + /// Get the deep value of the map. + /// + /// It will convert the state of sub-containers into a nested JSON value. + pub fn get_deep_value(&self) -> LoroValue { + self.handler.get_deep_value() + } + + /// Get or create a regular child container with the given key. + /// + /// This legacy method creates regular op-id child containers when the key is + /// empty or `null`. It is not mergeable: concurrent first creation at the + /// same map key can fork child state and leave one branch hidden by map + /// conflict resolution. + /// + /// Pitfalls: + /// - If other peers concurrently create a different container at the same key, their state + /// may be overwritten. Prefer `ensure_mergeable_*` for lazy map-key child creation. + #[deprecated( + note = "use ensure_mergeable_map/list/movable_list/text/tree/counter for lazy map-key child creation; this method creates regular op-id children" + )] + #[allow(deprecated)] + pub fn get_or_create_container(&self, key: &str, child: C) -> LoroResult { + Ok(C::from_handler( + self.handler + .get_or_create_container(key, child.to_handler())?, + )) + } + + /// Ensure a mergeable Counter exists at this map key and return it. + /// + /// See [`LoroMap`]'s [mergeable child containers](#mergeable-child-containers) + /// section for merge semantics. + /// + /// Returns [`LoroError::ArgErr`] if the key already holds a non-mergeable value; the existing + /// value is left unchanged. + /// Returns [`LoroError::MisuseDetachedContainer`] if this map is detached. + #[cfg(feature = "counter")] + pub fn ensure_mergeable_counter(&self, key: &str) -> LoroResult { + Ok(LoroCounter::from_handler( + self.handler.ensure_mergeable_counter(key)?, + )) + } + + /// Ensure a mergeable Map exists at this map key and return it. + /// + /// See [`LoroMap`]'s [mergeable child containers](#mergeable-child-containers) + /// section for merge semantics. + /// + /// Returns [`LoroError::ArgErr`] if the key already holds a non-mergeable value; the existing + /// value is left unchanged. + /// Returns [`LoroError::MisuseDetachedContainer`] if this map is detached. + pub fn ensure_mergeable_map(&self, key: &str) -> LoroResult { + Ok(LoroMap::from_handler( + self.handler.ensure_mergeable_map(key)?, + )) + } + + /// Ensure a mergeable List exists at this map key and return it. + /// + /// See [`LoroMap`]'s [mergeable child containers](#mergeable-child-containers) + /// section for merge semantics. + /// + /// Returns [`LoroError::ArgErr`] if the key already holds a non-mergeable value; the existing + /// value is left unchanged. + /// Returns [`LoroError::MisuseDetachedContainer`] if this map is detached. + pub fn ensure_mergeable_list(&self, key: &str) -> LoroResult { + Ok(LoroList::from_handler( + self.handler.ensure_mergeable_list(key)?, + )) + } + + /// Ensure a mergeable MovableList exists at this map key and return it. + /// + /// See [`LoroMap`]'s [mergeable child containers](#mergeable-child-containers) + /// section for merge semantics. + /// + /// Returns [`LoroError::ArgErr`] if the key already holds a non-mergeable value; the existing + /// value is left unchanged. + /// Returns [`LoroError::MisuseDetachedContainer`] if this map is detached. + pub fn ensure_mergeable_movable_list(&self, key: &str) -> LoroResult { + Ok(LoroMovableList::from_handler( + self.handler.ensure_mergeable_movable_list(key)?, + )) + } + + /// Ensure a mergeable Text exists at this map key and return it. + /// + /// See [`LoroMap`]'s [mergeable child containers](#mergeable-child-containers) + /// section for merge semantics. + /// + /// Returns [`LoroError::ArgErr`] if the key already holds a non-mergeable value; the existing + /// value is left unchanged. + /// Returns [`LoroError::MisuseDetachedContainer`] if this map is detached. + pub fn ensure_mergeable_text(&self, key: &str) -> LoroResult { + Ok(LoroText::from_handler( + self.handler.ensure_mergeable_text(key)?, + )) + } + + /// Ensure a mergeable Tree exists at this map key and return it. + /// + /// See [`LoroMap`]'s [mergeable child containers](#mergeable-child-containers) + /// section for merge semantics. + /// + /// Returns [`LoroError::ArgErr`] if the key already holds a non-mergeable value; the existing + /// value is left unchanged. + /// Returns [`LoroError::MisuseDetachedContainer`] if this map is detached. + pub fn ensure_mergeable_tree(&self, key: &str) -> LoroResult { + Ok(LoroTree::from_handler( + self.handler.ensure_mergeable_tree(key)?, + )) + } + + /// Delete all key-value pairs in the map. + pub fn clear(&self) -> LoroResult<()> { + self.handler.clear() + } + + /// Get the keys of the map. + pub fn keys(&self) -> impl Iterator + '_ { + self.handler.keys() + } + + /// Get the values of the map. + pub fn values(&self) -> impl Iterator + '_ { + self.handler.values().map(ValueOrContainer::from) + } + + /// Get the peer id of the last editor on the given entry + pub fn get_last_editor(&self, key: &str) -> Option { + self.handler.get_last_editor(key) + } +} + +impl Default for LoroMap { + fn default() -> Self { + Self::new() + } +} + +/// LoroText container. It's used to model plaintext/richtext. +/// +/// Indexing and lengths: +/// - Rust APIs default to Unicode scalar positions for `insert`/`delete` and `slice`. +/// - For byte-based integration, use `insert_utf8`/`delete_utf8`. +/// - For UTF-16 code unit offsets (e.g., integrating with JavaScript), use +/// `insert_utf16`/`delete_utf16`/`slice_utf16` and related helpers. +/// - You can inspect `len_unicode`, `len_utf8`, and `len_utf16` depending on your needs. +/// +/// # Example (emoji) +/// ``` +/// use loro::LoroDoc; +/// let doc = LoroDoc::new(); +/// let text = doc.get_text("text"); +/// text.insert(0, "Hello 😀 World").unwrap(); +/// assert_eq!(text.len_unicode(), 13); // visible characters +/// assert!(text.len_utf16() >= text.len_unicode()); // emoji may count as 2 in UTF-16 +/// // Delete the emoji safely by Unicode indices +/// let start = 6; // after "Hello " +/// text.delete(start, 1).unwrap(); +/// assert_eq!(text.to_string(), "Hello World"); +/// ``` +#[derive(Clone, Debug)] +pub struct LoroText { + handler: InnerTextHandler, +} + +impl SealedTrait for LoroText {} +impl ContainerTrait for LoroText { + type Handler = InnerTextHandler; + + fn id(&self) -> ContainerID { + self.handler.id() + } + + fn to_container(&self) -> Container { + Container::Text(self.clone()) + } + + fn to_handler(&self) -> Self::Handler { + self.handler.clone() + } + + fn from_handler(handler: Self::Handler) -> Self { + Self { handler } + } + + fn is_attached(&self) -> bool { + self.handler.is_attached() + } + + fn get_attached(&self) -> Option { + self.handler.get_attached().map(Self::from_handler) + } + + fn try_from_container(container: Container) -> Option { + container.into_text().ok() + } + + fn is_deleted(&self) -> bool { + self.handler.is_deleted() + } + + fn doc(&self) -> Option { + self.handler.doc().map(|doc| { + doc.start_auto_commit(); + LoroDoc::_new(doc) + }) + } +} + +impl LoroText { + /// Create a new container that is detached from the document. + /// + /// The edits on a detached container will not be persisted. + /// To attach the container to the document, please insert it into an attached container. + pub fn new() -> Self { + Self { + handler: InnerTextHandler::new_detached(), + } + } + + /// Whether the container is attached to a document + /// + /// The edits on a detached container will not be persisted. + /// To attach the container to the document, please insert it into an attached container. + pub fn is_attached(&self) -> bool { + self.handler.is_attached() + } + + /// Iterate over contiguous text chunks. + /// + /// The callback function will be called for each contiguous text segment (internal span), + /// not necessarily a single character. If you need per-character iteration, iterate the + /// returned `&str` within the callback. + /// If the callback returns `false`, the iteration will stop. + /// + /// The current text chunks are snapshotted before callbacks run, so the callback + /// may read or mutate the document without re-entering internal state locks. + pub fn iter(&self, callback: impl FnMut(&str) -> bool) { + self.handler.iter(callback); + } + + /// Insert a string at the given unicode position. + pub fn insert(&self, pos: usize, s: &str) -> LoroResult<()> { + self.handler.insert_unicode(pos, s) + } + + /// Insert a string at the given utf-8 position. + pub fn insert_utf8(&self, pos: usize, s: &str) -> LoroResult<()> { + self.handler.insert_utf8(pos, s) + } + + /// Insert a string at the given UTF-16 code unit position. + /// + /// This is useful when working with JavaScript or other UTF-16-based index systems. + pub fn insert_utf16(&self, pos: usize, s: &str) -> LoroResult<()> { + self.handler.insert_utf16(pos, s) + } + + /// Delete a range of text at the given unicode position with unicode length. + pub fn delete(&self, pos: usize, len: usize) -> LoroResult<()> { + self.handler.delete_unicode(pos, len) + } + + /// Delete a range of text at the given utf-8 position with utf-8 length. + pub fn delete_utf8(&self, pos: usize, len: usize) -> LoroResult<()> { + self.handler.delete_utf8(pos, len) + } + + /// Delete a range of text at the given UTF-16 code unit position with UTF-16 length. + pub fn delete_utf16(&self, pos: usize, len: usize) -> LoroResult<()> { + self.handler.delete_utf16(pos, len) + } + + /// Get a string slice at the given Unicode range + pub fn slice(&self, start_index: usize, end_index: usize) -> LoroResult { + self.handler + .slice(start_index, end_index, cursor::PosType::Unicode) + } + + /// Get a string slice using UTF-16 code unit offsets. + pub fn slice_utf16(&self, start_index: usize, end_index: usize) -> LoroResult { + self.handler.slice_utf16(start_index, end_index) + } + + /// Get the rich-text delta within a range. + /// + /// The range is expressed in the coordinate system selected by `pos_type`: + /// Unicode scalar indices (`PosType::Unicode`), UTF-8 bytes, UTF-16 units, etc. + /// The returned [`TextDelta`] segments contain only inserts and preserve the + /// attributes active inside the slice. + /// + /// # Errors + /// Returns [`LoroError::OutOfBound`](crate::LoroError::OutOfBound) when the range + /// exceeds the current text length in the chosen coordinate system, or + /// [`LoroError::EndIndexLessThanStartIndex`](crate::LoroError::EndIndexLessThanStartIndex) + /// if `end < start`. + /// + /// # Examples + /// ``` + /// use loro::{LoroDoc, TextDelta}; + /// use loro::cursor::PosType; + /// + /// let doc = LoroDoc::new(); + /// let text = doc.get_text("text"); + /// text.insert(0, "A😀BC").unwrap(); + /// text.mark(0..2, "bold", true).unwrap(); // A and 😀 + /// + /// let delta = text.slice_delta(1, 3, PosType::Unicode).unwrap(); + /// assert_eq!(delta.len(), 2); + /// if let TextDelta::Insert { insert, attributes } = &delta[0] { + /// assert_eq!(insert, "😀"); + /// assert_eq!(attributes.as_ref().unwrap().get("bold").unwrap(), &true.into()); + /// } else { + /// unreachable!(); + /// } + /// if let TextDelta::Insert { insert, attributes } = &delta[1] { + /// assert_eq!(insert, "B"); + /// assert!(attributes.is_none()); + /// } else { + /// unreachable!(); + /// } + /// ``` + pub fn slice_delta( + &self, + start: usize, + end: usize, + pos_type: cursor::PosType, + ) -> LoroResult> { + self.handler.slice_delta(start, end, pos_type) + } + + /// Get the characters at given unicode position. + pub fn char_at(&self, pos: usize) -> LoroResult { + self.handler.char_at(pos, cursor::PosType::Unicode) + } + + /// Delete specified character and insert string at the same position at given unicode position. + pub fn splice(&self, pos: usize, len: usize, s: &str) -> LoroResult { + self.handler.splice(pos, len, s, cursor::PosType::Unicode) + } + + /// Delete specified range and insert a string at the same UTF-16 position. + pub fn splice_utf16(&self, pos: usize, len: usize, s: &str) -> LoroResult<()> { + self.handler.splice_utf16(pos, len, s) + } + + /// Whether the text container is empty. + pub fn is_empty(&self) -> bool { + self.handler.is_empty() + } + + /// Get the length of the text container in UTF-8. + pub fn len_utf8(&self) -> usize { + self.handler.len_utf8() + } + + /// Get the length of the text container in Unicode. + pub fn len_unicode(&self) -> usize { + self.handler.len_unicode() + } + + /// Get the length of the text container in UTF-16. + pub fn len_utf16(&self) -> usize { + self.handler.len_utf16() + } + + /// Update the current text based on the provided text. + /// + /// It will calculate the minimal difference and apply it to the current text. + /// It uses Myers' diff algorithm to compute the optimal difference. + /// + /// This could take a long time for large texts (e.g. > 50_000 characters). + /// In that case, you should use `updateByLine` instead. + /// + /// # Example + /// ```rust + /// use loro::LoroDoc; + /// + /// let doc = LoroDoc::new(); + /// let text = doc.get_text("text"); + /// text.insert(0, "Hello").unwrap(); + /// text.update("Hello World", Default::default()).unwrap(); + /// assert_eq!(text.to_string(), "Hello World"); + /// ``` + /// + pub fn update(&self, text: &str, options: UpdateOptions) -> Result<(), UpdateTimeoutError> { + self.handler.update(text, options) + } + + /// Update the current text based on the provided text. + /// + /// This update calculation is line-based, which will be more efficient but less precise. + pub fn update_by_line( + &self, + text: &str, + options: UpdateOptions, + ) -> Result<(), UpdateTimeoutError> { + self.handler.update_by_line(text, options) + } + + /// Apply a [delta](https://quilljs.com/docs/delta/) to the text container. + pub fn apply_delta(&self, delta: &[TextDelta]) -> LoroResult<()> { + self.handler.apply_delta(delta) + } + + /// Mark a range of text with a key-value pair. + /// + /// The range uses Unicode scalar indices; use [`mark_utf8`] for UTF-8 byte offsets. + /// + /// You can use it to create a highlight, make a range of text bold, or add a link to a range of text. + /// + /// You can specify the `expand` option to set the behavior when inserting text at the boundary of the range. + /// + /// - `after`(default): when inserting text right after the given range, the mark will be expanded to include the inserted text + /// - `before`: when inserting text right before the given range, the mark will be expanded to include the inserted text + /// - `none`: the mark will not be expanded to include the inserted text at the boundaries + /// - `both`: when inserting text either right before or right after the given range, the mark will be expanded to include the inserted text + /// + /// *You should make sure that a key is always associated with the same expand type.* + pub fn mark( + &self, + range: Range, + key: &str, + value: impl Into, + ) -> LoroResult<()> { + self.handler.mark( + range.start, + range.end, + key, + value.into(), + cursor::PosType::Unicode, + ) + } + + /// Mark a range of text with a key-value pair using UTF-8 byte offsets. + pub fn mark_utf8( + &self, + range: Range, + key: &str, + value: impl Into, + ) -> LoroResult<()> { + self.handler.mark( + range.start, + range.end, + key, + value.into(), + cursor::PosType::Bytes, + ) + } + + /// Mark a range of text with a key-value pair using UTF-16 code unit offsets. + pub fn mark_utf16( + &self, + range: Range, + key: &str, + value: impl Into, + ) -> LoroResult<()> { + self.handler.mark( + range.start, + range.end, + key, + value.into(), + cursor::PosType::Utf16, + ) + } + + /// Unmark a range of text with a key and a value. + /// + /// You can use it to remove highlights, bolds or links + /// + /// You can specify the `expand` option to set the behavior when inserting text at the boundary of the range. + /// + /// **Note: You should specify the same expand type as when you mark the text.** + /// + /// - `after`(default): when inserting text right after the given range, the mark will be expanded to include the inserted text + /// - `before`: when inserting text right before the given range, the mark will be expanded to include the inserted text + /// - `none`: the mark will not be expanded to include the inserted text at the boundaries + /// - `both`: when inserting text either right before or right after the given range, the mark will be expanded to include the inserted text + /// + /// *You should make sure that a key is always associated with the same expand type.* + /// + /// Note: you cannot delete unmergeable annotations like comments by this method. + pub fn unmark(&self, range: Range, key: &str) -> LoroResult<()> { + self.handler + .unmark(range.start, range.end, key, cursor::PosType::Unicode) + } + + /// Unmark a range of text with a key and a value using UTF-16 code unit offsets. + pub fn unmark_utf16(&self, range: Range, key: &str) -> LoroResult<()> { + self.handler + .unmark(range.start, range.end, key, cursor::PosType::Utf16) + } + + /// Get the text in [Delta](https://quilljs.com/docs/delta/) format. + /// + /// # Example + /// ``` + /// use loro::{LoroDoc, ToJson, ExpandType, TextDelta}; + /// use serde_json::json; + /// use rustc_hash::FxHashMap; + /// + /// let doc = LoroDoc::new(); + /// let text = doc.get_text("text"); + /// text.insert(0, "Hello world!").unwrap(); + /// text.mark(0..5, "bold", true).unwrap(); + /// assert_eq!( + /// text.to_delta(), + /// vec![ + /// TextDelta::Insert { + /// insert: "Hello".to_string(), + /// attributes: Some(FxHashMap::from_iter([("bold".to_string(), true.into())])), + /// }, + /// TextDelta::Insert { + /// insert: " world!".to_string(), + /// attributes: None, + /// }, + /// ] + /// ); + /// text.unmark(3..5, "bold").unwrap(); + /// assert_eq!( + /// text.to_delta(), + /// vec![ + /// TextDelta::Insert { + /// insert: "Hel".to_string(), + /// attributes: Some(FxHashMap::from_iter([("bold".to_string(), true.into())])), + /// }, + /// TextDelta::Insert { + /// insert: "lo world!".to_string(), + /// attributes: None, + /// }, + /// ] + /// ); + /// ``` + pub fn to_delta(&self) -> Vec { + let delta = self.handler.get_richtext_value().into_list().unwrap(); + delta + .iter() + .map(|x| { + let map = x.as_map().unwrap(); + let insert = map.get("insert").unwrap().as_string().unwrap().to_string(); + let attributes = map + .get("attributes") + .map(|v| v.as_map().unwrap().deref().clone()); + TextDelta::Insert { insert, attributes } + }) + .collect() + } + + /// Get the rich text value in [Delta](https://quilljs.com/docs/delta/) format. + /// + /// # Example + /// ``` + /// # use loro::{LoroDoc, ToJson, ExpandType, TextDelta}; + /// # use serde_json::json; + /// + /// let doc = LoroDoc::new(); + /// let text = doc.get_text("text"); + /// text.insert(0, "Hello world!").unwrap(); + /// text.mark(0..5, "bold", true).unwrap(); + /// assert_eq!( + /// text.get_richtext_value().to_json_value(), + /// json!([ + /// { "insert": "Hello", "attributes": {"bold": true} }, + /// { "insert": " world!" }, + /// ]) + /// ); + /// text.unmark(3..5, "bold").unwrap(); + /// assert_eq!( + /// text.get_richtext_value().to_json_value(), + /// json!([ + /// { "insert": "Hel", "attributes": {"bold": true} }, + /// { "insert": "lo world!" }, + /// ]) + /// ); + /// ``` + pub fn get_richtext_value(&self) -> LoroValue { + self.handler.get_richtext_value() + } + + /// Get the text content of the text container. + #[allow(clippy::inherent_to_string)] + pub fn to_string(&self) -> String { + self.handler.to_string() + } + + /// Get the cursor at the given position in the given Unicode position. + /// + /// Using "index" to denote cursor positions can be unstable, as positions may + /// shift with document edits. To reliably represent a position or range within + /// a document, it is more effective to leverage the unique ID of each item/character + /// in a List CRDT or Text CRDT. + /// + /// Loro optimizes State metadata by not storing the IDs of deleted elements. This + /// approach complicates tracking cursors since they rely on these IDs. The solution + /// recalculates position by replaying relevant history to update stable positions + /// accurately. To minimize the performance impact of history replay, the system + /// updates cursor info to reference only the IDs of currently present elements, + /// thereby reducing the need for replay. + /// + /// # Example + /// + /// ``` + /// # use loro::{LoroDoc, ToJson}; + /// let doc = LoroDoc::new(); + /// let text = &doc.get_text("text"); + /// text.insert(0, "01234").unwrap(); + /// let pos = text.get_cursor(5, Default::default()).unwrap(); + /// assert_eq!(doc.get_cursor_pos(&pos).unwrap().current.pos, 5); + /// text.insert(0, "01234").unwrap(); + /// assert_eq!(doc.get_cursor_pos(&pos).unwrap().current.pos, 10); + /// text.delete(0, 10).unwrap(); + /// assert_eq!(doc.get_cursor_pos(&pos).unwrap().current.pos, 0); + /// text.insert(0, "01234").unwrap(); + /// assert_eq!(doc.get_cursor_pos(&pos).unwrap().current.pos, 5); + /// ``` + pub fn get_cursor(&self, pos: usize, side: Side) -> Option { + self.handler.get_cursor(pos, side) + } + + /// Whether the text container is deleted. + pub fn is_deleted(&self) -> bool { + self.handler.is_deleted() + } + + /// Convert a position between coordinate systems (Unicode, UTF-16, UTF-8 bytes, Event). + /// + /// Returns `None` when the position is out of bounds or the conversion isn't supported. + pub fn convert_pos( + &self, + index: usize, + from: cursor::PosType, + to: cursor::PosType, + ) -> Option { + self.handler.convert_pos(index, from, to) + } + + /// Push a string to the end of the text container. + pub fn push_str(&self, s: &str) -> LoroResult<()> { + self.handler.push_str(s) + } + + /// Get the editor of the text at the given position. + /// + /// Returns `None` if the position is out of bounds or attribution is unavailable. + /// + /// # Example + /// ``` + /// use loro::LoroDoc; + /// let doc = LoroDoc::new(); + /// let t = doc.get_text("t"); + /// t.insert(0, "hi").unwrap(); + /// let who = t.get_editor_at_unicode_pos(0); + /// assert!(who.is_some()); + /// ``` + pub fn get_editor_at_unicode_pos(&self, pos: usize) -> Option { + self.handler + .get_cursor(pos, Side::Middle) + .and_then(|x| x.id.map(|id| id.peer)) + } +} + +impl Default for LoroText { + fn default() -> Self { + Self::new() + } +} + +/// LoroTree container. It's used to model movable trees. +/// +/// You may use it to model directories, outline or other movable hierarchical data. +/// +/// Learn more at https://loro.dev/docs/tutorial/tree +#[derive(Clone, Debug)] +pub struct LoroTree { + handler: InnerTreeHandler, +} + +impl SealedTrait for LoroTree {} +impl ContainerTrait for LoroTree { + type Handler = InnerTreeHandler; + + fn id(&self) -> ContainerID { + self.handler.id() + } + + fn to_container(&self) -> Container { + Container::Tree(self.clone()) + } + + fn to_handler(&self) -> Self::Handler { + self.handler.clone() + } + + fn from_handler(handler: Self::Handler) -> Self { + Self { handler } + } + + fn is_attached(&self) -> bool { + self.handler.is_attached() + } + + fn get_attached(&self) -> Option { + self.handler.get_attached().map(Self::from_handler) + } + + fn try_from_container(container: Container) -> Option { + container.into_tree().ok() + } + + fn is_deleted(&self) -> bool { + self.handler.is_deleted() + } + fn doc(&self) -> Option { + self.handler.doc().map(|doc| { + doc.start_auto_commit(); + LoroDoc::_new(doc) + }) + } +} + +/// A tree node in the [LoroTree]. +#[derive(Debug, Clone)] +pub struct TreeNode { + /// ID of the tree node. + pub id: TreeID, + /// ID of the parent tree node. + /// If the node is deleted this value is TreeParentId::Deleted. + /// If you checkout to a version before the node is created, this value is TreeParentId::Unexist. + pub parent: TreeParentId, + /// Fraction index of the node + pub fractional_index: FractionalIndex, + /// The current index of the node in its parent's children list. + pub index: usize, +} + +impl LoroTree { + /// Create a new container that is detached from the document. + /// + /// The edits on a detached container will not be persisted. + /// To attach the container to the document, please insert it into an attached container. + pub fn new() -> Self { + Self { + handler: InnerTreeHandler::new_detached(), + } + } + + /// Whether the container is attached to a document + /// + /// The edits on a detached container will not be persisted. + /// To attach the container to the document, please insert it into an attached container. + pub fn is_attached(&self) -> bool { + self.handler.is_attached() + } + + /// Create a new tree node and return the [`TreeID`]. + /// + /// If the `parent` is `None`, the created node is the root of a tree. + /// Otherwise, the created node is a child of the parent tree node. + /// + /// # Example + /// + /// ```rust + /// use loro::LoroDoc; + /// + /// let doc = LoroDoc::new(); + /// let tree = doc.get_tree("tree"); + /// // create a root + /// let root = tree.create(None).unwrap(); + /// // create a new child + /// let child = tree.create(root).unwrap(); + /// ``` + pub fn create>(&self, parent: T) -> LoroResult { + self.handler.create(parent.into()) + } + + /// Get the root nodes of the forest. + pub fn roots(&self) -> Vec { + self.handler.roots() + } + + /// Create a new tree node at the given index and return the [`TreeID`]. + /// + /// If the `parent` is `None`, the created node is the root of a tree. + /// If the `index` is greater than the number of children of the parent, error will be returned. + /// + /// # Example + /// + /// ```rust + /// use loro::LoroDoc; + /// + /// let doc = LoroDoc::new(); + /// let tree = doc.get_tree("tree"); + /// // enable generate fractional index + /// tree.enable_fractional_index(0); + /// // create a root + /// let root = tree.create(None).unwrap(); + /// // create a new child at index 0 + /// let child = tree.create_at(root, 0).unwrap(); + /// ``` + pub fn create_at>(&self, parent: T, index: usize) -> LoroResult { + if !self.handler.is_fractional_index_enabled() { + return Err(LoroTreeError::FractionalIndexNotEnabled.into()); + } + self.handler.create_at(parent.into(), index) + } + + /// Move the `target` node to be a child of the `parent` node. + /// + /// If the `parent` is `None`, the `target` node will be a root. + /// + /// # Example + /// + /// ```rust + /// use loro::LoroDoc; + /// + /// let doc = LoroDoc::new(); + /// let tree = doc.get_tree("tree"); + /// let root = tree.create(None).unwrap(); + /// let root2 = tree.create(None).unwrap(); + /// // move `root2` to be a child of `root`. + /// tree.mov(root2, root).unwrap(); + /// ``` + pub fn mov>(&self, target: TreeID, parent: T) -> LoroResult<()> { + self.handler.mov(target, parent.into()) + } + + /// Move the `target` node to be a child of the `parent` node at the given index. + /// If the `parent` is `None`, the `target` node will be a root. + /// + /// # Example + /// + /// ```rust + /// use loro::LoroDoc; + /// + /// let doc = LoroDoc::new(); + /// let tree = doc.get_tree("tree"); + /// // enable generate fractional index + /// tree.enable_fractional_index(0); + /// let root = tree.create(None).unwrap(); + /// let root2 = tree.create(None).unwrap(); + /// // move `root2` to be a child of `root` at index 0. + /// tree.mov_to(root2, root, 0).unwrap(); + /// ``` + pub fn mov_to>( + &self, + target: TreeID, + parent: T, + to: usize, + ) -> LoroResult<()> { + if !self.handler.is_fractional_index_enabled() { + return Err(LoroTreeError::FractionalIndexNotEnabled.into()); + } + self.handler.move_to(target, parent.into(), to) + } + + /// Move the `target` node to be a child after the `after` node with the same parent. + /// + /// # Example + /// + /// ```rust + /// use loro::LoroDoc; + /// + /// let doc = LoroDoc::new(); + /// let tree = doc.get_tree("tree"); + /// // enable generate fractional index + /// tree.enable_fractional_index(0); + /// let root = tree.create(None).unwrap(); + /// let root2 = tree.create(None).unwrap(); + /// // move `root` to be a child after `root2`. + /// tree.mov_after(root, root2).unwrap(); + /// ``` + pub fn mov_after(&self, target: TreeID, after: TreeID) -> LoroResult<()> { + if !self.handler.is_fractional_index_enabled() { + return Err(LoroTreeError::FractionalIndexNotEnabled.into()); + } + self.handler.mov_after(target, after) + } + + /// Move the `target` node to be a child before the `before` node with the same parent. + /// + /// # Example + /// + /// ```rust + /// use loro::LoroDoc; + /// + /// let doc = LoroDoc::new(); + /// let tree = doc.get_tree("tree"); + /// // enable generate fractional index + /// tree.enable_fractional_index(0); + /// let root = tree.create(None).unwrap(); + /// let root2 = tree.create(None).unwrap(); + /// // move `root` to be a child before `root2`. + /// tree.mov_before(root, root2).unwrap(); + /// ``` + pub fn mov_before(&self, target: TreeID, before: TreeID) -> LoroResult<()> { + if !self.handler.is_fractional_index_enabled() { + return Err(LoroTreeError::FractionalIndexNotEnabled.into()); + } + self.handler.mov_before(target, before) + } + + /// Delete a tree node. + /// + /// Note: If the deleted node has children, the children do not appear in the state + /// rather than actually being deleted. + /// + /// # Example + /// + /// ```rust + /// use loro::LoroDoc; + /// + /// let doc = LoroDoc::new(); + /// let tree = doc.get_tree("tree"); + /// let root = tree.create(None).unwrap(); + /// tree.delete(root).unwrap(); + /// ``` + pub fn delete(&self, target: TreeID) -> LoroResult<()> { + self.handler.delete(target) + } + + /// Get the associated metadata map handler of a tree node. + /// + /// # Example + /// ```rust + /// use loro::LoroDoc; + /// + /// let doc = LoroDoc::new(); + /// let tree = doc.get_tree("tree"); + /// let root = tree.create(None).unwrap(); + /// let root_meta = tree.get_meta(root).unwrap(); + /// root_meta.insert("color", "red"); + /// ``` + pub fn get_meta(&self, target: TreeID) -> LoroResult { + self.handler + .get_meta(target) + .map(|h| LoroMap { handler: h }) + } + + /// Return the parent of target node. + /// + /// - If the target node does not exist, return `None`. + /// - If the target node is a root node, return `Some(None)`. + pub fn parent(&self, target: TreeID) -> Option { + self.handler.get_node_parent(&target) + } + + /// Return whether target node exists. including deleted node. + pub fn contains(&self, target: TreeID) -> bool { + self.handler.contains(target) + } + + /// Return whether target node is deleted. + /// + /// # Errors + /// + /// - If the target node does not exist, return `LoroTreeError::TreeNodeNotExist`. + pub fn is_node_deleted(&self, target: &TreeID) -> LoroResult { + self.handler.is_node_deleted(target) + } + + /// Return all nodes, including deleted nodes + pub fn nodes(&self) -> Vec { + self.handler.nodes() + } + + /// Return all nodes, if `with_deleted` is true, the deleted nodes will be included. + pub fn get_nodes(&self, with_deleted: bool) -> Vec { + let mut ans = self.handler.get_nodes_under(TreeParentId::Root); + if with_deleted { + ans.extend(self.handler.get_nodes_under(TreeParentId::Deleted)); + } + ans.into_iter() + .map(|x| TreeNode { + id: x.id, + parent: x.parent, + fractional_index: x.fractional_index, + index: x.index, + }) + .collect() + } + + /// Return all children of the target node. + /// + /// If the parent node does not exist, return `None`. + pub fn children>(&self, parent: T) -> Option> { + self.handler.children(&parent.into()) + } + + /// Return the number of children of the target node. + pub fn children_num>(&self, parent: T) -> Option { + let parent: TreeParentId = parent.into(); + self.handler.children_num(&parent) + } + + /// Return the fractional index of the target node with hex format. + pub fn fractional_index(&self, target: TreeID) -> Option { + self.handler + .get_position_by_tree_id(&target) + .map(|x| x.to_string()) + } + + /// Return the hierarchy array of the forest. + /// + /// Note: the metadata will be not resolved. So if you don't only care about hierarchy + /// but also the metadata, you should use [TreeHandler::get_value_with_meta()]. + pub fn get_value(&self) -> LoroValue { + self.handler.get_value() + } + + /// Return the hierarchy array of the forest, each node is with metadata. + pub fn get_value_with_meta(&self) -> LoroValue { + self.handler.get_deep_value() + } + + // This method is used for testing only. + #[doc(hidden)] + #[allow(non_snake_case)] + pub fn __internal__next_tree_id(&self) -> TreeID { + self.handler.__internal__next_tree_id() + } + + /// Whether the fractional index is enabled. + pub fn is_fractional_index_enabled(&self) -> bool { + self.handler.is_fractional_index_enabled() + } + + /// Enable fractional index for Tree Position. + /// + /// The jitter is used to avoid conflicts when multiple users are creating the node at the same position. + /// value 0 is default, which means no jitter, any value larger than 0 will enable jitter. + /// + /// Generally speaking, jitter will affect the growth rate of document size. + /// [Read more about it](https://www.loro.dev/blog/movable-tree#implementation-and-encoding-size) + #[inline] + pub fn enable_fractional_index(&self, jitter: u8) { + self.handler.enable_fractional_index(jitter); + } + + /// Disable the fractional index generation when you don't need the Tree's siblings to be sorted. + /// The fractional index will always be set to the same default value 0. + /// + /// After calling this, you cannot use `tree.moveTo()`, `tree.moveBefore()`, `tree.moveAfter()`, + /// and `tree.createAt()`. + #[inline] + pub fn disable_fractional_index(&self) { + self.handler.disable_fractional_index(); + } + + /// Whether the tree is empty. + /// + #[inline] + pub fn is_empty(&self) -> bool { + self.handler.is_empty() + } + + /// Get the last move id of the target node. + pub fn get_last_move_id(&self, target: &TreeID) -> Option { + self.handler.get_last_move_id(target) + } +} + +impl Default for LoroTree { + fn default() -> Self { + Self::new() + } +} + +/// [LoroMovableList container](https://loro.dev/docs/tutorial/list) +/// +/// It is used to model movable ordered lists. +/// +/// Using a combination of insert and delete operations, one can simulate set and move +/// operations on a List. However, this approach fails in concurrent editing scenarios. +/// For example, if the same element is set or moved concurrently, the simulation would +/// result in the deletion of the original element and the insertion of two new elements, +/// which does not meet expectations. +#[derive(Clone, Debug)] +pub struct LoroMovableList { + handler: InnerMovableListHandler, +} + +impl SealedTrait for LoroMovableList {} +impl ContainerTrait for LoroMovableList { + type Handler = InnerMovableListHandler; + + fn id(&self) -> ContainerID { + self.handler.id() + } + + fn to_container(&self) -> Container { + Container::MovableList(self.clone()) + } + + fn to_handler(&self) -> Self::Handler { + self.handler.clone() + } + + fn from_handler(handler: Self::Handler) -> Self { + Self { handler } + } + + fn try_from_container(container: Container) -> Option + where + Self: Sized, + { + match container { + Container::MovableList(x) => Some(x), + _ => None, + } + } + + fn is_attached(&self) -> bool { + self.handler.is_attached() + } + + fn get_attached(&self) -> Option + where + Self: Sized, + { + self.handler.get_attached().map(Self::from_handler) + } + + fn is_deleted(&self) -> bool { + self.handler.is_deleted() + } + + fn doc(&self) -> Option { + self.handler.doc().map(|doc| { + doc.start_auto_commit(); + LoroDoc::_new(doc) + }) + } +} + +impl LoroMovableList { + /// Create a new container that is detached from the document. + /// + /// The edits on a detached container will not be persisted. + /// To attach the container to the document, please insert it into an attached container. + pub fn new() -> LoroMovableList { + Self { + handler: InnerMovableListHandler::new_detached(), + } + } + + /// Whether the container is attached to a document + /// + /// The edits on a detached container will not be persisted. + /// To attach the container to the document, please insert it into an attached container. + pub fn is_attached(&self) -> bool { + self.handler.is_attached() + } + + /// Insert a value at the given position. + pub fn insert(&self, pos: usize, v: impl Into) -> LoroResult<()> { + self.handler.insert(pos, v) + } + + /// Delete the value at the given position. + pub fn delete(&self, pos: usize, len: usize) -> LoroResult<()> { + self.handler.delete(pos, len) + } + + /// Get the value at the given position. + pub fn get(&self, index: usize) -> Option { + self.handler.get_(index).map(ValueOrContainer::from) + } + + /// Get the length of the list. + pub fn len(&self) -> usize { + self.handler.len() + } + + /// Whether the list is empty. + #[must_use] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Get the shallow value of the list. + /// + /// It will not convert the state of sub-containers, but represent them as [LoroValue::Container]. + pub fn get_value(&self) -> LoroValue { + self.handler.get_value() + } + + /// Get the deep value of the list. + /// + /// It will convert the state of sub-containers into a nested JSON value. + pub fn get_deep_value(&self) -> LoroValue { + self.handler.get_deep_value() + } + + /// Pop the last element of the list. + pub fn pop(&self) -> LoroResult> { + let ans = self.handler.pop_()?.map(ValueOrContainer::from); + Ok(ans) + } + + /// Push a value to the end of the list. + pub fn push(&self, v: impl Into) -> LoroResult<()> { + self.handler.push(v.into()) + } + + /// Push a container to the end of the list. + pub fn push_container(&self, child: C) -> LoroResult { + let pos = self.handler.len(); + Ok(C::from_handler( + self.handler.insert_container(pos, child.to_handler())?, + )) + } + + /// Set the value at the given position. + /// + /// # Example + /// ``` + /// use loro::{LoroDoc, ToJson}; + /// use serde_json::json; + /// let doc = LoroDoc::new(); + /// let ml = doc.get_movable_list("ml"); + /// ml.insert(0, "a").unwrap(); + /// ml.set(0, "b").unwrap(); + /// assert_eq!(ml.get_deep_value().to_json_value(), json!(["b"])); + /// ``` + pub fn set(&self, pos: usize, value: impl Into) -> LoroResult<()> { + self.handler.set(pos, value.into()) + } + + /// Move the value at the given position to the given position. + /// + /// # Example + /// ``` + /// use loro::{LoroDoc, ToJson}; + /// use serde_json::json; + /// let doc = LoroDoc::new(); + /// let ml = doc.get_movable_list("ml"); + /// ml.insert(0, "a").unwrap(); + /// ml.insert(1, "b").unwrap(); + /// ml.insert(2, "c").unwrap(); + /// ml.mov(0, 2).unwrap(); + /// assert_eq!(ml.get_deep_value().to_json_value(), json!(["b","c","a"])); + /// ``` + pub fn mov(&self, from: usize, to: usize) -> LoroResult<()> { + self.handler.mov(from, to) + } + + /// Insert a container at the given position. + pub fn insert_container(&self, pos: usize, child: C) -> LoroResult { + Ok(C::from_handler( + self.handler.insert_container(pos, child.to_handler())?, + )) + } + + /// Set the container at the given position. + pub fn set_container(&self, pos: usize, child: C) -> LoroResult { + Ok(C::from_handler( + self.handler.set_container(pos, child.to_handler())?, + )) + } + + /// Log the internal state of the list. + pub fn log_internal_state(&self) { + info!( + "movable_list internal state: {}", + self.handler.log_internal_state() + ) + } + + /// Get the cursor at the given position. + /// + /// Using "index" to denote cursor positions can be unstable, as positions may + /// shift with document edits. To reliably represent a position or range within + /// a document, it is more effective to leverage the unique ID of each item/character + /// in a List CRDT or Text CRDT. + /// + /// Loro optimizes State metadata by not storing the IDs of deleted elements. This + /// approach complicates tracking cursors since they rely on these IDs. The solution + /// recalculates position by replaying relevant history to update stable positions + /// accurately. To minimize the performance impact of history replay, the system + /// updates cursor info to reference only the IDs of currently present elements, + /// thereby reducing the need for replay. + /// + /// # Example + /// + /// ``` + /// use loro::LoroDoc; + /// use loro_internal::cursor::Side; + /// + /// let doc = LoroDoc::new(); + /// let list = doc.get_movable_list("list"); + /// list.insert(0, 0).unwrap(); + /// let cursor = list.get_cursor(0, Side::Middle).unwrap(); + /// assert_eq!(doc.get_cursor_pos(&cursor).unwrap().current.pos, 0); + /// list.insert(0, 0).unwrap(); + /// assert_eq!(doc.get_cursor_pos(&cursor).unwrap().current.pos, 1); + /// list.insert(0, 0).unwrap(); + /// list.insert(0, 0).unwrap(); + /// assert_eq!(doc.get_cursor_pos(&cursor).unwrap().current.pos, 3); + /// list.insert(4, 0).unwrap(); + /// assert_eq!(doc.get_cursor_pos(&cursor).unwrap().current.pos, 3); + /// ``` + pub fn get_cursor(&self, pos: usize, side: Side) -> Option { + self.handler.get_cursor(pos, side) + } + + /// Get the elements of the list as a vector of LoroValues. + /// + /// This method returns a vector containing all the elements in the list as LoroValues. + /// It provides a convenient way to access the entire contents of the LoroMovableList + /// as a standard Rust vector. + /// + /// # Returns + /// + /// A `Vec` containing all elements of the list. + /// + /// # Example + /// + /// ``` + /// use loro::LoroDoc; + /// + /// let doc = LoroDoc::new(); + /// let list = doc.get_movable_list("mylist"); + /// list.insert(0, 1).unwrap(); + /// list.insert(1, "hello").unwrap(); + /// list.insert(2, true).unwrap(); + /// + /// let vec = list.to_vec(); + /// assert_eq!(vec.len(), 3); + /// assert_eq!(vec[0], 1.into()); + /// assert_eq!(vec[1], "hello".into()); + /// assert_eq!(vec[2], true.into()); + /// ``` + pub fn to_vec(&self) -> Vec { + self.get_value().into_list().unwrap().unwrap() + } + + /// Delete all elements in the list. + pub fn clear(&self) -> LoroResult<()> { + self.handler.clear() + } + + /// Iterate over the elements of the list. + pub fn for_each(&self, mut f: I) + where + I: FnMut(ValueOrContainer), + { + self.handler.for_each(&mut |v| { + f(ValueOrContainer::from(v)); + }) + } + + /// Get the creator of the list item at the given position. + pub fn get_creator_at(&self, pos: usize) -> Option { + self.handler.get_creator_at(pos) + } + + /// Get the last mover of the list item at the given position. + pub fn get_last_mover_at(&self, pos: usize) -> Option { + self.handler.get_last_mover_at(pos) + } + + /// Get the last editor of the list item at the given position. + pub fn get_last_editor_at(&self, pos: usize) -> Option { + self.handler.get_last_editor_at(pos) + } +} + +impl Default for LoroMovableList { + fn default() -> Self { + Self::new() + } +} + +/// Unknown container. +#[derive(Clone, Debug)] +pub struct LoroUnknown { + handler: InnerUnknownHandler, +} + +impl SealedTrait for LoroUnknown {} +impl ContainerTrait for LoroUnknown { + type Handler = InnerUnknownHandler; + + fn id(&self) -> ContainerID { + self.handler.id() + } + + fn to_container(&self) -> Container { + Container::Unknown(self.clone()) + } + + fn to_handler(&self) -> Self::Handler { + self.handler.clone() + } + + fn from_handler(handler: Self::Handler) -> Self { + Self { handler } + } + + fn try_from_container(container: Container) -> Option + where + Self: Sized, + { + match container { + Container::Unknown(x) => Some(x), + _ => None, + } + } + + fn is_attached(&self) -> bool { + self.handler.is_attached() + } + + fn get_attached(&self) -> Option + where + Self: Sized, + { + self.handler.get_attached().map(Self::from_handler) + } + + fn is_deleted(&self) -> bool { + self.handler.is_deleted() + } + + fn doc(&self) -> Option { + self.handler.doc().map(|doc| { + doc.start_auto_commit(); + LoroDoc::_new(doc) + }) + } +} + +use enum_as_inner::EnumAsInner; +#[cfg(feature = "jsonpath")] +use loro_internal::jsonpath::jsonpath_impl::JsonPathError; + +/// All the CRDT containers supported by Loro. +#[derive(Clone, Debug, EnumAsInner)] +pub enum Container { + /// [LoroList container](https://loro.dev/docs/tutorial/list) + List(LoroList), + /// [LoroMap container](https://loro.dev/docs/tutorial/map) + Map(LoroMap), + /// [LoroText container](https://loro.dev/docs/tutorial/text) + Text(LoroText), + /// [LoroTree container] + Tree(LoroTree), + /// [LoroMovableList container](https://loro.dev/docs/tutorial/list) + MovableList(LoroMovableList), + #[cfg(feature = "counter")] + /// [LoroCounter container] + Counter(counter::LoroCounter), + /// Unknown container + Unknown(LoroUnknown), +} + +impl SealedTrait for Container {} +impl ContainerTrait for Container { + type Handler = loro_internal::handler::Handler; + + fn id(&self) -> ContainerID { + match self { + Container::List(x) => x.id(), + Container::Map(x) => x.id(), + Container::Text(x) => x.id(), + Container::Tree(x) => x.id(), + Container::MovableList(x) => x.id(), + #[cfg(feature = "counter")] + Container::Counter(x) => x.id(), + Container::Unknown(x) => x.id(), + } + } + + fn to_container(&self) -> Container { + self.clone() + } + + fn to_handler(&self) -> Self::Handler { + match self { + Container::List(x) => Self::Handler::List(x.to_handler()), + Container::Map(x) => Self::Handler::Map(x.to_handler()), + Container::Text(x) => Self::Handler::Text(x.to_handler()), + Container::Tree(x) => Self::Handler::Tree(x.to_handler()), + Container::MovableList(x) => Self::Handler::MovableList(x.to_handler()), + #[cfg(feature = "counter")] + Container::Counter(x) => Self::Handler::Counter(x.to_handler()), + Container::Unknown(x) => Self::Handler::Unknown(x.to_handler()), + } + } + + fn from_handler(handler: Self::Handler) -> Self { + match handler { + InnerHandler::Text(x) => Container::Text(LoroText { handler: x }), + InnerHandler::Map(x) => Container::Map(LoroMap { handler: x }), + InnerHandler::List(x) => Container::List(LoroList { handler: x }), + InnerHandler::MovableList(x) => Container::MovableList(LoroMovableList { handler: x }), + InnerHandler::Tree(x) => Container::Tree(LoroTree { handler: x }), + #[cfg(feature = "counter")] + InnerHandler::Counter(x) => Container::Counter(counter::LoroCounter { handler: x }), + InnerHandler::Unknown(x) => Container::Unknown(LoroUnknown { handler: x }), + } + } + + fn is_attached(&self) -> bool { + match self { + Container::List(x) => x.is_attached(), + Container::Map(x) => x.is_attached(), + Container::Text(x) => x.is_attached(), + Container::Tree(x) => x.is_attached(), + Container::MovableList(x) => x.is_attached(), + #[cfg(feature = "counter")] + Container::Counter(x) => x.is_attached(), + Container::Unknown(x) => x.is_attached(), + } + } + + fn get_attached(&self) -> Option { + match self { + Container::List(x) => x.get_attached().map(Container::List), + Container::MovableList(x) => x.get_attached().map(Container::MovableList), + Container::Map(x) => x.get_attached().map(Container::Map), + Container::Text(x) => x.get_attached().map(Container::Text), + Container::Tree(x) => x.get_attached().map(Container::Tree), + #[cfg(feature = "counter")] + Container::Counter(x) => x.get_attached().map(Container::Counter), + Container::Unknown(x) => x.get_attached().map(Container::Unknown), + } + } + + fn try_from_container(container: Container) -> Option + where + Self: Sized, + { + Some(container) + } + + fn is_deleted(&self) -> bool { + match self { + Container::List(x) => x.is_deleted(), + Container::Map(x) => x.is_deleted(), + Container::Text(x) => x.is_deleted(), + Container::Tree(x) => x.is_deleted(), + Container::MovableList(x) => x.is_deleted(), + #[cfg(feature = "counter")] + Container::Counter(x) => x.is_deleted(), + Container::Unknown(x) => x.is_deleted(), + } + } + fn doc(&self) -> Option { + match self { + Container::List(x) => x.doc(), + Container::Map(x) => x.doc(), + Container::Text(x) => x.doc(), + Container::Tree(x) => x.doc(), + Container::MovableList(x) => x.doc(), + #[cfg(feature = "counter")] + Container::Counter(x) => x.doc(), + Container::Unknown(x) => x.doc(), + } + } +} + +impl Container { + /// Create a detached container of the given type. + /// + /// A detached container is a container that is not attached to a document. + /// The edits on a detached container will not be persisted. + /// To attach the container to the document, please insert it into an attached container. + pub fn new(kind: ContainerType) -> Self { + match kind { + ContainerType::List => Container::List(LoroList::new()), + ContainerType::MovableList => Container::MovableList(LoroMovableList::new()), + ContainerType::Map => Container::Map(LoroMap::new()), + ContainerType::Text => Container::Text(LoroText::new()), + ContainerType::Tree => Container::Tree(LoroTree::new()), + #[cfg(feature = "counter")] + ContainerType::Counter => Container::Counter(counter::LoroCounter::new()), + ContainerType::Unknown(_) => { + panic!("Cannot create a detached container of type Unknown"); + } + } + } + + /// Get the type of the container. + pub fn get_type(&self) -> ContainerType { + match self { + Container::List(_) => ContainerType::List, + Container::MovableList(_) => ContainerType::MovableList, + Container::Map(_) => ContainerType::Map, + Container::Text(_) => ContainerType::Text, + Container::Tree(_) => ContainerType::Tree, + #[cfg(feature = "counter")] + Container::Counter(_) => ContainerType::Counter, + Container::Unknown(x) => x.handler.id().container_type(), + } + } +} + +impl From for Container { + fn from(value: InnerHandler) -> Self { + match value { + InnerHandler::Text(x) => Container::Text(LoroText { handler: x }), + InnerHandler::Map(x) => Container::Map(LoroMap { handler: x }), + InnerHandler::List(x) => Container::List(LoroList { handler: x }), + InnerHandler::Tree(x) => Container::Tree(LoroTree { handler: x }), + InnerHandler::MovableList(x) => Container::MovableList(LoroMovableList { handler: x }), + #[cfg(feature = "counter")] + InnerHandler::Counter(x) => Container::Counter(counter::LoroCounter { handler: x }), + InnerHandler::Unknown(x) => Container::Unknown(LoroUnknown { handler: x }), + } + } +} + +/// It's a type that can be either a value or a container. +#[derive(Debug, Clone, EnumAsInner)] +pub enum ValueOrContainer { + /// A value. + Value(LoroValue), + /// A container. + Container(Container), +} + +impl ValueOrContainer { + /// Get the deep value of the value or container. + pub fn get_deep_value(&self) -> LoroValue { + match self { + ValueOrContainer::Value(v) => v.clone(), + ValueOrContainer::Container(c) => match c { + Container::List(c) => c.get_deep_value(), + Container::Map(c) => c.get_deep_value(), + Container::Text(c) => c.to_string().into(), + Container::Tree(c) => c.get_value(), + Container::MovableList(c) => c.get_deep_value(), + #[cfg(feature = "counter")] + Container::Counter(c) => c.get_value().into(), + Container::Unknown(_) => LoroValue::Null, + }, + } + } + + pub(crate) fn into_value_or_handler(self) -> ValueOrHandler { + match self { + ValueOrContainer::Value(v) => ValueOrHandler::Value(v), + ValueOrContainer::Container(c) => ValueOrHandler::Handler(c.to_handler()), + } + } +} + +/// UndoManager can be used to undo and redo the changes made to the document with a certain peer. +/// +/// Notes & pitfalls: +/// - Local-only: undo/redo affects only local operations from the bound peer; it does not revert +/// remote edits. For global rollback, use time travel (`checkout`/`revert_to`). +/// - Peer identity: keep the `peer_id` stable while an `UndoManager` is in use. Changing peer IDs +/// can disrupt undo grouping/semantics. +/// - Grouping: you may want to tune the merge interval and exclude origins to group related edits. +#[derive(Debug)] +#[repr(transparent)] +pub struct UndoManager(InnerUndoManager); + +impl UndoManager { + /// Create a new UndoManager. + pub fn new(doc: &LoroDoc) -> Self { + let inner = InnerUndoManager::new(&doc.doc); + inner.set_max_undo_steps(100); + Self(inner) + } + + /// Undo the last change made by the peer. + pub fn undo(&mut self) -> LoroResult { + self.0.undo() + } + + /// Redo the last change made by the peer. + pub fn redo(&mut self) -> LoroResult { + self.0.redo() + } + + /// Record a new checkpoint. + pub fn record_new_checkpoint(&mut self) -> LoroResult<()> { + self.0.record_new_checkpoint() + } + + /// Whether the undo manager can undo. + pub fn can_undo(&self) -> bool { + self.0.can_undo() + } + + /// Whether the undo manager can redo. + pub fn can_redo(&self) -> bool { + self.0.can_redo() + } + + /// How many times the undo manager can undo. + pub fn undo_count(&self) -> usize { + self.0.undo_count() + } + + /// How many times the undo manager can redo. + pub fn redo_count(&self) -> usize { + self.0.redo_count() + } + + /// If a local event's origin matches the given prefix, it will not be recorded in the + /// undo stack. + pub fn add_exclude_origin_prefix(&mut self, prefix: &str) { + self.0.add_exclude_origin_prefix(prefix) + } + + /// Set the maximum number of undo steps. The default value is 100. + pub fn set_max_undo_steps(&mut self, size: usize) { + self.0.set_max_undo_steps(size) + } + + /// Set the merge interval in ms. The default value is 0, which means no merge. + pub fn set_merge_interval(&mut self, interval: i64) { + self.0.set_merge_interval(interval) + } + + /// Set the listener for push events. + /// The listener will be called when a new undo/redo item is pushed into the stack. + pub fn set_on_push(&mut self, on_push: Option) { + if let Some(on_push) = on_push { + self.0.set_on_push(Some(Box::new(move |u, c, e| { + on_push(u, c, e.map(|x| x.into())) + }))); + } else { + self.0.set_on_push(None); + } + } + + /// Set the listener for pop events. + /// The listener will be called when an undo/redo item is popped from the stack. + pub fn set_on_pop(&mut self, on_pop: Option) { + self.0.set_on_pop(on_pop); + } + + /// Clear the undo stack and the redo stack + pub fn clear(&self) { + self.0.clear(); + } + + /// Will start a new group of changes, all subsequent changes will be merged + /// into a new item on the undo stack. If we receive remote changes, we determine + /// wether or not they are conflicting. If the remote changes are conflicting + /// we split the undo item and close the group. If there are no conflict + /// in changed container ids we continue the group merge. + pub fn group_start(&mut self) -> LoroResult<()> { + self.0.group_start() + } + + /// Ends the current group, calling UndoManager::undo() after this will + /// undo all changes that occurred during the group. + pub fn group_end(&mut self) { + self.0.group_end(); + } + + /// Get the peer id of the undo manager. + pub fn peer(&self) -> PeerID { + self.0.peer() + } + + /// Get the metadata of the top undo stack item, if any. + pub fn top_undo_meta(&self) -> Option { + self.0.top_undo_meta() + } + + /// Get the metadata of the top redo stack item, if any. + pub fn top_redo_meta(&self) -> Option { + self.0.top_redo_meta() + } + + /// Get the value associated with the top undo stack item, if any. + pub fn top_undo_value(&self) -> Option { + self.0.top_undo_value() + } + + /// Get the value associated with the top redo stack item, if any. + pub fn top_redo_value(&self) -> Option { + self.0.top_redo_value() + } +} +/// When a undo/redo item is pushed, the undo manager will call the on_push callback to get the meta data of the undo item. +/// The returned cursors will be recorded for a new pushed undo item. +pub type OnPush = + Box Fn(UndoOrRedo, CounterSpan, Option) -> UndoItemMeta + Send + Sync>; diff --git a/3rdparty/loro/crates/rle/Cargo.toml b/3rdparty/loro/crates/rle/Cargo.toml new file mode 100644 index 0000000000..a37a4f1818 --- /dev/null +++ b/3rdparty/loro/crates/rle/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "loro-rle" +version = "1.6.0" +edition = "2021" +license = "MIT" +description = "A internal library of loro for storing and manipulating run-length encoded data. Do not use it directly." +documentation = "https://docs.rs/loro/" +homepage = "https://loro.dev" +repository = "https://github.com/loro-dev/loro/" +authors = ["Zixuan Chen", "Liang Zhao"] +categories = ["data-structures"] +keywords = ["crdt", "local-first"] + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +smallvec = {workspace=true} +num = "0.4.0" +append-only-bytes = { version = "0.1.11", features = ["u32_range"] } + +[dev-dependencies] +ctor = "0.1.23" +proptest = "1.0.0" +smartstring = "1.0.1" +rand = "0.8.5" +static_assertions = "1.1.0" + +[features] +test_utils = [] +wasm = [] diff --git a/3rdparty/loro/crates/rle/LICENSE b/3rdparty/loro/crates/rle/LICENSE new file mode 100644 index 0000000000..3cf271d30f --- /dev/null +++ b/3rdparty/loro/crates/rle/LICENSE @@ -0,0 +1,21 @@ +MIT License Copyright (c) 2023 Zixuan Chen + +Permission is hereby granted, free of +charge, to any person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the Software without +restriction, including without limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to the +following conditions: + +The above copyright notice and this permission notice +(including the next paragraph) shall be included in all copies or substantial +portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO +EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. \ No newline at end of file diff --git a/3rdparty/loro/crates/rle/src/lib.rs b/3rdparty/loro/crates/rle/src/lib.rs new file mode 100644 index 0000000000..f91a9b3286 --- /dev/null +++ b/3rdparty/loro/crates/rle/src/lib.rs @@ -0,0 +1,95 @@ +//! Run length encoding library. +//! +//! There are many mergeable types. By merging them together we can get a more compact representation of the data. +//! For example, in many cases, `[0..5, 5..10]` can be merged into `0..10`. +//! +//! # RleVec +//! +//! RleVec is a vector that can be compressed using run-length encoding. +//! +//! A T value may be merged with its neighbors. When we push new element, the new value +//! may be merged with the last element in the array. Each value has a length, so there +//! are two types of indexes: +//! 1. (merged) It refers to the index of the merged element. +//! 2. (atom) The index of substantial elements. It refers to the index of the atom element. +//! +//! By default, we use atom index in RleVec. +//! - len() returns the number of atom elements in the array. +//! - get(index) returns the atom element at the index. +//! - slice(from, to) returns a slice of atom elements from the index from to the index to. +//! +//! +#![deny(clippy::undocumented_unsafe_blocks)] +mod rle_trait; +mod rle_vec; +pub use crate::rle_trait::{ + HasIndex, HasLength, Mergable, Rle, RleCollection, RlePush, Slice, Sliceable, ZeroElement, +}; +pub use crate::rle_vec::{slice_vec_by, RleVec, RleVecWithLen}; +pub mod rle_impl; + +use num::Integer; + +#[derive(Clone)] +pub struct SearchResult<'a, T, I: Integer> { + pub element: &'a T, + pub merged_index: usize, + pub offset: I, +} + +pub struct SliceIterator<'a, T> { + vec: &'a [T], + cur_index: usize, + cur_offset: usize, + end_index: Option, + end_offset: Option, +} + +impl SliceIterator<'_, T> { + fn new_empty() -> Self { + Self { + vec: &[], + cur_index: 0, + cur_offset: 0, + end_index: None, + end_offset: None, + } + } +} + +impl<'a, T: HasLength> Iterator for SliceIterator<'a, T> { + type Item = Slice<'a, T>; + + fn next(&mut self) -> Option { + if self.vec.is_empty() { + return None; + } + + let end_index = self.end_index.unwrap_or(self.vec.len() - 1); + if self.cur_index == end_index { + let elem = &self.vec[self.cur_index]; + let end = self.end_offset.unwrap_or_else(|| elem.atom_len()); + if self.cur_offset == end { + return None; + } + + let ans = Slice { + value: elem, + start: self.cur_offset, + end, + }; + self.cur_offset = end; + return Some(ans); + } + + let ans = Slice { + value: &self.vec[self.cur_index], + start: self.cur_offset, + end: self.vec[self.cur_index].atom_len(), + }; + + self.cur_index += 1; + self.cur_offset = 0; + Some(ans) + } +} diff --git a/3rdparty/loro/crates/rle/src/rle_impl.rs b/3rdparty/loro/crates/rle/src/rle_impl.rs new file mode 100644 index 0000000000..e6173f5cbd --- /dev/null +++ b/3rdparty/loro/crates/rle/src/rle_impl.rs @@ -0,0 +1,114 @@ +use std::ops::Range; + +use crate::{rle_trait::GlobalIndex, HasIndex, HasLength, Mergable, Sliceable}; +use append_only_bytes::{AppendOnlyBytes, BytesSlice}; +use num::{cast, Integer, NumCast}; +use smallvec::{Array, SmallVec}; + +impl Sliceable for bool { + fn slice(&self, _: usize, _: usize) -> Self { + *self + } +} + +impl Sliceable for Range { + fn slice(&self, start: usize, end: usize) -> Self { + self.start + cast(start).unwrap()..self.start + cast(end).unwrap() + } +} + +impl + Copy> Mergable for Range { + fn is_mergable(&self, other: &Self, _: &()) -> bool { + other.start <= self.end && other.start >= self.start + } + + fn merge(&mut self, other: &Self, _conf: &()) + where + Self: Sized, + { + self.end = other.end; + } +} + +impl HasLength for Range { + fn content_len(&self) -> usize { + cast(self.end - self.start).unwrap() + } +} + +impl HasIndex for Range { + type Int = T; + + fn get_start_index(&self) -> Self::Int { + self.start + } +} + +/// this can make iter return type has len +impl HasLength for (A, T) { + fn content_len(&self) -> usize { + self.1.content_len() + } +} + +/// this can make iter return type has len +impl HasLength for &T { + fn content_len(&self) -> usize { + (*self).content_len() + } +} + +impl> Sliceable for SmallVec { + fn slice(&self, from: usize, to: usize) -> Self { + let mut index = 0; + let mut ans: SmallVec = smallvec::smallvec![]; + if to == from { + return ans; + } + + for item in self.iter() { + if index < to && from < index + item.atom_len() { + let start = from.saturating_sub(index); + ans.push(item.slice(start, item.atom_len().min(to - index))); + } + + index += item.atom_len(); + } + + ans + } +} + +impl HasLength for AppendOnlyBytes { + fn content_len(&self) -> usize { + self.len() + } +} + +impl HasLength for BytesSlice { + fn content_len(&self) -> usize { + self.len() + } +} + +impl Sliceable for BytesSlice { + fn slice(&self, from: usize, to: usize) -> Self { + self.slice_clone(from..to) + } +} + +impl Mergable for BytesSlice { + fn is_mergable(&self, other: &Self, _conf: &()) -> bool + where + Self: Sized, + { + self.can_merge(other) + } + + fn merge(&mut self, other: &Self, _conf: &()) + where + Self: Sized, + { + self.try_merge(other).unwrap() + } +} diff --git a/3rdparty/loro/crates/rle/src/rle_trait.rs b/3rdparty/loro/crates/rle/src/rle_trait.rs new file mode 100644 index 0000000000..ba60e7e2fc --- /dev/null +++ b/3rdparty/loro/crates/rle/src/rle_trait.rs @@ -0,0 +1,215 @@ +use std::fmt::Debug; + +use num::{traits::AsPrimitive, FromPrimitive, Integer}; +use smallvec::{Array, SmallVec}; + +use crate::SearchResult; + +pub trait Mergable { + fn is_mergable(&self, _other: &Self, _conf: &Cfg) -> bool + where + Self: Sized, + { + false + } + + fn merge(&mut self, _other: &Self, _conf: &Cfg) + where + Self: Sized, + { + unreachable!() + } +} + +/// NOTE: [Sliceable] implementation should be coherent with [Mergable]: +/// +/// - For all k, a.slice(0,k).merge(a.slice(k, a.len())) == a +pub trait Sliceable { + fn slice(&self, from: usize, to: usize) -> Self; +} + +#[derive(Debug, Clone, Copy)] +pub struct Slice<'a, T> { + pub value: &'a T, + pub start: usize, + pub end: usize, +} + +impl Slice<'_, T> { + pub fn into_inner(&self) -> T { + self.value.slice(self.start, self.end) + } +} + +#[allow(clippy::len_without_is_empty)] +pub trait HasLength { + /// It is the length of the content, i.e. the length when no [Mergable::merge] ever happen. + /// + /// However, when the content is deleted, [HasLength::content_len] is expected to be zero in some [crate::RleTree] use cases + fn content_len(&self) -> usize; + + /// It is the length of the atom element underneath, i.e. the length when no [Mergable::merge] ever happen. + /// + /// It is the same as [HasLength::atom_len] in the most of the cases. + /// However, oppose to [HasLength::atom_len], when the content is deleted, [HasLength::content_len] should not change + fn atom_len(&self) -> usize { + self.content_len() + } +} + +pub trait Rle: HasLength + Sliceable + Mergable + Debug + Clone {} + +pub trait ZeroElement { + fn zero_element() -> Self; +} + +impl ZeroElement for T { + fn zero_element() -> Self { + Default::default() + } +} + +impl + Debug + Clone, Cfg> Rle for T {} + +pub trait HasIndex: HasLength { + type Int: GlobalIndex; + fn get_start_index(&self) -> Self::Int; + + #[inline] + fn get_end_index(&self) -> Self::Int { + self.get_start_index() + Self::Int::from_usize(self.atom_len()).unwrap() + } +} + +pub trait GlobalIndex: + Debug + Integer + Copy + Default + FromPrimitive + AsPrimitive +{ +} + +impl> GlobalIndex for T {} + +impl HasLength for String { + fn content_len(&self) -> usize { + self.len() + } +} + +impl HasLength for Vec { + fn content_len(&self) -> usize { + self.len() + } +} + +impl Sliceable for Vec { + fn slice(&self, from: usize, to: usize) -> Self { + self[from..to].to_vec() + } +} + +impl Sliceable for String { + fn slice(&self, from: usize, to: usize) -> Self { + self[from..to].to_string() + } +} + +pub trait RlePush { + fn push_rle_element(&mut self, element: T); +} + +pub trait RleCollection { + fn start(&self) -> ::Int; + fn end(&self) -> ::Int; + fn sum_atom_len(&self) -> ::Int; + fn search_atom_index(&self, index: ::Int) -> usize; + fn get_by_atom_index( + &self, + index: ::Int, + ) -> Option::Int>>; +} + +impl RlePush for Vec { + fn push_rle_element(&mut self, element: T) { + match self.last_mut() { + Some(last) if last.is_mergable(&element, &()) => { + last.merge(&element, &()); + } + _ => { + self.push(element); + } + } + } +} + +impl RleCollection for Vec { + fn search_atom_index(&self, index: ::Int) -> usize { + let mut start = 0; + let mut end = self.len() - 1; + while start < end { + let mid = (start + end) / 2; + match self[mid].get_start_index().cmp(&index) { + std::cmp::Ordering::Equal => { + start = mid; + break; + } + std::cmp::Ordering::Less => { + start = mid + 1; + } + std::cmp::Ordering::Greater => { + end = mid; + } + } + } + + if index < self[start].get_start_index() { + start -= 1; + } + start + } + + fn get_by_atom_index( + &self, + index: ::Int, + ) -> Option::Int>> { + if index > self.end() { + return None; + } + + let merged_index = self.search_atom_index(index); + let value = &self[merged_index]; + Some(SearchResult { + merged_index, + element: value, + offset: index - self[merged_index].get_start_index(), + }) + } + + fn start(&self) -> ::Int { + self.first() + .map(|x| x.get_start_index()) + .unwrap_or_default() + } + + fn end(&self) -> ::Int { + self.last().map(|x| x.get_end_index()).unwrap_or_default() + } + + fn sum_atom_len(&self) -> ::Int { + self.end() - self.start() + } +} + +impl RlePush for SmallVec +where + A::Item: Mergable, +{ + fn push_rle_element(&mut self, element: A::Item) { + match self.last_mut() { + Some(last) if last.is_mergable(&element, &()) => { + last.merge(&element, &()); + } + _ => { + self.push(element); + } + } + } +} diff --git a/3rdparty/loro/crates/rle/src/rle_vec.rs b/3rdparty/loro/crates/rle/src/rle_vec.rs new file mode 100644 index 0000000000..e464300ea9 --- /dev/null +++ b/3rdparty/loro/crates/rle/src/rle_vec.rs @@ -0,0 +1,681 @@ +use std::{ + fmt::Debug, + marker::PhantomData, + ops::{Deref, Index, Range}, +}; + +use num::{traits::AsPrimitive, FromPrimitive}; +use smallvec::{Array, SmallVec}; + +use crate::{rle_trait::HasIndex, HasLength, Mergable, SearchResult, SliceIterator, Sliceable}; + +/// RleVec is a vector that can be compressed using run-length encoding. +/// +/// A T value may be merged with its neighbors. When we push new element, the new value +/// may be merged with the last element in the array. Each value has a length, so there +/// are two types of indexes: +/// 1. (merged) It refers to the index of the merged element. +/// 2. (atom) The index of substantial elements. It refers to the index of the atom element. +/// +/// By default, we use atom index in RleVec. +/// - len() returns the number of atom elements in the array. +/// - get(index) returns the atom element at the index. +/// - slice(from, to) returns a slice of atom elements from the index from to the index to. +pub struct RleVec { + _p: PhantomData A::Item>, + vec: SmallVec, +} + +pub struct RleVecWithLen { + vec: RleVec, + atom_len: usize, +} + +impl RleVecWithLen +where + A::Item: HasLength + Mergable, +{ + pub fn push(&mut self, value: A::Item) { + self.atom_len += value.atom_len(); + self.vec.push(value); + } + + pub fn new() -> Self { + Self { + vec: Default::default(), + atom_len: 0, + } + } + + pub fn with_capacity(size: usize) -> Self { + Self { + vec: RleVec::with_capacity(size), + atom_len: 0, + } + } + + pub fn merged_len(&self) -> usize { + self.vec.merged_len() + } + + pub fn iter(&self) -> std::slice::Iter<'_, A::Item> { + self.vec.iter() + } + + /// # Safety + /// + /// should not change the element's length during iter + pub unsafe fn iter_mut(&mut self) -> std::slice::IterMut<'_, A::Item> { + self.vec.iter_mut() + } + + pub fn reverse(&mut self) { + self.vec.reverse() + } + + pub fn check(&self) { + assert_eq!( + self.atom_len, + self.vec.iter().map(|x| x.atom_len()).sum::() + ); + } +} + +impl Default for RleVecWithLen +where + A::Item: HasLength + Mergable, +{ + fn default() -> Self { + Self::new() + } +} + +impl RleVecWithLen +where + A::Item: HasLength + Mergable + HasIndex, +{ + pub fn get( + &self, + index: ::Int, + ) -> Option::Int>> { + self.vec.get_by_atom_index(index) + } + + pub fn iter_by_index( + &self, + from: ::Int, + to: ::Int, + ) -> SliceIterator<'_, A::Item> { + self.vec.iter_by_index(from, to) + } +} + +impl HasLength for RleVecWithLen +where + A::Item: HasLength + Mergable, +{ + fn content_len(&self) -> usize { + self.atom_len + } +} + +impl RleVec { + #[inline] + pub fn is_empty(&self) -> bool { + self.vec.is_empty() + } + + #[inline] + pub fn new() -> Self { + RleVec { + vec: SmallVec::new(), + _p: PhantomData, + } + } + + #[inline] + pub fn with_capacity(size: usize) -> Self { + RleVec { + vec: SmallVec::with_capacity(size), + _p: PhantomData, + } + } + + #[inline] + pub fn capacity(&self) -> usize { + self.vec.capacity() + } + + /// this is the length of merged elements + pub fn len(&self) -> usize { + self.vec.len() + } + + pub fn reverse(&mut self) { + self.vec.reverse() + } + + pub fn clear(&mut self) { + self.vec.clear() + } +} + +impl IntoIterator for RleVec { + type Item = A::Item; + + type IntoIter = smallvec::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.vec.into_iter() + } +} + +impl Debug for RleVecWithLen +where + A::Item: Debug, +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RleVecWithLen") + .field("vec", &self.vec) + .field("atom_len", &self.atom_len) + .finish() + } +} + +impl Clone for RleVecWithLen +where + A::Item: Clone, +{ + fn clone(&self) -> Self { + Self { + vec: self.vec.clone(), + atom_len: self.atom_len, + } + } +} + +impl Clone for RleVec +where + A::Item: Clone, +{ + fn clone(&self) -> Self { + Self { + vec: self.vec.clone(), + _p: PhantomData, + } + } +} +impl Debug for RleVec +where + A::Item: Debug, +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RleVec").field("vec", &self.vec).finish() + } +} + +impl Index for RleVec { + type Output = A::Item; + + fn index(&self, index: usize) -> &Self::Output { + &self.vec[index] + } +} + +impl Index for RleVecWithLen { + type Output = A::Item; + + fn index(&self, index: usize) -> &Self::Output { + &self.vec[index] + } +} + +impl PartialEq for RleVec +where + A::Item: PartialEq, +{ + fn eq(&self, other: &Self) -> bool { + self.vec == other.vec + } +} + +impl Eq for RleVec where A::Item: Eq + PartialEq {} +impl PartialEq for RleVecWithLen +where + A::Item: Eq + PartialEq, +{ + fn eq(&self, other: &Self) -> bool { + self.vec == other.vec + } +} + +impl Eq for RleVecWithLen where A::Item: Eq + PartialEq {} + +impl RleVec +where + A::Item: Mergable + HasLength, +{ + /// push a new element to the end of the array. It may be merged with last element. + /// + /// Return whether merged + pub fn push(&mut self, value: A::Item) -> bool { + if let Some(last) = self.vec.last_mut() { + if last.is_mergable(&value, &()) { + last.merge(&value, &()); + return true; + } + } + + self.vec.push(value); + false + } +} +impl RleVec +where + A::Item: Mergable + HasLength + HasIndex, +{ + /// return end - start + pub fn span(&self) -> ::Int { + match (self.vec.first(), self.vec.last()) { + (Some(first), Some(last)) => last.get_end_index() - first.get_start_index(), + _ => <::Int as FromPrimitive>::from_usize(0).unwrap(), + } + } + + pub fn iter_by_index( + &self, + from: ::Int, + to: ::Int, + ) -> SliceIterator<'_, A::Item> { + if from == to { + return SliceIterator::new_empty(); + } + + let start = self.get_by_atom_index(from); + if start.is_none() { + return SliceIterator::new_empty(); + } + + let start = start.unwrap(); + let end = self.get_by_atom_index(to); + if let Some(end) = end { + SliceIterator { + vec: &self.vec, + cur_index: start.merged_index, + cur_offset: start.offset.as_(), + end_index: Some(end.merged_index), + end_offset: Some(end.offset.as_()), + } + } else { + SliceIterator { + vec: &self.vec, + cur_index: start.merged_index, + cur_offset: start.offset.as_(), + end_index: None, + end_offset: None, + } + } + } + + #[inline] + pub fn end(&self) -> ::Int { + self.vec + .last() + .map(|x| x.get_end_index()) + .unwrap_or_else(|| { + <::Int as FromPrimitive>::from_usize(0).unwrap() + }) + } + + /// Get the element at the given atom index. + /// + /// If the index is larger than the end index, return None. + /// Otherwise, return: Some((element, merged_index, offset)) + pub fn get_by_atom_index( + &self, + index: ::Int, + ) -> Option::Int>> { + if index > self.end() { + return None; + } + + let merged_index = self.search_atom_index(index); + let value = &self.vec[merged_index]; + Some(SearchResult { + merged_index, + element: value, + offset: index - self[merged_index].get_start_index(), + }) + } + + pub fn search_atom_index(&self, index: <::Item as HasIndex>::Int) -> usize { + let mut start = 0; + let mut end = self.vec.len() - 1; + while start < end { + let mid = (start + end) / 2; + match self[mid].get_start_index().cmp(&index) { + std::cmp::Ordering::Equal => { + start = mid; + break; + } + std::cmp::Ordering::Less => { + start = mid + 1; + } + std::cmp::Ordering::Greater => { + end = mid; + } + } + } + + if index < self[start].get_start_index() { + start -= 1; + } + start + } + + pub fn slice_iter( + &self, + from: ::Int, + to: ::Int, + ) -> SliceIterator<'_, A::Item> { + if from == to || self.merged_len() == 0 { + return SliceIterator::new_empty(); + } + + let from_result = self.get_by_atom_index(from); + if from_result.is_none() { + return SliceIterator::new_empty(); + } + + let from_result = from_result.unwrap(); + let to_result = if to == self.atom_len() { + None + } else { + self.get_by_atom_index(to) + }; + if let Some(to_result) = to_result { + SliceIterator { + vec: &self.vec, + cur_index: from_result.merged_index, + cur_offset: from_result.offset.as_(), + end_index: Some(to_result.merged_index), + end_offset: Some(to_result.offset.as_()), + } + } else { + SliceIterator { + vec: &self.vec, + cur_index: from_result.merged_index, + cur_offset: from_result.offset.as_(), + end_index: None, + end_offset: None, + } + } + } + + #[inline] + pub fn slice_merged(&self, range: Range) -> &[A::Item] { + &self.vec[range] + } + + pub fn atom_len(&self) -> ::Int { + self.vec + .last() + .map(|x| x.get_end_index() - self.vec.first().unwrap().get_start_index()) + .unwrap_or(::Int::from_usize(0).unwrap()) + } +} +impl RleVec +where + A::Item: Mergable + HasLength + HasIndex + Sliceable, +{ + /// This is different from [Sliceable::slice]. + /// This slice method is based on each element's [HasIndex]. + /// [Sliceable::slice] is based on the accumulated length of each element + pub fn slice_by_index( + &self, + from: ::Int, + to: ::Int, + ) -> Self { + self.iter_by_index(from, to) + .map(|x| x.value.slice(x.start, x.end)) + .collect() + } +} + +impl From> for RleVec +where + A::Item: Mergable + HasLength, +{ + fn from(vec: Vec) -> Self { + let mut ans: RleVec = RleVec::with_capacity(vec.len()); + for v in vec { + ans.push(v); + } + ans.vec.shrink_to_fit(); + ans + } +} + +impl From<&[A::Item]> for RleVec +where + A::Item: Mergable + HasLength + Clone, +{ + fn from(value: &[A::Item]) -> Self { + let mut ans: RleVec = RleVec::with_capacity(value.len()); + for v in value.iter() { + ans.push(v.clone()); + } + ans.vec.shrink_to_fit(); + ans + } +} + +impl From> for RleVec { + fn from(value: SmallVec) -> Self { + RleVec { + vec: value, + _p: PhantomData, + } + } +} + +impl From> for SmallVec { + fn from(value: RleVec) -> Self { + value.vec + } +} + +impl RleVec { + #[inline(always)] + pub fn merged_len(&self) -> usize { + self.vec.len() + } + + #[inline(always)] + pub fn vec(&self) -> &SmallVec { + &self.vec + } + + #[inline(always)] + pub fn vec_mut(&mut self) -> &mut SmallVec { + &mut self.vec + } + + #[inline(always)] + pub fn iter(&self) -> std::slice::Iter<'_, A::Item> { + self.vec.iter() + } + + #[inline(always)] + pub fn iter_mut(&mut self) -> std::slice::IterMut<'_, A::Item> { + self.vec.iter_mut() + } + + #[inline(always)] + pub fn get_merged(&self, index: usize) -> Option<&A::Item> { + self.vec.get(index) + } +} + +impl Default for RleVec { + fn default() -> Self { + Self::new() + } +} + +impl FromIterator for RleVec +where + A::Item: Mergable + HasLength, +{ + fn from_iter>(iter: I) -> Self { + let mut vec = RleVec::new(); + for item in iter { + vec.push(item); + } + vec + } +} + +impl Mergable for RleVec +where + A::Item: Clone + Mergable + HasLength + Sliceable, +{ + fn is_mergable(&self, other: &Self, _: &()) -> bool { + self.vec.len() + other.vec.len() < self.capacity() + } + + fn merge(&mut self, other: &Self, _: &()) { + for item in other.vec.iter() { + self.push(item.clone()); + } + } +} + +impl Sliceable for RleVec +where + A::Item: Mergable + HasLength + Sliceable, +{ + fn slice(&self, start: usize, end: usize) -> Self { + if start >= end { + return Self::new(); + } + + let mut ans = SmallVec::new(); + let mut index = 0; + for i in 0..self.vec.len() { + if index >= end { + break; + } + + let len = self[i].atom_len(); + if start < index + len { + ans.push(self[i].slice(start.saturating_sub(index), (end - index).min(len))) + } + + index += len; + } + + Self { + vec: ans, + _p: PhantomData, + } + } +} + +impl Mergable for RleVecWithLen +where + A::Item: Clone + Mergable + HasLength + Sliceable, +{ + fn is_mergable(&self, other: &Self, _: &()) -> bool { + self.vec.is_mergable(&other.vec, &()) + } + + fn merge(&mut self, other: &Self, _: &()) { + for item in other.vec.iter() { + self.push(item.clone()); + } + } +} + +impl Sliceable for RleVecWithLen +where + A::Item: Mergable + HasLength + Sliceable, +{ + fn slice(&self, start: usize, end: usize) -> Self { + Self { + vec: self.vec.slice(start, end), + atom_len: end - start, + } + } +} + +impl Deref for RleVec { + type Target = [A::Item]; + + fn deref(&self) -> &Self::Target { + &self.vec + } +} + +impl Deref for RleVecWithLen { + type Target = RleVec; + + fn deref(&self) -> &Self::Target { + &self.vec + } +} + +pub fn slice_vec_by(vec: &[T], index: F, start: usize, end: usize) -> Vec +where + F: Fn(&T) -> usize, + T: Sliceable + HasLength, +{ + if start >= end || vec.is_empty() { + return Vec::new(); + } + + let start = start - index(&vec[0]); + let end = end - index(&vec[0]); + let mut ans = Vec::new(); + let mut index = 0; + for item in vec.iter() { + if index >= end { + break; + } + + let len = item.atom_len(); + if start < index + len { + ans.push(item.slice(start.saturating_sub(index), (end - index).min(len))) + } + + index += len; + } + + ans +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + #[allow(clippy::single_range_in_vec_init)] + fn slice() { + let mut a: RleVec<[Range; 4]> = RleVec::new(); + a.push(0..5); + a.push(5..8); + a.push(10..13); + assert_eq!(&*a.slice(0, 5), &vec![0..5]); + assert_eq!(&*a.slice(5, 10), &vec![5..8, 10..12]); + assert_eq!(&*a.slice(5, 5), &vec![]); + + let ans = a.slice_by_index(3, 11); + assert_eq!(&*ans, &vec![3..8, 10..11]); + let ans = a.slice_by_index(3, 100); + assert_eq!(&*ans, &vec![3..8, 10..13]); + assert_eq!(*a.last().unwrap(), 10..13); + for k in a.iter() { + println!("{k:?}"); + } + } +} diff --git a/3rdparty/loro/rust-toolchain b/3rdparty/loro/rust-toolchain new file mode 100644 index 0000000000..2bf5ad0447 --- /dev/null +++ b/3rdparty/loro/rust-toolchain @@ -0,0 +1 @@ +stable diff --git a/3rdparty/mogan-loro-ffi/Cargo.lock b/3rdparty/mogan-loro-ffi/Cargo.lock new file mode 100644 index 0000000000..f9d0d0c757 --- /dev/null +++ b/3rdparty/mogan-loro-ffi/Cargo.lock @@ -0,0 +1,1433 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "getrandom 0.3.4", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "append-only-bytes" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac436601d6bdde674a0d7fb593e829ffe7b3387c351b356dd20e2d40f5bf3ee5" + +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + +[[package]] +name = "arref" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ccd462b64c3c72f1be8305905a85d85403d768e8690c9b8bd3b9009a5761679" + +[[package]] +name = "atomic-polyfill" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cf2bce30dfe09ef0bfaef228b9d414faaf7e563035494d7fe092dba54b300f4" +dependencies = [ + "critical-section", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "bitmaps" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031043d04099746d8db04daf1fa424b2bc8bd69d92b25962dcde24da39ab64a2" +dependencies = [ + "typenum", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cc" +version = "1.2.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cobs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1" +dependencies = [ + "thiserror 2.0.18", +] + +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.119", +] + +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "diff" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" + +[[package]] +name = "either" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" + +[[package]] +name = "embedded-io" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" + +[[package]] +name = "embedded-io" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" + +[[package]] +name = "ensure-cov" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33753185802e107b8fa907192af1f0eca13b1fb33327a59266d650fef29b2b4e" + +[[package]] +name = "enum-as-inner" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9720bba047d567ffc8a3cba48bf19126600e249ab7f128e9233e6376976a116" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "enum-as-inner" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "enum_dispatch" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa18ce2bc66555b3218614519ac839ddb759a7d6720732f979ef8d13be147ecd" +dependencies = [ + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generator" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3b854b0e584ead1a33f18b2fcad7cf7be18b3875c78816b753639aa501513ae" +dependencies = [ + "cc", + "cfg-if", + "libc", + "log", + "rustversion", + "windows-link", + "windows-result", +] + +[[package]] +name = "generic-btree" +version = "0.10.7" +dependencies = [ + "arref", + "heapless 0.9.3", + "itertools 0.11.0", + "loro-thunderdome", + "proc-macro2", + "rustc-hash", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", +] + +[[package]] +name = "hash32" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c35f58762feb77d74ebe43bdbc3210f09be9fe6742234d573bacc26ed92b67" +dependencies = [ + "byteorder", +] + +[[package]] +name = "hash32" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d60b12902ba28e2730cd37e95b8c9223af2808df9e902d4df49588d1470606" +dependencies = [ + "byteorder", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" + +[[package]] +name = "heapless" +version = "0.7.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdc6457c0eb62c71aac4bc17216026d8410337c4126773b9c5daba343f17964f" +dependencies = [ + "atomic-polyfill", + "hash32 0.2.1", + "rustc_version", + "serde", + "spin", + "stable_deref_trait", +] + +[[package]] +name = "heapless" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bfb9eb618601c89945a70e254898da93b13be0388091d42117462b265bb3fad" +dependencies = [ + "hash32 0.3.1", + "stable_deref_trait", +] + +[[package]] +name = "heapless" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ba4bd83f9415b58b4ed8dc5714c76e626a105be4646c02630ad730ad3b5aa4" +dependencies = [ + "hash32 0.3.1", + "stable_deref_trait", +] + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "im" +version = "15.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0acd33ff0285af998aaf9b57342af478078f53492322fafc47450e09397e0e9" +dependencies = [ + "bitmaps", + "rand_core", + "rand_xoshiro", + "serde", + "sized-chunks", + "typenum", + "version_check", +] + +[[package]] +name = "itertools" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "leb128" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c83bff1d572d6b9aeef67ddfc8448e4a3737909cb28e81f97c791b9018703e52" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "loom" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "serde", + "serde_json", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "loro" +version = "1.13.7" +dependencies = [ + "enum-as-inner 0.6.1", + "generic-btree", + "loro-common", + "loro-delta", + "loro-internal", + "loro-kv-store", + "rustc-hash", + "tracing", +] + +[[package]] +name = "loro-common" +version = "1.13.1" +dependencies = [ + "arbitrary", + "enum-as-inner 0.6.1", + "leb128", + "loro-rle", + "nonmax", + "rustc-hash", + "serde", + "serde_columnar", + "serde_json", + "thiserror 1.0.69", +] + +[[package]] +name = "loro-delta" +version = "1.13.0" +dependencies = [ + "arrayvec", + "enum-as-inner 0.5.1", + "generic-btree", + "heapless 0.8.0", +] + +[[package]] +name = "loro-internal" +version = "1.13.7" +dependencies = [ + "append-only-bytes", + "arref", + "bytes", + "either", + "ensure-cov", + "enum-as-inner 0.6.1", + "enum_dispatch", + "generic-btree", + "getrandom 0.2.17", + "im", + "itertools 0.12.1", + "leb128", + "loom", + "loro-common", + "loro-delta", + "loro-kv-store", + "loro-rle", + "loro_fractional_index", + "md5", + "nonmax", + "num", + "num-traits", + "once_cell", + "parking_lot", + "pest", + "pest_derive", + "postcard", + "pretty_assertions", + "rand", + "rustc-hash", + "serde", + "serde_columnar", + "serde_json", + "smallvec", + "thiserror 1.0.69", + "thread_local", + "tracing", + "wasm-bindgen", + "xxhash-rust", +] + +[[package]] +name = "loro-kv-store" +version = "1.13.7" +dependencies = [ + "bytes", + "ensure-cov", + "loro-common", + "lz4_flex", + "once_cell", + "quick_cache", + "rustc-hash", + "tracing", + "xxhash-rust", +] + +[[package]] +name = "loro-rle" +version = "1.6.0" +dependencies = [ + "append-only-bytes", + "num", + "smallvec", +] + +[[package]] +name = "loro-thunderdome" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f3d053a135388e6b1df14e8af1212af5064746e9b87a06a345a7a779ee9695a" + +[[package]] +name = "loro_fractional_index" +version = "1.13.0" +dependencies = [ + "once_cell", + "rand", + "serde", +] + +[[package]] +name = "lz4_flex" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a" +dependencies = [ + "twox-hash", +] + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "md5" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "mogan_loro_ffi" +version = "0.1.0" +dependencies = [ + "loro", +] + +[[package]] +name = "nonmax" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "610a5acd306ec67f907abe5567859a3c693fb9886eb1f012ab8f2a47bef3db51" + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "pest" +version = "2.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47627dd7305c6a2d6c8c6bcd24c5a4c17dbbf425f4f9c5313e724b38fc9782e9" +dependencies = [ + "memchr", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b4254325ecad416ab689e27ba51da03ba01a9632bc6e108f5fe7c3c4ad29d58" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c4c0e91ead7a8f7acecbca6f003fc2e8282b1dbe2dd9c9d2f16aba42995e0a7" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "pest_meta" +version = "2.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9744bc48116fee06334924bb5f2bad41eed5e89bd26e29b0b799f9a3f82c210" +dependencies = [ + "pest", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "postcard" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24" +dependencies = [ + "cobs", + "embedded-io 0.4.0", + "embedded-io 0.6.1", + "heapless 0.7.17", + "serde", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "pretty_assertions" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d" +dependencies = [ + "diff", + "yansi", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quick_cache" +version = "0.6.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9c6658afe513a3b484e3abfdaa0d03ef3c0bbf017542c178dd55f94eb3051f9" +dependencies = [ + "ahash", + "equivalent", + "hashbrown", + "parking_lot", +] + +[[package]] +name = "quote" +version = "1.0.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_xoshiro" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa" +dependencies = [ + "rand_core", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex-automata" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_columnar" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a16e404f17b16d0273460350e29b02d76ba0d70f34afdc9a4fa034c97d6c6eb" +dependencies = [ + "itertools 0.11.0", + "postcard", + "serde", + "serde_columnar_derive", + "thiserror 1.0.69", +] + +[[package]] +name = "serde_columnar_derive" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45958fce4903f67e871fbf15ac78e289269b21ebd357d6fecacdba233629112e" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "sized-chunks" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16d69225bde7a69b235da73377861095455d298f2b970996eec25ddbb42b3d1e" +dependencies = [ + "bitmaps", + "typenum", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" +dependencies = [ + "serde", +] + +[[package]] +name = "spin" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" +dependencies = [ + "lock_api", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "thread_local" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "twox-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "ucd-trie" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "xxhash-rust" +version = "0.8.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985eec839aaf2a1270af8f4ebcf63cf9401cfd90f0902f97c28d9f104ffbde72" + +[[package]] +name = "yansi" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" + +[[package]] +name = "zerocopy" +version = "0.8.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7cbbc0a705a0fd05cc3676525980d2bf5a9bc4adac6d6475209a7887cf59d19" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/3rdparty/mogan-loro-ffi/Cargo.toml b/3rdparty/mogan-loro-ffi/Cargo.toml new file mode 100644 index 0000000000..f6ed71b405 --- /dev/null +++ b/3rdparty/mogan-loro-ffi/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "mogan_loro_ffi" +version = "0.1.0" +edition = "2021" +description = "Mogan 专用的最小 C FFI shim:把 moebius 文档树 IR 与 Loro CRDT snapshot 互转" +license = "GPL-3.0-or-later" + +[lib] +# staticlib:产出单个 .a 静态库(含所有 Rust 依赖),供 C++ 直接链接 +crate-type = ["staticlib"] + +[dependencies] +# 依赖本地 loro 源码(与本仓库 3rdparty/loro 工作区成员) +loro = { path = "../loro/crates/loro" } + +# 复制 3rdparty/loro 工作区的 patch:loro 的 generic-btree 依赖重定向到其 in-tree fork, +# 否则 crates.io 的版本可能落后导致 loro 编译失败。本 crate 作为独立工作区根,patch +# 在此声明即可对其整个依赖图(含 loro)生效。 +[patch.crates-io] +generic-btree = { path = "../loro/crates/generic-btree" } diff --git a/3rdparty/mogan-loro-ffi/src/lib.rs b/3rdparty/mogan-loro-ffi/src/lib.rs new file mode 100644 index 0000000000..ae64a0da07 --- /dev/null +++ b/3rdparty/mogan-loro-ffi/src/lib.rs @@ -0,0 +1,1110 @@ +//! Mogan ↔ Loro CRDT 的最小 C FFI shim。 +//! +//! Loro 官方只提供 UniFFI 绑定(Swift/Python 等),无现成 C 接口,故本 crate 自行 +//! 暴露一组 `extern "C"` 函数。FFI 边界用**粗粒度字节接口**:C++ 侧把 moebius 文档 +//! 树的中性 IR(`loro_ir_node`)用扁平二进制编码喂进来,本层据此构建 LoroDoc 的 +//! LoroTree(可移动树),导出 snapshot 字节;反向 import snapshot 后遍历 LoroTree +//! 输出同一编码。这样边界只有 2 个转换函数 + 1 个 free,无需 TreeID 跨界。 +//! +//! # IR 扁平编码(小端) +//! ```text +//! node := kind:u8 label_len:u32 label:bytes text_len:u32 text:bytes +//! n_children:u32 node × n_children +//! ``` +//! - `kind`:0 = atomic,1 = compound,2 = generic(与 C++ 侧 loro_ir.hpp 一致) +//! - 节点映射:compound/generic 写 `label`;atomic 写 `text`;`kind` 字段始终写入 +//! +//! 每个节点的元数据存在其关联 LoroMap 上(key: `kind`/`label`/`text`)。 +//! +//! Author: JimZhouZZY & Claude +//! Date : Thu Jul 16, 2026 +//! Copyright (C) 2026 Liii Network +//! +//! This program is free software: you can redistribute it and/or modify +//! it under the terms of the GNU General Public License as published by +//! the Free Software Foundation, either version 3 of the License, or +//! (at your option) any later version. +//! +//! This program is distributed in the hope that it will be useful, +//! but WITHOUT ANY WARRANTY; without even the implied warranty of +//! MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//! GNU General Public License for more details. +//! +//! You should have received a copy of the GNU General Public License +//! along with this program. If not, see . +//! + +use std::collections::HashMap; +use std::sync::{Arc, Mutex, OnceLock}; + +use loro::{ + Container, ExportMode, LoroDoc, LoroMap, LoroText, LoroTree, Subscription, TreeID, + TreeParentId, ValueOrContainer, +}; + +// doc 句柄 -> 其 local-update 订阅。Subscription 必须保活,否则自动取消订阅。 +static LOCAL_SUBS: OnceLock>> = OnceLock::new(); +fn local_subs() -> &'static Mutex> { + LOCAL_SUBS.get_or_init(|| Mutex::new(HashMap::new())) +} + +const KIND_ATOMIC: u8 = 0; +const KIND_COMPOUND: u8 = 1; +const KIND_GENERIC: u8 = 2; + +/// LoroTree 容器名(C++ 侧也用同名 root tree) +const TREE_NAME: &str = "tree"; + +// ============================================================================= +// Rust 侧 IR 节点 +// ============================================================================= + +struct IrNode { + kind: u8, + label: Vec, + text: Vec, + children: Vec, +} + +// ============================================================================= +// 扁平编码:Reader / Writer +// ============================================================================= + +struct Reader<'a> { + buf: &'a [u8], + pos: usize, +} + +impl<'a> Reader<'a> { + fn u8(&mut self) -> Result { + match self.buf.get(self.pos) { + Some(&v) => { + self.pos += 1; + Ok(v) + } + None => Err(()), + } + } + + fn u32(&mut self) -> Result { + if self.pos + 4 > self.buf.len() { + return Err(()); + } + let mut arr = [0u8; 4]; + arr.copy_from_slice(&self.buf[self.pos..self.pos + 4]); + self.pos += 4; + Ok(u32::from_le_bytes(arr)) + } + + fn bytes(&mut self, n: usize) -> Result<&'a [u8], ()> { + if self.pos + n > self.buf.len() { + return Err(()); + } + let s = &self.buf[self.pos..self.pos + n]; + self.pos += n; + Ok(s) + } + + /// 读一个 length-prefixed 字节串(保留原始字节,不做 UTF-8 lossy 转换—— + /// atomic 内容可能是图片等任意二进制) + fn bytes_field(&mut self) -> Result, ()> { + let n = self.u32()? as usize; + let b = self.bytes(n)?; + Ok(b.to_vec()) + } + + fn node(&mut self) -> Result { + let kind = self.u8()?; + let label = self.bytes_field()?; + let text = self.bytes_field()?; + let n = self.u32()? as usize; + let mut children = Vec::with_capacity(n); + for _ in 0..n { + children.push(self.node()?); + } + Ok(IrNode { + kind, + label, + text, + children, + }) + } +} + +struct Writer { + buf: Vec, +} + +impl Writer { + fn u8(&mut self, v: u8) { + self.buf.push(v); + } + + fn u32(&mut self, v: u32) { + self.buf.extend_from_slice(&v.to_le_bytes()); + } + + fn bytes_field(&mut self, b: &[u8]) { + self.u32(b.len() as u32); + self.buf.extend_from_slice(b); + } + + fn node(&mut self, n: &IrNode) { + self.u8(n.kind); + self.bytes_field(&n.label); + self.bytes_field(&n.text); + self.u32(n.children.len() as u32); + for c in &n.children { + self.node(c); + } + } +} + +// ============================================================================= +// IR → LoroDoc +// ============================================================================= + +fn kind_to_str(k: u8) -> &'static str { + match k { + KIND_ATOMIC => "atomic", + KIND_GENERIC => "generic", + _ => "compound", + } +} + +fn build_node(tree: &LoroTree, parent: TreeParentId, ir: &IrNode) -> Result<(), ()> { + let id = tree.create(parent).map_err(|_| ())?; + let meta = tree.get_meta(id).map_err(|_| ())?; + meta.insert("kind", kind_to_str(ir.kind).to_string()) + .map_err(|_| ())?; + match ir.kind { + KIND_ATOMIC => { + // 文本(合法 UTF-8)→ LoroText 子容器(字符级 CRDT,可增量 insert/delete); + // 二进制(如图片,非法 UTF-8)→ Binary 值 + if let Ok(s) = std::str::from_utf8(&ir.text) { + let txt = meta.insert_container("text", LoroText::new()).map_err(|_| ())?; + if !s.is_empty() { + txt.insert(0, s).map_err(|_| ())?; + } + } else { + meta.insert("text", ir.text.clone()).map_err(|_| ())?; + } + } + _ => meta.insert("label", ir.label.clone()).map_err(|_| ())?, + } + for child in &ir.children { + build_node(tree, TreeParentId::Node(id), child)?; + } + Ok(()) +} + +fn doc_from_ir(root: &IrNode) -> Result { + let doc = LoroDoc::new(); + let tree = doc.get_tree(TREE_NAME); + // 启用分数索引,保证子节点顺序在并发与往返中稳定 + tree.enable_fractional_index(0); + // 根 IR 节点作为 LoroTree 的唯一 root + build_node(&tree, TreeParentId::Root, root)?; + Ok(doc) +} + +// ============================================================================= +// LoroDoc → IR +// ============================================================================= + +/// 从节点的元数据 Map 中取字符串字段 +fn get_meta_str(meta: &LoroMap, key: &str) -> Option { + match meta.get(key)? { + ValueOrContainer::Value(v) => { + let arc: Arc = Arc::::try_from(v).ok()?; + Some(arc.as_str().to_string()) + } + _ => None, + } +} + +/// 从节点的元数据 Map 中取二进制字段(label 以 Binary 存储,保留原始字节) +fn get_meta_bytes(meta: &LoroMap, key: &str) -> Option> { + match meta.get(key)? { + ValueOrContainer::Value(v) => { + let arc: Arc> = Arc::>::try_from(v).ok()?; + Some((*arc).clone()) + } + _ => None, + } +} + +/// 取原子节点的 text:文本原子存为 LoroText 子容器,二进制原子存为 Binary 值 +fn get_meta_text_or_binary(meta: &LoroMap) -> Option> { + match meta.get("text")? { + ValueOrContainer::Value(v) => { + let arc: Arc> = Arc::>::try_from(v).ok()?; + Some((*arc).clone()) + } + ValueOrContainer::Container(Container::Text(t)) => { + let mut s = String::new(); + t.iter(|chunk| { + s.push_str(chunk); + true + }); + Some(s.into_bytes()) + } + _ => None, + } +} + +fn str_to_kind(s: &str) -> u8 { + match s { + "atomic" => KIND_ATOMIC, + "generic" => KIND_GENERIC, + _ => KIND_COMPOUND, + } +} + +fn read_node(tree: &LoroTree, id: TreeID) -> Result { + let meta = tree.get_meta(id).map_err(|_| ())?; + let kind = get_meta_str(&meta, "kind").unwrap_or_default(); + let kind_byte = str_to_kind(&kind); + let label = get_meta_bytes(&meta, "label").unwrap_or_default(); + let text = get_meta_text_or_binary(&meta).unwrap_or_default(); + let child_ids = tree.children(TreeParentId::Node(id)).unwrap_or_default(); + let mut children = Vec::with_capacity(child_ids.len()); + for cid in child_ids { + children.push(read_node(tree, cid)?); + } + Ok(IrNode { + kind: kind_byte, + label, + text, + children, + }) +} + +fn doc_to_ir(doc: &LoroDoc) -> Result { + let tree = doc.get_tree(TREE_NAME); + let roots = tree.roots(); + // 单 root:即 moebius 文档树的根 + let root = *roots.get(0).ok_or(())?; + read_node(&tree, root) +} + +// ============================================================================= +// 输出缓冲区管理:Rust 分配,C 侧用 mogan_loro_free 释放 +// ============================================================================= + +/// 把 Vec 泄漏成裸指针交给 C;容量已 shrink 到 == 长度,便于 from_raw_parts 回收 +fn emit_out(data: Vec, out: *mut *mut u8, out_len: *mut usize) { + let mut data = data; + data.shrink_to_fit(); + let len = data.len(); + let p = data.as_mut_ptr(); + std::mem::forget(data); + unsafe { + if !out.is_null() { + *out = p; + } + if !out_len.is_null() { + *out_len = len; + } + } +} + +/// 释放由本 crate 通过 emit_out 分配的缓冲区 +/// +/// # Safety +/// `ptr` 必须是本 crate 返回的、长度为 `len` 的缓冲区 +#[no_mangle] +pub unsafe extern "C" fn mogan_loro_free(ptr: *mut u8, len: usize) { + if ptr.is_null() { + return; + } + drop(Vec::from_raw_parts(ptr, len, len)); +} + +// ============================================================================= +// 对外 C ABI +// ============================================================================= + +/// IR 扁平字节 → Loro snapshot 字节。 +/// +/// 成功返回 0,并把 snapshot 写入 `*out`(长度 `*out_len`,Rust 分配,需 +/// [`mogan_loro_free`] 释放);失败返回负数错误码。 +/// +/// # Safety +/// `ir` 须指向 `ir_len` 字节的有效缓冲区;`out`/`out_len` 可空(空则不写出)。 +#[no_mangle] +pub extern "C" fn mogan_loro_encode( + ir: *const u8, + ir_len: usize, + out: *mut *mut u8, + out_len: *mut usize, +) -> i32 { + if ir.is_null() { + return -1; + } + let buf = unsafe { std::slice::from_raw_parts(ir, ir_len) }; + let mut r = Reader { buf, pos: 0 }; + let root = match r.node() { + Ok(n) => n, + Err(_) => return -2, // IR 解码失败 + }; + let doc = match doc_from_ir(&root) { + Ok(d) => d, + Err(_) => return -3, // LoroTree 构建失败 + }; + doc.commit(); + let snap = match doc.export(ExportMode::Snapshot) { + Ok(b) => b, + Err(_) => return -4, // snapshot 导出失败 + }; + emit_out(snap, out, out_len); + 0 +} + +/// Loro snapshot 字节 → IR 扁平字节。 +/// +/// 成功返回 0,并把 IR 编码写入 `*out`(长度 `*out_len`,Rust 分配,需 +/// [`mogan_loro_free`] 释放);失败返回负数错误码。 +/// +/// # Safety +/// `snap` 须指向 `snap_len` 字节的有效缓冲区;`out`/`out_len` 可空。 +#[no_mangle] +pub extern "C" fn mogan_loro_decode( + snap: *const u8, + snap_len: usize, + out: *mut *mut u8, + out_len: *mut usize, +) -> i32 { + if snap.is_null() { + return -1; + } + let buf = unsafe { std::slice::from_raw_parts(snap, snap_len) }; + let doc = LoroDoc::new(); + if doc.import(buf).is_err() { + return -2; // snapshot 导入失败 + } + doc.commit(); + doc.checkout_to_latest(); + let root = match doc_to_ir(&doc) { + Ok(n) => n, + Err(_) => return -3, // LoroTree 遍历失败 + }; + let mut w = Writer { buf: Vec::new() }; + w.node(&root); + emit_out(w.buf, out, out_len); + 0 +} + +// ============================================================================= +// live-doc API(Phase 2:增量 op 镜像) +// ============================================================================= + +/// TreeID 的 C 表示(与 loro::TreeID {peer:u64, counter:i32} 布局一致,按值跨界) +#[repr(C)] +#[derive(Clone, Copy)] +pub struct MoganTreeId { + pub peer: u64, + pub counter: i32, +} + +/// 表示 Root 父节点的哨兵 peer 值(C++ 传 Root 时用 {MOGAN_ROOT_PEER, 0}) +const MOGAN_ROOT_PEER: u64 = u64::MAX; +/// 失败返回值(真实 TreeID 的 peer 不会是 0) +const NULL_ID: MoganTreeId = MoganTreeId { peer: 0, counter: 0 }; + +fn parent_from(id: MoganTreeId) -> TreeParentId { + if id.peer == MOGAN_ROOT_PEER { + TreeParentId::Root + } else { + TreeParentId::Node(treeid_from(id)) + } +} + +fn treeid_from(id: MoganTreeId) -> TreeID { + TreeID { + peer: id.peer, + counter: id.counter, + } +} + +fn treeid_to(id: TreeID) -> MoganTreeId { + MoganTreeId { + peer: id.peer, + counter: id.counter, + } +} + +/// 新建 live doc(已 enable 分数索引)。返回句柄,用 mogan_loro_doc_free 释放。 +#[no_mangle] +pub extern "C" fn mogan_loro_doc_new() -> *mut LoroDoc { + let doc = LoroDoc::new(); + doc.get_tree(TREE_NAME).enable_fractional_index(0); + Box::into_raw(Box::new(doc)) +} + +/// # Safety: doc 须为 mogan_loro_doc_new 返回的句柄 +#[no_mangle] +pub unsafe extern "C" fn mogan_loro_doc_free(doc: *mut LoroDoc) { + if !doc.is_null() { + // 先摘除 local-update 订阅(保活),再释放 doc + if let Ok(mut subs) = local_subs().lock() { + subs.remove(&(doc as usize)); + } + drop(Box::from_raw(doc)); + } +} + +/// 显式提交 pending 本地 op。auto_commit 是延迟的(debounced),不会同步触发 +/// local-update 事件;事件级增量同步需在镜像后显式 commit 以同步触发回调。 +#[no_mangle] +pub unsafe extern "C" fn mogan_loro_doc_commit(doc: *mut LoroDoc) { + if !doc.is_null() { + (*doc).commit(); + } +} + +/// 订阅本地编辑产生的增量 update:每次本地 op 提交时回调 cb(user_data, bytes, len), +/// bytes 是这次编辑的 delta(仅新增 op,非整 snapshot)。事件级增量同步的核心。 +/// 返回 0 成功。订阅随 doc 释放自动取消。 +#[no_mangle] +pub unsafe extern "C" fn mogan_loro_doc_on_local_update( + doc: *mut LoroDoc, + cb: extern "C" fn(*mut std::ffi::c_void, *const u8, usize), + user_data: *mut std::ffi::c_void, +) -> i32 { + if doc.is_null() { + return -1; + } + // 把裸指针转 usize 捕获(usize 天然 Send+Sync,避开 edition2021 离散捕获 *mut 的 !Send 问题) + let ud = user_data as usize; + let sub = (*doc).subscribe_local_update(Box::new(move |bytes: &Vec| { + cb(ud as *mut std::ffi::c_void, bytes.as_ptr(), bytes.len()); + true // 保持订阅 + })); + local_subs().lock().unwrap().insert(doc as usize, sub); + 0 +} + +/// 用 IR 扁平字节 seed(构建整棵树)。成功返回 0。 +#[no_mangle] +pub unsafe extern "C" fn mogan_loro_doc_seed( + doc: *mut LoroDoc, + ir: *const u8, + ir_len: usize, +) -> i32 { + if doc.is_null() { + return -1; + } + let buf = if ir.is_null() || ir_len == 0 { + &[][..] + } else { + std::slice::from_raw_parts(ir, ir_len) + }; + let mut r = Reader { buf, pos: 0 }; + let root = match r.node() { + Ok(n) => n, + Err(_) => return -2, + }; + let tree = (*doc).get_tree(TREE_NAME); + match build_node(&tree, TreeParentId::Root, &root) { + Ok(()) => 0, + Err(()) => -3, + } +} + +/// 导出 snapshot 字节到 *out/*out_len(Rust 分配,需 mogan_loro_free 释放)。 +#[no_mangle] +pub unsafe extern "C" fn mogan_loro_doc_export( + doc: *mut LoroDoc, + out: *mut *mut u8, + out_len: *mut usize, +) -> i32 { + if doc.is_null() { + return -1; + } + (*doc).commit(); + match (*doc).export(ExportMode::Snapshot) { + Ok(b) => { + emit_out(b, out, out_len); + 0 + } + Err(_) => -2, + } +} + +/// live doc 的树 → IR 扁平字节(与 Phase 1 encode 输出兼容)。 +#[no_mangle] +pub unsafe extern "C" fn mogan_loro_doc_to_ir( + doc: *mut LoroDoc, + out: *mut *mut u8, + out_len: *mut usize, +) -> i32 { + if doc.is_null() { + return -1; + } + match doc_to_ir(&*doc) { + Ok(root) => { + let mut w = Writer { buf: Vec::new() }; + w.node(&root); + emit_out(w.buf, out, out_len); + 0 + } + Err(()) => -2, + } +} + +/// 把 snapshot/update 字节 import(合并)进 live doc。成功返回 0。 +/// 用于 Phase 3 反向同步:远端 Loro 数据 → 本地 shadow doc。 +#[no_mangle] +pub unsafe extern "C" fn mogan_loro_doc_import( + doc: *mut LoroDoc, + bytes: *const u8, + len: usize, +) -> i32 { + if doc.is_null() || bytes.is_null() { + return -1; + } + let buf = std::slice::from_raw_parts(bytes, len); + match (*doc).import(buf) { + Ok(_) => { + (*doc).commit(); + 0 + } + Err(_) => -2, + } +} + +/// 递归写出"带 TreeID 的增强 IR":每节点前缀 peer:u64 counter:i32,再跟普通 IR 节点。 +/// 用于 Phase 3:B 导入后据此把 buffer rep 关联到导入的 TreeID,使 B 的后续编辑能合并。 +fn write_node_with_id(tree: &LoroTree, id: TreeID, w: &mut Writer) { + w.buf.extend_from_slice(&id.peer.to_le_bytes()); + w.buf.extend_from_slice(&id.counter.to_le_bytes()); + let (kind_byte, label, text, child_ids) = match tree.get_meta(id) { + Ok(meta) => { + let kind = get_meta_str(&meta, "kind").unwrap_or_default(); + let label = get_meta_bytes(&meta, "label").unwrap_or_default(); + let text = get_meta_text_or_binary(&meta).unwrap_or_default(); + let children = tree.children(TreeParentId::Node(id)).unwrap_or_default(); + (str_to_kind(&kind), label, text, children) + } + Err(_) => (KIND_COMPOUND, Vec::new(), Vec::new(), Vec::new()), + }; + w.u8(kind_byte); + w.bytes_field(&label); + w.bytes_field(&text); + w.u32(child_ids.len() as u32); + for cid in child_ids { + write_node_with_id(tree, cid, w); + } +} + +/// live doc -> 带 TreeID 的增强 IR 字节。 +#[no_mangle] +pub unsafe extern "C" fn mogan_loro_doc_to_ir_with_ids( + doc: *mut LoroDoc, + out: *mut *mut u8, + out_len: *mut usize, +) -> i32 { + if doc.is_null() { + return -1; + } + let tree = (*doc).get_tree(TREE_NAME); + let root = match tree.roots().get(0) { + Some(r) => *r, + None => return -2, + }; + let mut w = Writer { buf: Vec::new() }; + write_node_with_id(&tree, root, &mut w); + emit_out(w.buf, out, out_len); + 0 +} + +/// 在 parent 的 index 处创建子节点。compound/generic 传 label;atomic 不用。 +/// 返回新节点 TreeID;失败返回 NULL_ID。 +#[no_mangle] +pub unsafe extern "C" fn mogan_loro_node_create( + doc: *mut LoroDoc, + parent: MoganTreeId, + index: u32, + kind: u8, + label: *const u8, + label_len: usize, +) -> MoganTreeId { + if doc.is_null() { + return NULL_ID; + } + let tree = (*doc).get_tree(TREE_NAME); + let id = match tree.create_at(parent_from(parent), index as usize) { + Ok(id) => id, + Err(_) => return NULL_ID, + }; + if let Ok(meta) = tree.get_meta(id) { + let _ = meta.insert("kind", kind_to_str(kind).to_string()); + if kind != KIND_ATOMIC && !label.is_null() && label_len > 0 { + let lbl = std::slice::from_raw_parts(label, label_len); + let _ = meta.insert("label", lbl.to_vec()); + } + } + treeid_to(id) +} + +/// 删除节点(子节点不真正删除,从状态中消失)。 +#[no_mangle] +pub unsafe extern "C" fn mogan_loro_node_delete(doc: *mut LoroDoc, id: MoganTreeId) -> i32 { + if doc.is_null() { + return -1; + } + (*doc) + .get_tree(TREE_NAME) + .delete(treeid_from(id)) + .map(|_| 0) + .unwrap_or(-2) +} + +/// 移动 target 到 parent 的 index 处(包/拆/重排)。 +#[no_mangle] +pub unsafe extern "C" fn mogan_loro_node_mov( + doc: *mut LoroDoc, + target: MoganTreeId, + parent: MoganTreeId, + index: u32, +) -> i32 { + if doc.is_null() { + return -1; + } + (*doc) + .get_tree(TREE_NAME) + .mov_to(treeid_from(target), parent_from(parent), index as usize) + .map(|_| 0) + .unwrap_or(-2) +} + +/// 原子文本 JOIN:把 Y 的文本追加到 X 的 LoroText 末尾(unicode 长度处)+ 删除 Y。 +/// X/Y 都须是文本容器(meta.text = LoroText)。用于精确翻译 MOD_JOIN(退格合并)。 +#[no_mangle] +pub unsafe extern "C" fn mogan_loro_node_join_text( + doc: *mut LoroDoc, + x_id: MoganTreeId, + y_id: MoganTreeId, +) -> i32 { + if doc.is_null() { + return -1; + } + let tree = (*doc).get_tree(TREE_NAME); + let x_meta = match tree.get_meta(treeid_from(x_id)) { + Ok(m) => m, + Err(_) => return -2, + }; + let y_meta = match tree.get_meta(treeid_from(y_id)) { + Ok(m) => m, + Err(_) => return -3, + }; + let x_text = match x_meta.get("text") { + Some(ValueOrContainer::Container(Container::Text(t))) => t, + _ => return -4, // X 非文本容器(调用方改走复合 join) + }; + // X 的 unicode 长度(追加位置) + let mut x_len = 0usize; + x_text.iter(|c| { + x_len += c.chars().count(); + true + }); + // Y 的文本内容 + let mut y_content = String::new(); + match y_meta.get("text") { + Some(ValueOrContainer::Container(Container::Text(t))) => { + t.iter(|c| { + y_content.push_str(c); + true + }); + } + _ => return -5, // Y 非文本容器 + } + if !y_content.is_empty() && x_text.insert(x_len, &y_content).is_err() { + return -6; + } + if tree.delete(treeid_from(y_id)).is_err() { + return -7; + } + 0 +} + +/// 取 parent 的子节点 TreeID 列表,序列化为 N*12 字节(每条 peer:u64 LE + counter:i32 LE)。 +/// 写入 *out(*out_len = 字节数),Rust 分配,需 mogan_loro_free 释放。REMOVE 按位置删用。 +#[no_mangle] +pub unsafe extern "C" fn mogan_loro_node_children( + doc: *mut LoroDoc, + parent: MoganTreeId, + out: *mut *mut u8, + out_len: *mut usize, +) -> i32 { + if doc.is_null() { + return -1; + } + let tree = (*doc).get_tree(TREE_NAME); + let children = match tree.children(parent_from(parent)) { + Some(c) => c, + None => return -2, + }; + let mut buf = Vec::with_capacity(children.len() * 12); + for c in children { + buf.extend_from_slice(&c.peer.to_le_bytes()); + buf.extend_from_slice(&c.counter.to_le_bytes()); + } + emit_out(buf, out, out_len); + 0 +} + +/// 改 compound/generic 节点的 label。 +#[no_mangle] +pub unsafe extern "C" fn mogan_loro_node_set_label( + doc: *mut LoroDoc, + id: MoganTreeId, + label: *const u8, + label_len: usize, +) -> i32 { + if doc.is_null() || label.is_null() { + return -1; + } + let tree = (*doc).get_tree(TREE_NAME); + let meta = match tree.get_meta(treeid_from(id)) { + Ok(m) => m, + Err(_) => return -2, + }; + let lbl = std::slice::from_raw_parts(label, label_len); + meta.insert("label", lbl.to_vec()).map(|_| 0).unwrap_or(-3) +} + +/// 设置二进制原子的 text(图片等)。 +#[no_mangle] +pub unsafe extern "C" fn mogan_loro_node_set_binary( + doc: *mut LoroDoc, + id: MoganTreeId, + bytes: *const u8, + len: usize, +) -> i32 { + if doc.is_null() || bytes.is_null() { + return -1; + } + let tree = (*doc).get_tree(TREE_NAME); + let meta = match tree.get_meta(treeid_from(id)) { + Ok(m) => m, + Err(_) => return -2, + }; + let b = std::slice::from_raw_parts(bytes, len); + meta.insert("text", b.to_vec()).map(|_| 0).unwrap_or(-3) +} + +/// 取节点的 LoroText(没有则新建),用于文本原子的字符级编辑。 +fn get_or_create_text(meta: &LoroMap) -> Option { + if let Some(ValueOrContainer::Container(Container::Text(t))) = meta.get("text") { + return Some(t); + } + meta.insert_container("text", LoroText::new()).ok() +} + +/// 文本原子:在 pos 处插入 bytes(须为合法 UTF-8)。 +#[no_mangle] +pub unsafe extern "C" fn mogan_loro_node_text_insert( + doc: *mut LoroDoc, + id: MoganTreeId, + pos: u32, + bytes: *const u8, + len: usize, +) -> i32 { + if doc.is_null() || bytes.is_null() { + return -1; + } + let tree = (*doc).get_tree(TREE_NAME); + let meta = match tree.get_meta(treeid_from(id)) { + Ok(m) => m, + Err(_) => return -2, + }; + let txt = match get_or_create_text(&meta) { + Some(t) => t, + None => return -3, + }; + let s = match std::str::from_utf8(std::slice::from_raw_parts(bytes, len)) { + Ok(s) => s, + Err(_) => return -4, + }; + if !s.is_empty() { + txt.insert(pos as usize, s).map(|_| 0).unwrap_or(-5) + } else { + 0 + } +} + +/// 文本原子:在 pos 处删除 len 个字符。 +#[no_mangle] +pub unsafe extern "C" fn mogan_loro_node_text_delete( + doc: *mut LoroDoc, + id: MoganTreeId, + pos: u32, + len: u32, +) -> i32 { + if doc.is_null() { + return -1; + } + let tree = (*doc).get_tree(TREE_NAME); + let meta = match tree.get_meta(treeid_from(id)) { + Ok(m) => m, + Err(_) => return -2, + }; + let txt = match meta.get("text") { + Some(ValueOrContainer::Container(Container::Text(t))) => t, + _ => return -3, + }; + txt.delete(pos as usize, len as usize) + .map(|_| 0) + .unwrap_or(-4) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn atomic(text: &str) -> IrNode { + IrNode { + kind: KIND_ATOMIC, + label: vec![], + text: text.as_bytes().to_vec(), + children: vec![], + } + } + + fn compound(label: &str, children: Vec) -> IrNode { + IrNode { + kind: KIND_COMPOUND, + label: label.as_bytes().to_vec(), + text: vec![], + children, + } + } + + /// 隔离诊断:分步定位 round-trip 失败在「读路径」还是「export/import」。 + /// 运行:cd 3rdparty/mogan-loro-ffi && cargo test -- --nocapture + #[test] + fn roundtrip_atomic_isolate() { + let orig = atomic("hello 世界"); + let doc = doc_from_ir(&orig).expect("doc_from_ir"); + + // (1) 直接读 encoding doc(不经 export/import)——验证 get_meta 读路径 + let direct = doc_to_ir(&doc).expect("doc_to_ir direct"); + eprintln!( + "[direct] kind={} label={:?} text={:?}", + direct.kind, direct.label, direct.text + ); + assert_eq!(direct.kind, KIND_ATOMIC, "direct read: kind mismatch"); + assert_eq!( + direct.text, + "hello 世界".as_bytes(), + "direct read: text mismatch (get_meta 路径有问题)" + ); + + // (2) 经 export / import ——验证快照是否保留 tree + 元数据 + let snap = doc.export(ExportMode::Snapshot).expect("export"); + eprintln!("[snap] len={}", snap.len()); + let doc2 = LoroDoc::new(); + doc2.import(&snap).expect("import"); + doc2.commit(); + doc2.checkout_to_latest(); + let tree2 = doc2.get_tree(TREE_NAME); + eprintln!("[import] roots={}", tree2.roots().len()); + let back = doc_to_ir(&doc2).expect("doc_to_ir after import"); + eprintln!( + "[back] kind={} label={:?} text={:?}", + back.kind, back.label, back.text + ); + assert_eq!(back.kind, KIND_ATOMIC, "after import: kind mismatch"); + assert_eq!( + back.text, + "hello 世界".as_bytes(), + "after import: text mismatch (export/import 丢数据)" + ); + } + + /// 二进制(非法 UTF-8,如 PNG 头 89 50 4E 47)必须逐字节保留——这是真实图片 + /// 文档往返的关键。用合法 UTF-8 的测试压不到这条路径。 + #[test] + fn roundtrip_binary_atomic() { + let binary= vec![0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, 0x80, 0xFE, 0xFF, 0xC0]; + let orig = IrNode { + kind: KIND_ATOMIC, + label: vec![], + text: binary.clone(), + children: vec![], + }; + let doc = doc_from_ir(&orig).expect("doc_from_ir"); + let snap = doc.export(ExportMode::Snapshot).expect("export"); + let doc2 = LoroDoc::new(); + doc2.import(&snap).expect("import"); + doc2.commit(); + doc2.checkout_to_latest(); + let back = doc_to_ir(&doc2).expect("doc_to_ir"); + eprintln!("[binary] orig={:?} back={:?}", orig.text, back.text); + assert_eq!(back.text, binary, "二进制字节未保留(EFBFBD 即 U+FFFD 替换)"); + } + + #[test] + fn roundtrip_compound_isolate() { + let orig = compound( + "document", + vec![compound("para", vec![atomic("hi")]), atomic("x")], + ); + let doc = doc_from_ir(&orig).expect("doc_from_ir"); + + let direct = doc_to_ir(&doc).expect("direct"); + eprintln!( + "[direct] label={:?} n_children={}", + direct.label, + direct.children.len() + ); + assert_eq!(direct.label, "document".as_bytes(), "direct: label"); + assert_eq!(direct.children.len(), 2, "direct: n_children"); + + let snap = doc.export(ExportMode::Snapshot).expect("export"); + let doc2 = LoroDoc::new(); + doc2.import(&snap).expect("import"); + doc2.commit(); + doc2.checkout_to_latest(); + let back = doc_to_ir(&doc2).expect("back"); + eprintln!( + "[back] label={:?} n_children={}", + back.label, + back.children.len() + ); + assert_eq!(back.label, "document".as_bytes(), "after import: label"); + assert_eq!(back.children.len(), 2, "after import: n_children"); + assert_eq!(back.children[0].label, "para".as_bytes(), "after import: child0 label"); + } + + /// 全 FFI 字节往返:IrNode -> Writer -> mogan_loro_encode -> mogan_loro_decode + /// -> Reader -> IrNode。打印 Writer 字节,用于与 C++ 侧 loro_ir_encode 输出比对。 + #[test] + fn full_ffi_byte_roundtrip() { + let orig = compound("document", vec![atomic("hi"), atomic("x")]); + let mut w = Writer { buf: vec![] }; + w.node(&orig); + eprintln!("[rust-writer] {} bytes: {:?}", w.buf.len(), w.buf); + + let ir_bytes= w.buf.clone(); + let mut snap_ptr: *mut u8 = std::ptr::null_mut(); + let mut snap_len: usize = 0; + let rc = mogan_loro_encode( + ir_bytes.as_ptr(), + ir_bytes.len(), + &mut snap_ptr, + &mut snap_len, + ); + assert_eq!(rc, 0, "mogan_loro_encode rc={}", rc); + let snap: Vec = + unsafe { std::slice::from_raw_parts(snap_ptr, snap_len) }.to_vec(); + unsafe { mogan_loro_free(snap_ptr, snap_len) }; + + let mut out_ptr: *mut u8 = std::ptr::null_mut(); + let mut out_len: usize = 0; + let rc = + mogan_loro_decode(snap.as_ptr(), snap.len(), &mut out_ptr, &mut out_len); + assert_eq!(rc, 0, "mogan_loro_decode rc={}", rc); + let ir_bytes2: Vec = + unsafe { std::slice::from_raw_parts(out_ptr, out_len) }.to_vec(); + eprintln!("[rust-decode] {} bytes: {:?}", ir_bytes2.len(), ir_bytes2); + unsafe { mogan_loro_free(out_ptr, out_len) }; + + let mut r = Reader { buf: &ir_bytes2, pos: 0 }; + let back = r.node().expect("Reader::node"); + eprintln!( + "[rust-back] kind={} label={:?} n_children={}", + back.kind, + back.label, + back.children.len() + ); + assert_eq!(back.label, "document".as_bytes()); + assert_eq!(back.children.len(), 2); + assert_eq!(back.children[0].text, "hi".as_bytes()); + assert_eq!(back.children[1].text, "x".as_bytes()); + // Writer 输出(encode 输入)应与 decode 输出逐字节一致 + assert_eq!(ir_bytes, ir_bytes2, "ir bytes changed across FFI roundtrip"); + } + + /// live API:doc_new → seed(IR) → to_ir 往返;且 export snapshot 非空。 + #[test] + fn live_seed_and_export() { + let orig = compound("document", vec![atomic("hi"), atomic("世界")]); + let mut w = Writer { buf: Vec::new() }; + w.node(&orig); + let ir = w.buf; + + let doc = mogan_loro_doc_new(); + let rc = unsafe { mogan_loro_doc_seed(doc, ir.as_ptr(), ir.len()) }; + assert_eq!(rc, 0, "seed"); + + let mut out = std::ptr::null_mut(); + let mut out_len = 0usize; + let rc = unsafe { mogan_loro_doc_to_ir(doc, &mut out, &mut out_len) }; + assert_eq!(rc, 0, "to_ir"); + let back_bytes = unsafe { std::slice::from_raw_parts(out, out_len) }.to_vec(); + unsafe { mogan_loro_free(out, out_len) }; + let back = Reader { + buf: &back_bytes, + pos: 0, + } + .node() + .expect("parse back"); + assert_eq!(back.label, b"document"); + assert_eq!(back.children.len(), 2); + assert_eq!(back.children[0].text, b"hi"); + assert_eq!(back.children[1].text, "世界".as_bytes()); + + let mut snap = std::ptr::null_mut(); + let mut snap_len = 0usize; + let rc = unsafe { mogan_loro_doc_export(doc, &mut snap, &mut snap_len) }; + assert_eq!(rc, 0, "export"); + assert!(snap_len > 0, "snapshot non-empty"); + unsafe { mogan_loro_free(snap, snap_len) }; + + unsafe { mogan_loro_doc_free(doc) }; + } + + /// live API:node_create + 文本 insert/delete(Phase 2 字符级镜像的核心)。 + #[test] + fn live_node_create_and_text() { + let doc = mogan_loro_doc_new(); + let root = unsafe { + mogan_loro_node_create( + doc, + MoganTreeId { + peer: MOGAN_ROOT_PEER, + counter: 0, + }, + 0, + KIND_COMPOUND, + b"document".as_ptr(), + 8, + ) + }; + assert!(root.peer != 0, "root created"); + let atom = + unsafe { mogan_loro_node_create(doc, root, 0, KIND_ATOMIC, b"".as_ptr(), 0) }; + assert!(atom.peer != 0, "atom created"); + + let rc = unsafe { mogan_loro_node_text_insert(doc, atom, 0, b"hi".as_ptr(), 2) }; + assert_eq!(rc, 0, "insert hi"); + let rc = unsafe { mogan_loro_node_text_insert(doc, atom, 2, b"x".as_ptr(), 1) }; + assert_eq!(rc, 0, "insert x → hix"); + let rc = unsafe { mogan_loro_node_text_delete(doc, atom, 2, 1) }; + assert_eq!(rc, 0, "delete → hi"); + + let mut out = std::ptr::null_mut(); + let mut out_len = 0usize; + unsafe { mogan_loro_doc_to_ir(doc, &mut out, &mut out_len) }; + let bb = unsafe { std::slice::from_raw_parts(out, out_len) }.to_vec(); + unsafe { mogan_loro_free(out, out_len) }; + let back = Reader { buf: &bb, pos: 0 }.node().expect("parse"); + assert_eq!(back.children[0].kind, KIND_ATOMIC); + assert_eq!(back.children[0].text, b"hi"); + + unsafe { mogan_loro_doc_free(doc) }; + } +} diff --git a/TeXmacs/doc/about/mogan/stem.en.tmu b/TeXmacs/doc/about/mogan/stem.en.tmu index b92dfce45b..0ec0b290ef 100644 --- a/TeXmacs/doc/about/mogan/stem.en.tmu +++ b/TeXmacs/doc/about/mogan/stem.en.tmu @@ -1,4 +1,4 @@ -> +> > diff --git a/devel/0760.md b/devel/0760.md new file mode 100644 index 0000000000..0e995b1d55 --- /dev/null +++ b/devel/0760.md @@ -0,0 +1,912 @@ +# Mogan × Loro CRDT 协作编辑集成——完整技术文档 + +> **文档版本**:2026-07-17 +> **对应分支**:`jimzhou/200_27/loro_crdt_convert` +> **任务文档**:[devel/2030.md](2030.md) + +--- + +## 目录 + +1. [项目概述](#1-项目概述) +2. [系统全景架构](#2-系统全景架构) +3. [Mogan 文档树模型](#3-mogan-文档树模型) +4. [Loro CRDT 基础](#4-loro-crdt-基础) +5. [Phase 1:树形文档 ↔ Loro CRDT 快照互转](#5-phase-1树形文档--loro-crdt-快照互转) +6. [Phase 2:本地操作镜像(op-level mirroring)](#6-phase-2本地操作镜像op-level-mirroring) +7. [Phase 3:远端同步与合并](#7-phase-3远端同步与合并) +8. [事件级增量同步](#8-事件级增量同步) +9. [网络传输层](#9-网络传输层) +10. [构建系统与配置](#10-构建系统与配置) +11. [测试体系](#11-测试体系) +12. [已知限制与后续方向](#12-已知限制与后续方向) +13. [关键文件索引](#13-关键文件索引) +14. [常见问题排查](#14-常见问题排查) + +--- + +## 1. 项目概述 + +### 1.1 目标 + +为 Mogan(TeXmacs 衍生的结构化编辑器)接入 [Loro CRDT](https://loro.dev),实现: + +- **多人实时协作编辑**:两个或更多编辑器同时打开同一文档,各自编辑后通过 WebSocket 中继服务器交换增量更新,文档状态自动合并收敛。 +- **不改变现有架构**:Mogan 现有的树形文档模型(`tree` + 全局元树 `the_et`)、编辑流程(`modification` → `edit_announce` → `edit_done`)、排版(typesetter)全部保留不变。Loro 作为"影子"(shadow)层并行运行。 + +### 1.2 为什么选择 Loro + +| 需求 | Loro 对应能力 | +|---|---| +| 文档是树形结构 | LoroTree(可移动树,CRDT 安全的 move/insert/delete) | +| 文本叶子需要字符级编辑 | LoroText(RGA 文本,支持 unicode 位置 insert/delete) | +| 并发编辑自动合并 | CRDT 语义(无需手写 OT) | +| 增量同步(只传变化) | `subscribe_local_update` 回调 + `import(bytes)` | +| 快照持久化 | `export(ExportMode::Snapshot)` + `import(bytes)` | +| 高性能 | Rust 实现,操作编码压缩 | + +### 1.3 技术栈 + +- **Loro 核心**:Rust 编写的 [loro crate](https://crates.io/crates/loro) v1.13.7,源码位于 `3rdparty/loro/`(完整 workspace 拉取)。 +- **C FFI shim**:自写的 `3rdparty/mogan-loro-ffi/`——由于 Loro 官方只提供 UniFFI 绑定(Swift/Python),我们自行暴露一组 `extern "C"` 函数。 +- **C++ 胶水**:`moebius/Data/Convert/` 下的 `loro_ir`、`loro`、`loro_shadow`——负责 IR 编解码、快照转换、shadow doc 管理、modification 增量翻译。 +- **编辑器挂载**:`src/Edit/Modify/edit_modify.cpp`——把 shadow doc 挂到编辑器实例,在 `edit_done` 中镜像每次编辑,在收到远端更新时反向应用。 +- **网络中继**:`tools/loro-relay/server.js`——Node.js + `ws` 库的 WebSocket 转发服务器。 +- **编辑器 WebSocket 客户端**:`src/Plugins/ImGui/im_websocket.{hpp,cpp}`——基于 libcurl 的轻量 WS 客户端(不依赖 Qt)。 + +--- + +## 2. 系统全景架构 + +``` +┌─────────────────────────── 编辑器 A(Mogan STEM) ───────────────────────────┐ +│ │ +│ 用户敲字 → edit_announce(mod) → edit_insert → notify_insert → 树更新 │ +│ ↓ │ +│ edit_done(mod) │ +│ ↓ │ +│ mirror_loro(mod/rp) │ +│ ↓ │ +│ ┌─────────────────────────────────────────────┐ │ +│ │ loro_shadow (shadow doc) │ │ +│ │ │ │ +│ │ mirror_mod(buffer, mod) │ │ +│ │ → 按mod类型分派增量 Loro op: │ │ +│ │ INSERT/REMOVE原子 → LoroText │ │ +│ │ INSERT/REMOVE复合 → node_create/delete │ │ +│ │ ASSIGN_NODE → node_set_label │ │ +│ │ SPLIT → text_delete+node_create │ │ +│ │ JOIN → node_join_text/mov+delete │ │ +│ │ INSERT_NODE → node_create+mov │ │ +│ │ REMOVE_NODE → mov+delete │ │ +│ │ ASSIGN → delete+seed_node │ │ +│ │ → doc.commit() │ │ +│ │ ↓ │ │ +│ │ subscribe_local_update 回调 → delta bytes │ │ +│ └────────────────────┬─────────────────────────┘ │ +│ │ delta bytes │ +│ g_loro_broadcast_update(data) │ +│ │ │ +│ im_websocket_client │ +│ │ ws.send(binary) │ +└─────────────────────────────────────┼─────────────────────────────────────────┘ + │ + ┌───────▼───────┐ + │ JS Relay │ + │ server.js │ + │ (ws :8765) │ + │ │ + │ 按 room 广播 │ + └───┬───────┬───┘ + │ │ + ws.send(binary) │ │ ws.send(binary) +┌─────────────────────────────────┼───────┼─────────────────────────────────────┐ +│ 编辑器 B │ │ 编辑器 C │ +│ im_websocket_client │ │ (同 B) │ +│ ↓ on_message(data) │ │ ↓ │ +│ ed->apply_remote(data) │ │ apply_remote │ +│ ↓ │ │ ↓ │ +│ loro_shadow::remote_diff_mods │ │ 同左 │ +│ (import bytes → diff → mods) │ │ ↓ │ +│ ↓ │ │ edit_announce │ +│ edit_announce(rp * mod) │ │ (versioning 模式) │ +│ (loro_applying_remote=true │ │ → buffer 更新 │ +│ → mirror_loro 跳过, 避免循环) │ │ → 重排版 │ +│ ↓ │ │ │ +│ buffer 更新 → typeset → 重绘 │ │ │ +└─────────────────────────────────┘ └───────────────────────────────────────┘ +``` + +### 2.1 数据流总结 + +| 方向 | 路径 | 产物 | +|---|---|---| +| **本地编辑 → 远端** | 用户操作 → `modification` → `mirror_mod` → Loro 增量 op → `commit()` → `subscribe_local_update` 回调 → delta bytes → WS 发送 | 几十~几百字节的增量 update | +| **远端 → 本地** | WS 接收 → `apply_remote(bytes)` → `import` 合并到 shadow → `diff_from_current` 生成 `modification` → `edit_announce` 应用到 buffer(versioning 模式跳过镜像) | buffer 更新 + 重排版 | +| **快照(全量)** | `export_snapshot()` → snapshot bytes → 可存盘/可发给新加入者 | 完整文档状态 | +| **新加入者初始化** | relay 发送 room 快照 → `import_and_build(bytes, buffer)` → shadow import + 身份表重建 | 新编辑器获得当前协作状态 | + +--- + +## 3. Mogan 文档树模型 + +### 3.1 tree 类型 + +Mogan 的核心数据类型是 `tree`(`lolly_tree`),定义在 [lolly_tree.hpp](../../lolly/lolly/data/lolly_tree.hpp): + +```cpp +// 原子节点(字符串叶子) +tree("hello") // op=0 (STRING), label="hello" + +// 复合节点(标签 + 子节点数组) +tree(DOCUMENT, para1, para2) // op=DOCUMENT, children=[para1, para2] + +// 三种节点类别: +// - is_atomic(t): op==0, 持字符串叶子 +// - is_compound(t): op>0, 持 op(标签int) + 子节点数组 +// - is_generic(t): op<0, 罕见(内部使用) +``` + +**关键属性**: +- 标签是 **整数**(`op`),通过 `STD_CODE` / `make_tree_label` 与字符串名字(如 `"document"`、`"paragraph"`)互转。 +- `raw_insert` 修改原子字符串时**原地改 `ref->label`**([tree_observer.cpp:248](../../moebius/Data/Tree/tree_observer.cpp)),**tree_rep* 指针稳定**——这是精确镜像身份表能工作的基础。 +- `raw_split` / `raw_join` / `raw_assign` 会用**新树替换** `ref[pos]`,产生**新的 tree_rep***。 + +### 3.2 path + +`path` = `list`(整数链表),从文档根到某节点的路径。如 `(0 2 1)` = 根→第 0 个子节点→第 2 个子节点→第 1 个子节点。 + +### 3.3 modification(9 种编辑操作) + +定义在 [modification.hpp](../../moebius/Kernel/Types/modification.hpp): + +| 类型 | 宏 | 含义 | accessor | +|---|---|---|---| +| ASSIGN | `MOD_ASSIGN` | 替换 `root(mod)` 处的节点为 `mod->t` | root=p, 无 index/argument | +| INSERT | `MOD_INSERT` | 在 `root(mod)` 的 `index(mod)` 位置插入 `mod->t` | root=path_up(p), index=last_item | +| REMOVE | `MOD_REMOVE` | 从 `root(mod)` 删除 `index(mod)` 起 `argument(mod)` 个子节点 | root=path_up(path_up(p)), index/argument | +| SPLIT | `MOD_SPLIT` | 将 `root(mod)` 的第 `index(mod)` 个子节点在 `argument(mod)` 处拆成两半 | root=path_up(path_up(p)), index/argument | +| JOIN | `MOD_JOIN` | 将 `root(mod)` 的第 `index(mod)` 与 `index+1` 个子节点合并 | root=path_up(p), index | +| ASSIGN_NODE | `MOD_ASSIGN_NODE` | 改 `root(mod)` 的标签为 `mod->t->op` | root=p | +| INSERT_NODE | `MOD_INSERT_NODE` | 在 `root(mod)` 的 `argument(mod)` 处包一层 `mod->t` | root=path_up(p), argument | +| REMOVE_NODE | `MOD_REMOVE_NODE` | 移除 `root(mod)` 的包装层,提升其子节点 | root=path_up(p), index | +| SET_CURSOR | `MOD_SET_CURSOR` | 光标移动(不涉及内容,镜像时忽略) | - | + +**编辑流程**: +``` +用户操作 → edit_announce(ed, mod) → edit_assign/insert/... → ed->notify_X(p) + → 树更新 + observer 通知 → edit_done(ed, mod) → post_notify + mirror_loro(mod) +``` + +### 3.4 编辑时各操作产生的 mod 序列 + +**敲一个字符 "a"**(在原子 "bc" 的位置 0 插入): +``` +insert([path-to-atomic], 0, "a") → 原子变 "abc" +``` +→ 1 个 mod,原子 rep 稳定。 + +**退格删一个字符**(删原子 "abc" 的位置 2): +``` +remove([path-to-atomic], 2, 1) → 原子变 "ab" +``` +→ 1 个 mod,原子 rep 稳定。 + +**回车(在段落末尾插入新段落)**——5 个 mod 序列: +``` +insert_node([p,0], 0, CONCAT) # 包一层 CONCAT +split([p,0], 0, char_pos) # 拆原子 +split([p], 0, 1) # 拆段落成两段 +remove_node([p,0], 0) # 移除临时包装 +remove_node([p,1], 0) # 移除临时包装 +``` + +**删除空行**——4 个 mod 序列: +``` +insert_node([p,0], 0, CONCAT) # 包装 +insert_node([p,1], 0, CONCAT) # 包装 +join([p], 0) # 合并两段 +join([p,0], 0) # 合并文本 +remove_node([p,0], 0) # 移除包装 +``` + +--- + +## 4. Loro CRDT 基础 + +### 4.1 核心概念 + +| 概念 | 说明 | +|---|---| +| **LoroDoc** | 一个完整的 CRDT 文档实例。持有 OpLog(操作历史)+ DocState(当前状态)。 | +| **LoroTree** | 可移动树容器。节点用 TreeID(`{peer:u64, counter:i32}`)唯一标识;支持 `create_at(parent, index)`、`delete(id)`、`mov_to(target, parent, index)`。 | +| **LoroMap** | 每个树节点自动关联一个 Map(通过 `tree.get_meta(id)` 获取),可存储键值对。我们用它存节点的 `kind`/`label`/`text`。 | +| **LoroText** | 协作文本文器。insert(pos, s) / delete(pos, len)。pos 是 unicode 字符位置。 | +| **PeerID** | 每个 LoroDoc 有唯一的 peer(u64),用于区分不同客户端的操作。 | +| **Subscription** | `subscribe_local_update(callback)` 注册回调,每次本地 op commit 时回调收到增量 bytes。 | +| **Snapshot** | `export(ExportMode::Snapshot)` 导出全量快照。`import(bytes)` 可导入快照或增量 update(统一接口)。 | +| **分数索引** | LoroTree 用 fractional index 维护子节点顺序,支持 CRDT 安全的并发插入/移动。 | + +### 4.2 我们如何使用 Loro + +``` +LoroDoc + └─ LoroTree "tree"(根容器) + ├─ Node[root] (document) + │ ├─ kind="compound", label="document" + │ ├─ Node[child0] (paragraph) + │ │ ├─ kind="compound", label="paragraph" + │ │ └─ Node[child0_0] (concat) + │ │ ├─ kind="compound", label="concat" + │ │ └─ Node[child0_0_0] (atomic) + │ │ ├─ kind="atomic" + │ │ └─ text=LoroText("hello") + │ └─ Node[child1] (atomic) + │ ├─ kind="atomic" + │ └─ text=Binary(<图片bytes>) + ... +``` + +**每个 Mogan 树节点 → 一个 LoroTree 节点**: +- 复合节点:meta Map 存 `kind="compound"` + `label=标签名字符串`(如 `"document"`、`"paragraph"`)。 +- 原子节点(合法 UTF-8 文本):meta Map 存 `kind="atomic"` + LoroText 子容器(key `"text"`),支持字符级 CRDT。 +- 原子节点(非法 UTF-8,如图片 raw-data):meta Map 存 `kind="atomic"` + Binary 值(key `"text"`),不支持字符级编辑但能忠实往返。 +- generic 节点(op<0,极罕见):`kind="generic"` + `label="generic:"`。 + +--- + +## 5. Phase 1:树形文档 ↔ Loro CRDT 快照互转 + +### 5.1 设计 + +**目标**:给定一棵 Mogan 文档树 `tree`,生成等价的 Loro 快照字节;反向,给定 Loro 快照,重建 Mogan 文档树。要求 round-trip 无损(236 个真实 .tmu 文档全部验证通过)。 + +**分层架构**: + +``` +tree ←──────→ LoroIRNode ←──────→ 扁平二进制字节 ←──→ LoroDoc snapshot + loro_ir loro_ir (编解码) loro (FFI) +``` + +### 5.2 LoroIRNode(中性中间表示) + +定义在 [loro_ir.hpp](../../moebius/Data/Convert/loro_ir.hpp): + +```cpp +enum loro_node_kind { LORO_ATOMIC, LORO_COMPOUND, LORO_GENERIC }; + +struct loro_ir_node { + loro_node_kind kind; + string label; // compound: 标签名; generic: "generic:" + string text; // atomic: 字符串内容 + array children; // compound/generic: 有序子节点 +}; + +// tree ↔ LoroIRNode +loro_ir_node tree_to_loro_ir (tree t); +tree loro_ir_to_tree (loro_ir_node node); + +// LoroIRNode ↔ 扁平二进制(小端) +// 格式: node := kind:u8 label_len:u32 label:bytes text_len:u32 text:bytes +// n_children:u32 node*n_children +string loro_ir_encode (loro_ir_node node); +loro_ir_node loro_ir_decode (string bytes); +``` + +### 5.3 标签 int↔name 互转 + +```cpp +// op int → 标签名字符串 +as_string(L(t)) // → "document" / "paragraph" / ... + +// 标签名字符串 → op int +moebius::make_tree_label(name) // → DOCUMENT / PARA / ... +``` + +`STD_CODE`([drd_std.cpp](../../moebius/moebius/drd/drd_std.cpp))提供标准标签的 name→code 映射,需在测试中调 `init_std_drd()` 初始化。 + +### 5.4 tree ↔ Loro 快照(公开接口) + +定义在 [loro.hpp](../../moebius/Data/Convert/loro.hpp): + +```cpp +// tree → Loro snapshot bytes +string tree_to_loro (tree t); + +// Loro snapshot bytes → tree +tree loro_to_tree (string snapshot); +``` + +实现链路: +- `tree_to_loro`:`tree_to_loro_ir(t)` → `loro_ir_encode(ir)` → FFI `mogan_loro_encode(ir_bytes)` → Rust 建 LoroDoc + LoroTree → `export(Snapshot)` → snapshot bytes。 +- `loro_to_tree`:FFI `mogan_loro_decode(snap_bytes)` → Rust import + 遍历 LoroTree → IR bytes → `loro_ir_decode` → `loro_ir_to_tree` → tree。 + +### 5.5 FFI 接口(C ABI) + +定义在 [3rdparty/mogan-loro-ffi/src/lib.rs](../../3rdparty/mogan-loro-ffi/src/lib.rs): + +```rust +// 快照编解码(Phase 1) +mogan_loro_encode(*ir_bytes, len, **out, *out_len) -> i32 +mogan_loro_decode(*snap_bytes, len, **out, *out_len) -> i32 +mogan_loro_free(*ptr, len) + +// live-doc API(Phase 2/3) +mogan_loro_doc_new() -> *mut LoroDoc +mogan_loro_doc_free(*doc) +mogan_loro_doc_seed(*doc, *ir_bytes, len) -> i32 +mogan_loro_doc_export(*doc, **out, *out_len) -> i32 +mogan_loro_doc_to_ir(*doc, **out, *out_len) -> i32 +mogan_loro_doc_import(*doc, *bytes, len) -> i32 +mogan_loro_doc_commit(*doc) +mogan_loro_doc_to_ir_with_ids(*doc, **out, *out_len) -> i32 + +// 节点操作 +mogan_loro_node_create(*doc, parent_id, index, kind, *label, label_len) -> MoganTreeId +mogan_loro_node_delete(*doc, id) -> i32 +mogan_loro_node_mov(*doc, target, parent, index) -> i32 +mogan_loro_node_set_label(*doc, id, *label, len) -> i32 +mogan_loro_node_set_binary(*doc, id, *bytes, len) -> i32 +mogan_loro_node_text_insert(*doc, id, pos, *bytes, len) -> i32 +mogan_loro_node_text_delete(*doc, id, pos, len) -> i32 +mogan_loro_node_join_text(*doc, x_id, y_id) -> i32 +mogan_loro_node_children(*doc, parent, **out, *out_len) -> i32 + +// 事件 +mogan_loro_doc_on_local_update(*doc, cb, user_data) -> i32 +``` + +**MoganTreeId**:`#[repr(C)] #[derive(Clone, Copy)] pub struct MoganTreeId { peer: u64, counter: i32 }`,与 Loro 的 `TreeID` 布局一致,按值跨 FFI 传递。Root 哨兵 = `{peer: u64::MAX, counter: 0}`。 + +--- + +## 6. Phase 2:本地操作镜像(op-level mirroring) + +### 6.1 设计目标 + +每次用户编辑产生一个 `modification`,将其**增量翻译**为对应的 Loro 操作,应用到 shadow LoroDoc 上。保持 shadow 与 buffer 一致,同时只产生小的增量 delta(而非整树重建)。 + +### 6.2 loro_shadow 类 + +定义在 [loro_shadow.hpp](../../moebius/Data/Convert/loro_shadow.hpp) / [loro_shadow.cpp](../../moebius/Data/Convert/loro_shadow.cpp): + +```cpp +class loro_shadow_rep : public concrete_struct { + void* doc; // mogan_loro_doc 句柄 + hashmap id_map; // buffer 节点 rep* → Loro TreeID + mogan_tree_id root_id; // 文档根的 TreeID + + void seed(tree root); // 用树构建 live doc + 填充身份表 + void mirror_mod(tree doc_root, modification mod); // 核心:增量镜像 + string export_snapshot(); // snapshot bytes + tree to_tree(); // live doc → tree + bool import_data(string bytes); // import 远端 update/snapshot + bool import_and_build(string bytes, tree& out_buffer); // import + 重建身份表 + void on_local_update(cb, ud); // 订阅增量回调 + list remote_diff_mods(string bytes, tree buffer); // 远端→mod + list diff_from_current(tree buffer); // diff(不 import) + mogan_tree_id get_id(tree t); // 查 TreeID + bool has_id(tree t); +private: + mogan_tree_id seed_node(tree t, mogan_tree_id parent, uint32_t index); + array node_children(mogan_tree_id parent); +}; +``` + +### 6.3 身份表(id_map) + +**核心设计**:`hashmap`,key 是 `inside(node)`(tree_rep 裸指针),value 是 Loro TreeID。 + +**为什么用 rep* 做 key**: +- Mogan 的 `tree` 是 COW 句柄(`concrete_struct` + refcount),`tree_rep*` 在节点被替换前稳定。 +- `raw_insert` 修改原子字符串时**原地改 `ref->label`**,rep* 不变 → 敲字时身份表直接命中。 +- Observer 系统(tree_pointer)也是用 rep* 跟踪节点的。 + +**身份表维护规则**(每个 mod 类型): + +| mod 类型 | 身份表更新 | +|---|---| +| INSERT 原子 | 无变化(原子 rep 稳定) | +| INSERT 复合子节点 | `seed_node` 创建新子树,映射各 rep → 新 TreeID | +| REMOVE 原子 | 无变化(原子 rep 稳定) | +| REMOVE 复合子节点 | 被删 TreeID 的 rep 表项变 stale(留,除非 rep 复用) | +| ASSIGN_NODE | 无变化(rep 稳定,只改 label) | +| SPLIT 原子 | 映射左半 t1 → X_id(X 截断后即 t1);映射右半 t2 → Y_id(新建) | +| SPLIT 复合 | 映射左半 t1 → X_id;建 Y(t2),mov X 的子节点到 Y | +| JOIN 原子 | 映射 merged → X_id(X 保留了合并内容) | +| JOIN 复合 | 同上 + mov Y 子节点到 X | +| INSERT_NODE | 映射 W(新建包装)→ w_id;被包节点 rep 稳定 | +| REMOVE_NODE | unmap W(已删);被提升子节点 rep 稳定 | +| ASSIGN | 删旧 + 建新,映射新子树各 rep | + +### 6.4 mirror_mod 的 9 种 mod 翻译表 + +**关键洞察**:mirror_mod 在 `edit_done`(post-apply)调用,此时 **buffer 已是编辑后状态**,而 **shadow LoroDoc 仍是编辑前状态**(尚未镜像本次 mod)。所以删除类操作的被删节点可直接从 LoroDoc 的当前子节点获取(通过 `node_children(parent_id)`),不需要 buffer 中已消失的 rep。 + +``` +mirror_mod(doc_root, mod): + rp_mod = root(mod) + + INSERT (parent = subtree(doc_root, rp_mod)): + if parent 是原子且 id_map 命中: + → mogan_loro_node_text_insert(id, index, mod->t->label) + elif parent 是复合且 id_map 命中: + → mogan_loro_node_create(pid, index, ...) 不适用(atomic 无 label) + 实际: seed_node(child, pid, index) # 创建子树 + 映射身份 + + REMOVE (parent = subtree(doc_root, rp_mod)): + if parent 是原子且 id_map 命中: + → mogan_loro_node_text_delete(id, index, argument) + elif parent 是复合且 id_map 命中: + → kids = node_children(pid) # LoroDoc 镜像前状态的子节点 + for j in [index+nr-1 .. index]: mogan_loro_node_delete(kids[j]) + + ASSIGN_NODE (node = subtree(doc_root, rp_mod)): + if id_map 命中: + → mogan_loro_node_set_label(id, as_string(L(mod))) + + SPLIT (P = rp_mod, pos = index, at = argument): + X = node_children(P_id)[pos] # LoroDoc 镜像前的第 pos 个子节点 + t1 = subtree(buffer, P*pos) # buffer 镜像后的左半 + t2 = subtree(buffer, P*(pos+1)) # buffer 镜像后的右半 + 映射 t1 → X_id # X 截断后即 t1 + if t2 是原子: + text_delete(X_id, at, N(t2->label)) # 截断 X 的文本 + Y = node_create(P_id, pos+1, ATOMIC) # 建右半 + text_insert(Y, 0, t2->label) + 映射 t2 → Y_id + elif t2 是复合: + Y = node_create(P_id, pos+1, COMPOUND, label(t2)) # 建右半 + for child in t2: node_mov(id_map[child], Y_id, i) # mov 子节点 + 映射 t2 → Y_id + + JOIN (P = rp_mod, pos = index): + X = node_children(P_id)[pos] + Y = node_children(P_id)[pos+1] + if X/Y 都是文本容器: + node_join_text(X_id, Y_id) # Rust 侧: 读 X 的 unicode 长度 + Y 的文本 → append + delete Y + else: # 复合 + for child in node_children(Y_id): node_mov(child, X_id, ...) + node_delete(Y_id) + 映射 merged → X_id + + INSERT_NODE (W = subtree(buffer, rp_mod), pos = argument): + parent_id = (path_up(rp_mod) 为空) ? root_id : id_map[parent] + rf_id = id_map[ref] # 被包节点(post-wrap 在 W 下 pos) + W_id = node_create(parent_id, last_item(rp_mod), COMPOUND, label(W)) + node_mov(rf_id, W_id, pos) + 映射 W → W_id + + REMOVE_NODE (wrapper at rp_mod, pos = index): + parent_id = (path_up(rp_mod) 为空) ? root_id : id_map[parent] + W_id = node_children(parent_id)[last_item(rp_mod)] # 镜像前状态的包装节点 + c_id = id_map[subtree(buffer, rp_mod)] # 被提升子节点(post-unwrap) + node_mov(c_id, parent_id, last_item(rp_mod)) # 提到 W 的位置 + node_delete(W_id) # 删空包装 + + ASSIGN (node = subtree(buffer, rp_mod)): + parent_id = (path_up(rp_mod) 为空) ? root_id : id_map[parent] + old_id = node_children(parent_id)[last_item(rp_mod)] + node_delete(old_id) + seed_node(node, parent_id, last_item(rp_mod)) # 从 buffer 当前子树重建 + + 兜底(极少数边界:root 级结构改动、身份缺失等): + 块级重 seed: 找到包含改动的 buffer 根直接子节点(块/段落)→ delete + seed_node + (远小于整篇,保 peer 血统) + + 最后: mogan_loro_doc_commit(doc) # 同步提交 → 触发 local_update 事件 +``` + +### 6.5 编辑器挂载 + +定义在 [edit_modify.hpp](../../src/Edit/Modify/edit_modify.hpp) / [edit_modify.cpp](../../src/Edit/Modify/edit_modify.cpp): + +```cpp +class edit_modify_rep : virtual public editor_rep { + loro_shadow loro_doc; // shadow LoroDoc + bool loro_seeded = false; // 是否已 seed 当前 buffer + bool loro_applying_remote = false; // 远端应用期间跳过镜像(versioning) + bool loro_routing = false; // debug_loro round-trip 中防递归 + + void mirror_loro(const modification& mod); // 在 edit_done 中调用 + void apply_remote(string bytes); // 收到远端 update 时调用 + bool route_through_loro(const modification& mod); // debug_loro 模式 +}; +``` + +**挂载点**:`edit_done(ed, mod)` → `ed->mirror_loro(mod)`: + +```cpp +void edit_done(editor_rep* ed, modification mod) { + ... + if (mod->k != MOD_SET_CURSOR) { + ed->post_notify(p); + ed->mirror_loro(mod); // ← Phase 2 镜像 + } +} +``` + +`mirror_loro` 的逻辑: +```cpp +void mirror_loro(const modification& mod) { + if (loro_applying_remote) return; // versioning:远端应用时跳过 + if (mod->k == MOD_SET_CURSOR) return; + if (!loro_seeded) { loro_doc->seed(the_buffer()); loro_seeded = true; } + loro_doc->mirror_mod(the_buffer(), mod / rp); // mod->p 从 the_et 根路径转为 buffer 相对 +} +``` + +--- + +## 7. Phase 3:远端同步与合并 + +### 7.1 apply_remote(远端 → 本地) + +当编辑器收到远端 Loro update bytes 时: + +```cpp +void apply_remote(string bytes) { + loro_applying_remote = true; // 守卫:下面应用的 mod 不会触发 mirror_loro + list mods = loro_doc->remote_diff_mods(bytes, the_buffer()); + for (auto l = mods; !is_nil(l); l = l->next) + edit_announce(this, rp * l->item); // 应用到 buffer + loro_applying_remote = false; + // 触发重排版 + notify_change(THE_TREE); + typeset_invalidate_all(); +} +``` + +### 7.2 remote_diff_mods(import + diff) + +```cpp +list remote_diff_mods(string bytes, tree buffer) { + import_data(bytes); // 把远端 update 合并进 shadow + return diff_from_current(buffer); // diff buffer(旧) vs shadow(新) → mods +} +``` + +**diff_from_current** 用树 diff 算法: +- 并行遍历 before(buffer)和 after(shadow.to_tree()): + - 对应节点都是原子且文本不同 → 字符级 diff(公共前缀/后缀对齐)→ MOD_INSERT/MOD_REMOVE。 + - 对应节点都是复合且结构相同(同 op、同子节点数)→ 递归遍历子节点。 + - 结构不同 → MOD_ASSIGN(root, after) 兜底(整树替换)。 + +### 7.3 import_and_build(新加入者初始化) + +当编辑器 B 加入协作时,relay 发送 room 快照。B 需要: +1. import 快照到 shadow。 +2. 重建 buffer = shadow 的当前树。 +3. 重建身份表(buffer rep → 导入的 TreeID)。 + +```cpp +bool import_and_build(string bytes, tree& out_buffer) { + import_data(bytes); + // 用"带 TreeID 的增强 IR"重建 buffer + 身份表 + uint8_t* out; size_t out_len; + mogan_loro_doc_to_ir_with_ids(doc, &out, &out_len); // 遍历 LoroTree,输出 peer+counter+node + // 解码增强 IR → 建 buffer 树 + 填 id_map + id_map = hashmap<...>(); // 重置 + out_buffer = decode_id_node(ir_bytes, pos, id_map); + return true; +} +``` + +**增强 IR 格式**:每个节点前缀 12 字节(peer:u64 LE + counter:i32 LE),然后是普通 IR 节点(kind + label + text + children)。 + +### 7.4 双向合并(CRDT 收敛) + +两个编辑器 A 和 B 各自从公共初始快照编辑不同节点: +1. A 在段落 0 编辑,导出 snapshot Sa。 +2. B 在段落 1 编辑,导出 snapshot Sb。 +3. A import Sb,B import Sa。 +4. 两端收敛到相同状态(段落 0 + 段落 1 的编辑都在)。 + +**前提**:A 和 B 共享相同的 TreeID(通过公共初始快照 `import_and_build` 建立)。如果各自独立 `seed`,TreeID 不同,import 后无法正确合并。 + +--- + +## 8. 事件级增量同步 + +### 8.1 subscribe_local_update + +```rust +// Rust FFI (lib.rs) +mogan_loro_doc_on_local_update(doc, cb, user_data) + → doc.subscribe_local_update(Box::new(move |bytes: &Vec| { + cb(user_data_as_usize as *mut c_void, bytes.as_ptr(), bytes.len()); + true // 保持订阅 + })) +``` + +**关键点**: +- `user_data` 裸指针转 `usize` 捕获(避开 edition 2021 离散闭包捕获 `*mut` 的 `!Send` 问题)。 +- Subscription 存在全局 `LOCAL_SUBS: Mutex>` 中保活(doc_free 时清理)。 +- `mogan_loro_doc_commit()` 显式提交 → 同步触发回调(Loro 的 auto_commit 是延迟的,不会同步触发)。 + +### 8.2 C++ 端回调 + +```cpp +// edit_modify.cpp +static void local_update_cb(void* user_data, const uint8_t* bytes, size_t len) { + if (g_loro_broadcast_update) { + string data((const char*)bytes, len); + g_loro_broadcast_update(data); // → ws_client->send(data) + } +} + +// 构造函数中注册 +edit_modify_rep::edit_modify_rep() { + ... + loro_doc->on_local_update(local_update_cb, this); +} +``` + +`g_loro_broadcast_update` 是函数指针,在 [im_gui.cpp](../../src/Plugins/ImGui/im_gui.cpp) 中设置为 WS 客户端的发送 lambda。 + +--- + +## 9. 网络传输层 + +### 9.1 JS WebSocket 中继服务器 + +文件:[tools/loro-relay/server.js](../../tools/loro-relay/server.js) + +功能: +- 监听 `ws://127.0.0.1:8765`(可通过 `MOGAN_LORO_RELAY_HOST`/`MOGAN_LORO_RELAY_PORT` 环境变量配置)。 +- 支持"房间"(room):客户端发文本消息 `JOIN ` 加入房间。 +- 二进制消息按房间广播给除发送者外的所有客户端。 +- 新加入者收到房间的最后一条状态(`room.state`)。 + +**已知限制**:`room.state` 存的是最后一条消息(可能只是小 delta),不是合并后的完整状态。生产环境应服务端维护一个 Loro doc 实例 merge 各 delta,给新加入者发 snapshot。 + +运行: +```bash +cd tools/loro-relay +npm install # 安装 ws 依赖 +node server.js +``` + +### 9.2 编辑器 WebSocket 客户端 + +文件:[src/Plugins/ImGui/im_websocket.{hpp,cpp}](../../src/Plugins/ImGui/im_websocket.hpp) + +- 基于 libcurl 的 WebSocket 实现(不依赖 Qt)。 +- `connect(url)` / `send(data, is_binary)` / `poll()`(非阻塞轮询)。 +- `on_message(data, is_binary)` 虚函数回调——在 [im_gui.cpp](../../src/Plugins/ImGui/im_gui.cpp) 的 `test_websocket_client` 中 override 为 `ed->apply_remote(data)`。 + +**连接流程**: +1. 编辑器启动时(ImGui 后端初始化)创建 `test_websocket_client`。 +2. 连接 `ws://127.0.0.1:8765`。 +3. 发送 `JOIN <文件名作为 room_id>`。 +4. 设置 `g_loro_broadcast_update` 为发送 lambda。 +5. `poll()` 在 ImGui 帧循环中调用,收到消息 → `apply_remote`。 + +--- + +## 10. 构建系统与配置 + +### 10.1 xmake 选项 + +| 选项 | 默认 | 作用 | +|---|---|---| +| `libloro` | false | 编译 Loro FFI(cargo 构建 `3rdparty/mogan-loro-ffi`);定义 `LORO_ENABLED` | +| `debug_loro` | false | 定义 `LORO_DEBUG`;启用 mirror_loro 中的 buffer↔Loro 一致性验证 | + +配置命令: +```bash +xmake f -m debug --libloro=y # 启用 Loro +xmake f -m debug --libloro=y --debug_loro=y # 启用 + 验证模式 +``` + +### 10.2 影响的目标 + +| 目标 | 影响 | +|---|---| +| `libmoebius` | `LORO_ENABLED` → 编译 `loro.cpp`/`loro_ir.cpp`/`loro_shadow.cpp` 的真实 FFI 调用;否则为空桩。cargo 构建 Rust staticlib 并链接。 | +| `libmogan` | `LORO_ENABLED` → 编译 `edit_modify.cpp` 中的 `mirror_loro`/`apply_remote`/`route_through_loro`;否则为 no-op。`LORO_DEBUG` → 编译一致性验证。 | +| `moebius_tests` | `LORO_ENABLED` → 编译 loro 相关测试用例。 | + +### 10.3 Rust FFI 构建链路 + +``` +3rdparty/mogan-loro-ffi/Cargo.toml + → 依赖 loro = { path = "../loro/crates/loro" }(本地源码) + → [patch.crates-io] generic-btree = { path = "../loro/crates/generic-btree" } + → crate-type = ["staticlib"] → libmogan_loro_ffi.a + +moebius/xmake.lua (libmoebius target): + before_build → cargo build --profile dev/release + on_config → add_linkdirs + add_links("mogan_loro_ffi") + add_syslinks(pthread/dl/m/...) +``` + +**注意**:xmake 的增量链接不自动跟踪外部 `.a` 变化,Rust-only 改动后 C++ 目标需 `--rebuild`。 + +### 10.4 跨平台链接 + +| 平台 | 额外系统库 | +|---|---| +| Linux | pthread, dl, m, util | +| macOS | iconv, resolv, System + Security/Foundation/CoreServices frameworks | + +--- + +## 11. 测试体系 + +### 11.1 moebius 级单元测试 + +| 测试文件 | 内容 | +|---|---| +| [loro_ir_test.cpp](../../moebius/tests/Data/loro_ir_test.cpp) | IR 往返(compound/atomic/generic/嵌套/顺序)、扁平编解码自一致 | +| [loro_tmu_test.cpp](../../moebius/tests/Data/loro_tmu_test.cpp) | 236 个真实 .tmu 文档快照往返全过 | +| [loro_shadow_test.cpp](../../moebius/tests/Data/loro_shadow_test.cpp) | seed 往返、敲字精确(id 稳定)、退格、结构兜底、混合编辑会话、e2e 传播、双向合并、事件增量、远端 delta→mod | + +运行: +```bash +xmake b loro_ir_test && xmake r loro_ir_test +xmake b loro_tmu_test && xmake r loro_tmu_test +xmake b loro_shadow_test && xmake r loro_shadow_test +``` + +### 11.2 Rust 级单元测试 + +在 `3rdparty/mogan-loro-ffi/src/lib.rs` 的 `#[cfg(test)] mod tests` 中: +- `roundtrip_atomic_isolate`:IrNode → doc → export → import → read 一致。 +- `roundtrip_compound_isolate`:复合节点。 +- `roundtrip_binary_atomic`:PNG 头等非法 UTF-8 字节保留。 +- `full_ffi_byte_roundtrip`:Writer → encode → decode → Reader 全链路。 +- `live_seed_and_export` / `live_node_create_and_text`:live-doc API。 + +运行: +```bash +cd 3rdparty/mogan-loro-ffi && cargo test -- --nocapture +``` + +### 11.3 端到端集成测试 + +```bash +# 启动 relay +cd tools/loro-relay && node server.js + +# 启动编辑器 A(连 relay,开同一文件) +xmake b stem --rebuild && xmake r stem + +# 启动编辑器 B(同样连 relay,开同一文件) +xmake r stem + +# 在 A 中编辑,观察: +# - [Loro] Local update generated, size: N bytes(N 应远小于整篇) +# - B 中应实时看到 A 的编辑 +``` + +### 11.4 debug_loro 验证模式 + +```bash +xmake f -m debug --libloro=y --debug_loro=y +xmake b stem && xmake r stem +``` + +mirror_loro 每次镜像后断言 `buffer == Loro.to_tree()`,不一致时打印 `[loro-verify] MISMATCH`。 + +--- + +## 12. 已知限制与后续方向 + +### 12.1 已知限制 + +| 限制 | 说明 | 影响 | +|---|---|---| +| **id_map stale 表项** | 删除类操作后,被删 rep 的表项不清(需反向表)。除非 rep 地址复用,否则无害。 | 轻微内存泄漏 | +| **mixed JOIN** | X/Y 一个文本一个复合时,join_text 返回错误,复合分支可能处理不当。 | 罕见(raw_join 有 security code 处理) | +| **relay room.state** | 服务端只存最后一条消息(可能是小 delta),新加入者无法重建全态。 | 需服务端 merge 或用 snapshot | +| **WASM 目标** | loro-c 的 WASM 链路未验证。 | 需单独适配 | +| **UTF-8 位置精度** | mirror_mod 的 SPLIT/JOIN 对多字节字符的位置计算依赖 Loro 的 unicode 长度(在 Rust 侧通过 iter 计算)。 | 已在 Rust 侧正确处理 | +| **并发块级冲突** | 块级重 seed(兜底)会 churn 该块的 TreeID,同块并发编辑可能冲突。 | 顺序编辑无碍 | +| **性能** | 每次 mirror_mod 调用 FFI(跨 C/Rust 边界),高频敲字可能有开销。 | 未做性能基准 | + +### 12.2 后续方向 + +1. **服务端 Loro merge**:relay 用 `loro-crdt` npm 包维护合并后的文档状态,给新加入者发 snapshot。 +2. **光标共享**:利用 Loro 的 awareness + cursor 机制,实时显示其他协作者的光标位置。 +3. **离线编辑**:利用 Loro 的 time travel(checkout/version),支持离线编辑后重连自动合并。 +4. **减少 FFI 调用**:将多个连续 mod 批量传递给 Rust 侧处理(减少跨边界开销)。 +5. **持久化**:将 Loro snapshot 存入 .tmu 文件(或旁路文件),实现增量保存。 +6. **性能基准**:大文档(1000+ 段落)下的 mirror/export/import 性能测试。 +7. **WASM 支持**:验证 loro-c 的 WASM 构建,适配 Mogan 的 WASM 前端。 + +--- + +## 13. 关键文件索引 + +### 13.1 核心代码 + +| 文件 | 职责 | +|---|---| +| [3rdparty/mogan-loro-ffi/src/lib.rs](../../3rdparty/mogan-loro-ffi/src/lib.rs) | Rust C FFI shim(所有 `extern "C"` 函数) | +| [3rdparty/mogan-loro-ffi/Cargo.toml](../../3rdparty/mogan-loro-ffi/Cargo.toml) | Rust crate 配置(依赖本地 loro + generic-btree patch) | +| [moebius/Data/Convert/loro_ir.hpp](../../moebius/Data/Convert/loro_ir.hpp) | LoroIRNode 结构 + tree↔IR + 扁平编解码声明 | +| [moebius/Data/Convert/loro_ir.cpp](../../moebius/Data/Convert/loro_ir.cpp) | tree↔IR + 扁平编解码实现 | +| [moebius/Data/Convert/loro.hpp](../../moebius/Data/Convert/loro.hpp) | 公开接口 `tree_to_loro`/`loro_to_tree` + FFI C 声明 + `mogan_tree_id` | +| [moebius/Data/Convert/loro.cpp](../../moebius/Data/Convert/loro.cpp) | FFI 调用(encode/decode 空桩模式) | +| [moebius/Data/Convert/loro_shadow.hpp](../../moebius/Data/Convert/loro_shadow.hpp) | `loro_shadow` 类声明(seed/mirror_mod/import/apply) | +| [moebius/Data/Convert/loro_shadow.cpp](../../moebius/Data/Convert/loro_shadow.cpp) | **核心**:shadow doc 管理 + 9 种 mod 增量翻译 + diff + import_and_build | +| [src/Edit/Modify/edit_modify.hpp](../../src/Edit/Modify/edit_modify.hpp) | `edit_modify_rep` 成员(loro_doc/loro_seeded/守卫) | +| [src/Edit/Modify/edit_modify.cpp](../../src/Edit/Modify/edit_modify.cpp) | 编辑器挂载(mirror_loro/apply_remote/local_update_cb) | +| [src/Edit/editor.hpp](../../src/Edit/editor.hpp) | `mirror_loro`/`apply_remote`/`route_through_loro` 虚函数 | + +### 13.2 网络层 + +| 文件 | 职责 | +|---|---| +| [tools/loro-relay/server.js](../../tools/loro-relay/server.js) | JS WebSocket 中继服务器 | +| [tools/loro-relay/package.json](../../tools/loro-relay/package.json) | npm 配置 | +| [src/Plugins/ImGui/im_websocket.hpp](../../src/Plugins/ImGui/im_websocket.hpp) | WS 客户端接口 | +| [src/Plugins/ImGui/im_websocket.cpp](../../src/Plugins/ImGui/im_websocket.cpp) | libcurl WS 实现 | +| [src/Plugins/ImGui/im_gui.cpp](../../src/Plugins/ImGui/im_gui.cpp) | WS 客户端实例 + `g_loro_broadcast_update` 设置 | + +### 13.3 构建配置 + +| 文件 | 职责 | +|---|---| +| [xmake/options.lua](../../xmake/options.lua) | `libloro` / `debug_loro` 选项声明 | +| [moebius/xmake.lua](../../moebius/xmake.lua) | libmoebius:cargo 构建 + 链接 Rust staticlib | +| [xmake/targets/libmogan.lua](../../xmake/targets/libmogan.lua) | libmogan:LORO_ENABLED / LORO_DEBUG 宏 | + +### 13.4 测试 + +| 文件 | 内容 | +|---|---| +| [moebius/tests/Data/loro_ir_test.cpp](../../moebius/tests/Data/loro_ir_test.cpp) | IR 往返 + 扁平编解码 | +| [moebius/tests/Data/loro_tmu_test.cpp](../../moebius/tests/Data/loro_tmu_test.cpp) | 236 真实 .tmu 往返 | +| [moebius/tests/Data/loro_shadow_test.cpp](../../moebius/tests/Data/loro_shadow_test.cpp) | shadow 全量测试 | +| 3rdparty/mogan-loro-ffi/src/lib.rs (`mod tests`) | Rust 单元测试 | + +### 13.5 文档 + +| 文件 | 内容 | +|---|---| +| [devel/2030.md](2030.md) | 任务文档(What/Why/How + 分阶段计划) | +| **本文档** | 完整技术文档 | + +--- + +## 14. 常见问题排查 + +### Q: 编辑器编译后 Loro 功能没生效? + +**A**:检查 `xmake f` 是否带了 `--libloro=y`。`libmogan` 和 `libmoebius` 都需要 `LORO_ENABLED` 宏才会编译真实代码。确认 `xmake f --libloro=y` 后重新 `xmake b stem --rebuild`。 + +### Q: Rust 改了 FFI 代码后 C++ 测试没更新? + +**A**:xmake 的增量链接不自动跟踪外部 `.a` 变化。Rust-only 改动后 C++ 目标需 `--rebuild`:`xmake b stem --rebuild`。 + +### Q: `[loro-verify] MISMATCH` 告警? + +**A**:说明某个 mod 的增量翻译有 bug(mirror 后 buffer != Loro)。开 `--debug_loro=y`,看哪个 mod 之后出现 MISMATCH,检查对应的翻译分支。常见原因:身份表未正确维护(漏映射 t1/merged)、path_up 顶层处理遗漏。 + +### Q: 两个编辑器打开同一文件但编辑没同步? + +**A**:检查: +1. relay 是否在运行(`node tools/loro-relay/server.js`)。 +2. 两个编辑器是否加入了同一 room(文件名一致)。 +3. 日志中是否有 `[Loro] Local update generated` 和 `[WS] Sending` + `[Loro] Applying remote update`。 +4. shadow 是否已 seed(检查 `[Loro]` 日志中是否有 seed 相关消息)。 + +### Q: delta 字节数太大(接近整篇文档)? + +**A**:某个 mod 走了块级兜底。检查 `[loro-mod]` 日志中哪个 mod 类型出现,对应到 `mirror_mod` 的分支: +- 确认该 mod 类型有精确翻译分支。 +- 检查身份表是否命中(`id_map->contains` 是否为 true)。 +- 检查 path_up 顶层(空 path)是否正确 fallback 到 root_id。 + +### Q: 接收端 buffer 更新后没有重排版? + +**A**:`apply_remote` 末尾需调用 `notify_change(THE_TREE)` + `typeset_invalidate_all()` 触发重排版。确认这两行存在。 + +### Q: 链接错误 `undefined symbols`(the_et/obtain_ip 等)? + +**A**:moebius 单元测试不能使用 `raw_apply`(它引用 libmogan 符号如 `the_et`/`obtain_ip`)。测试中用手工赋值代替 `raw_apply`,或者把测试移到 libmogan 级别。 + +--- + +> **本文档持续更新。如有疑问或发现过时内容,请更新对应章节并注明日期。** diff --git a/moebius/Data/Convert/loro.cpp b/moebius/Data/Convert/loro.cpp new file mode 100644 index 0000000000..32d578afec --- /dev/null +++ b/moebius/Data/Convert/loro.cpp @@ -0,0 +1,71 @@ +/** \file loro.cpp + * \copyright GPLv3 + * \details tree <-> Loro CRDT snapshot 的 FFI 胶水层。 + * 扁平编解码在 loro_ir.{hpp,cpp}(loro_ir_encode/decode,与 + * 3rdparty/mogan-loro-ffi/src/lib.rs 一致)。 + * LORO_ENABLED 关时为空桩(见 loro.hpp)。 + * \author Jim Zhou + * \date 2026 + */ + +#include "loro.hpp" + +#include "loro_ir.hpp" + +#include +#include + +#ifdef LORO_ENABLED +// mogan-loro-ffi 暴露的 C ABI(见 3rdparty/mogan-loro-ffi/src/lib.rs) +extern "C" { +int32_t mogan_loro_encode (const uint8_t* ir, size_t ir_len, uint8_t** out, + size_t* out_len); +int32_t mogan_loro_decode (const uint8_t* snap, size_t snap_len, uint8_t** out, + size_t* out_len); +void mogan_loro_free (uint8_t* ptr, size_t len); +} +#endif + +string +tree_to_loro (tree t) { +#ifdef LORO_ENABLED + string ir_bytes= loro_ir_encode (tree_to_loro_ir (t)); + + uint8_t* out = nullptr; + size_t out_len= 0; + int rc= + mogan_loro_encode (reinterpret_cast (ir_bytes.begin ()), + (size_t) N (ir_bytes), &out, &out_len); + if (rc != 0 || out == nullptr) { + if (out) mogan_loro_free (out, out_len); + return string (); + } + string snapshot ((const char*) out, (int) out_len); + mogan_loro_free (out, out_len); + return snapshot; +#else + (void) t; + return string (); +#endif +} + +tree +loro_to_tree (string snapshot) { +#ifdef LORO_ENABLED + uint8_t* out = nullptr; + size_t out_len= 0; + int rc= + mogan_loro_decode (reinterpret_cast (snapshot.begin ()), + (size_t) N (snapshot), &out, &out_len); + if (rc != 0 || out == nullptr) { + if (out) mogan_loro_free (out, out_len); + return tree (""); + } + string ir_bytes ((const char*) out, (int) out_len); + mogan_loro_free (out, out_len); + return loro_ir_to_tree (loro_ir_decode (ir_bytes)); +#else + (void) snapshot; + return tree (""); +#endif +} diff --git a/moebius/Data/Convert/loro.hpp b/moebius/Data/Convert/loro.hpp new file mode 100644 index 0000000000..40db434c66 --- /dev/null +++ b/moebius/Data/Convert/loro.hpp @@ -0,0 +1,87 @@ +/** \file loro.hpp + * \copyright GPLv3 + * \details body 树 <-> Loro CRDT snapshot 的公开接口。 + * + * 链路:tree --(loro_ir)--> loro_ir_node --(扁平编码)--> bytes + * --(mogan-loro-ffi)--> LoroDoc::export(snapshot)。反向同理。 + * 依赖 mogan-loro-ffi 静态库(见 3rdparty/mogan-loro-ffi)。 + * + * 编译开关 LORO_ENABLED(xmake option libloro): + * - 开启:调用真实 FFI(mogan_loro_encode/decode),链 + * mogan-loro-ffi。 + * - 关闭(默认):tree_to_loro/loro_to_tree 为空桩,返回空,无 Rust + * 依赖, 保证主构建不被未接好的 FFI 拖垮。 + * \author Jim Zhou + * \date 2026 + */ + +#ifndef LORO_H +#define LORO_H + +#include "string.hpp" +#include "tree.hpp" + +/** @brief body 树 -> Loro CRDT snapshot 字节(包成 lolly + * string)。失败返回空串。 */ +string tree_to_loro (tree t); + +/** @brief Loro CRDT snapshot 字节 -> body 树。失败返回空树。 */ +tree loro_to_tree (string snapshot); + +/****************************************************************************** + * live-doc API(Phase 2:增量 op 镜像) + * + * 仅在 LORO_ENABLED 下声明。对应 3rdparty/mogan-loro-ffi/src/lib.rs 的 extern + *"C"。 + *****************************************************************************/ +#ifdef LORO_ENABLED +#include +#include + +/// TreeID 的 C 表示(与 Rust 侧 #[repr(C)] MoganTreeId 布局一致,按值跨界) +struct mogan_tree_id { + uint64_t peer; + int32_t counter; +}; + +/// local-update 事件回调:本地编辑产生增量 delta 时回调 cb(user_data, bytes, +/// len) +typedef void (*mogan_local_update_cb) (void* user_data, const uint8_t* bytes, + size_t len); + +extern "C" { +// doc 句柄为不透明 void*(即 Rust 的 *mut LoroDoc) +void* mogan_loro_doc_new (); +void mogan_loro_doc_free (void* doc); +int32_t mogan_loro_doc_seed (void* doc, const uint8_t* ir, size_t ir_len); +int32_t mogan_loro_doc_export (void* doc, uint8_t** out, size_t* out_len); +int32_t mogan_loro_doc_to_ir (void* doc, uint8_t** out, size_t* out_len); +int32_t mogan_loro_doc_to_ir_with_ids (void* doc, uint8_t** out, + size_t* out_len); +int32_t mogan_loro_doc_import (void* doc, const uint8_t* bytes, size_t len); +void mogan_loro_doc_commit (void* doc); +int32_t mogan_loro_doc_on_local_update (void* doc, mogan_local_update_cb cb, + void* user_data); +mogan_tree_id mogan_loro_node_create (void* doc, mogan_tree_id parent, + uint32_t index, uint8_t kind, + const uint8_t* label, size_t label_len); +int32_t mogan_loro_node_delete (void* doc, mogan_tree_id id); +int32_t mogan_loro_node_children (void* doc, mogan_tree_id parent, + uint8_t** out, size_t* out_len); +int32_t mogan_loro_node_mov (void* doc, mogan_tree_id target, + mogan_tree_id parent, uint32_t index); +int32_t mogan_loro_node_set_label (void* doc, mogan_tree_id id, + const uint8_t* label, size_t len); +int32_t mogan_loro_node_set_binary (void* doc, mogan_tree_id id, + const uint8_t* bytes, size_t len); +int32_t mogan_loro_node_text_insert (void* doc, mogan_tree_id id, uint32_t pos, + const uint8_t* bytes, size_t len); +int32_t mogan_loro_node_text_delete (void* doc, mogan_tree_id id, uint32_t pos, + uint32_t len); +int32_t mogan_loro_node_join_text (void* doc, mogan_tree_id x_id, + mogan_tree_id y_id); +void mogan_loro_free (uint8_t* ptr, size_t len); +} +#endif // LORO_ENABLED + +#endif // defined LORO_H diff --git a/moebius/Data/Convert/loro_ir.cpp b/moebius/Data/Convert/loro_ir.cpp new file mode 100644 index 0000000000..96fa517f29 --- /dev/null +++ b/moebius/Data/Convert/loro_ir.cpp @@ -0,0 +1,140 @@ +/** \file loro_ir.cpp + * \copyright GPLv3 + * \details 实现 tree <-> LoroIRNode 的依赖无关双向映射。 + * 标签走 moebius 的 name<->op 注册表:导出用 as_string(tree_label) + * 取名字, 导入用 make_tree_label(name) 反查(含扩展标签)。generic + * 节点(op<0) 用十进制存取(as_string 支持 int、as_int + * 支持负号),保证忠实还原。 + * + * 本文件不依赖 loro-c。后续 loro.cpp 的 FFI 薄层只需把 LoroIRNode + * <-> LoroDoc 字节。 + * \author Jim Zhou + * \date 2026 + */ + +#include "loro_ir.hpp" + +#include "tree_helper.hpp" + +#include + +#include + +// generic 节点 label 前缀,op 以十进制存于其后 +static const string GENERIC_PREFIX= "generic:"; + +loro_ir_node +tree_to_loro_ir (tree t) { + loro_ir_node node; + if (is_atomic (t)) { + node.kind= LORO_ATOMIC; + node.text= t->label; + } + else { + int n= N (t); + if (is_compound (t)) { + node.kind = LORO_COMPOUND; + node.label= as_string (L (t)); // op -> 标签名 + } + else { // is_generic: op<0 + node.kind = LORO_GENERIC; + node.label= GENERIC_PREFIX * as_string ((int) L (t)); + } + for (int i= 0; i < n; i++) + node.children << tree_to_loro_ir (t[i]); + } + return node; +} + +tree +loro_ir_to_tree (loro_ir_node node) { + if (node.kind == LORO_ATOMIC) return tree (node.text); + + int op; + if (node.kind == LORO_COMPOUND) + op= (int) moebius::make_tree_label (node.label); // 标签名 -> op(含扩展) + else // LORO_GENERIC:label 形如 "generic:" + op= as_int (node.label (N (GENERIC_PREFIX), N (node.label))); + + int n= N (node.children); + tree r (op, n); + for (int i= 0; i < n; i++) + r[i]= loro_ir_to_tree (node.children[i]); + return r; +} + +/****************************************************************************** + * 扁平二进制编解码(与 3rdparty/mogan-loro-ffi/src/lib.rs 逐字节一致,小端) + ******************************************************************************/ + +static void +put_u32 (string& s, uint32_t v) { + s << (char) (v & 0xff); + s << (char) ((v >> 8) & 0xff); + s << (char) ((v >> 16) & 0xff); + s << (char) ((v >> 24) & 0xff); +} + +static void +put_str (string& s, string t) { + int n= N (t); + put_u32 (s, (uint32_t) n); + for (int i= 0; i < n; i++) + s << t[i]; +} + +static void +encode_node_into (string& s, loro_ir_node node) { + s << (char) node.kind; + put_str (s, node.label); + put_str (s, node.text); + int n= N (node.children); + put_u32 (s, (uint32_t) n); + for (int i= 0; i < n; i++) + encode_node_into (s, node.children[i]); +} + +string +loro_ir_encode (loro_ir_node node) { + string s; + encode_node_into (s, node); + return s; +} + +static uint32_t +get_u32_dec (string& b, int& pos) { + uint32_t v= (uint32_t) (unsigned char) b[pos] | + ((uint32_t) (unsigned char) b[pos + 1] << 8) | + ((uint32_t) (unsigned char) b[pos + 2] << 16) | + ((uint32_t) (unsigned char) b[pos + 3] << 24); + pos+= 4; + return v; +} + +static string +get_str_dec (string& b, int& pos) { + uint32_t n= get_u32_dec (b, pos); + string r; + for (uint32_t i= 0; i < n; i++) + r << b[pos + i]; + pos+= n; + return r; +} + +static loro_ir_node +decode_node_from (string& b, int& pos) { + loro_ir_node node; + node.kind = (loro_node_kind) (unsigned char) b[pos++]; + node.label= get_str_dec (b, pos); + node.text = get_str_dec (b, pos); + uint32_t n= get_u32_dec (b, pos); + for (uint32_t i= 0; i < n; i++) + node.children << decode_node_from (b, pos); + return node; +} + +loro_ir_node +loro_ir_decode (string bytes) { + int pos= 0; + return decode_node_from (bytes, pos); +} diff --git a/moebius/Data/Convert/loro_ir.hpp b/moebius/Data/Convert/loro_ir.hpp new file mode 100644 index 0000000000..bde0f57eca --- /dev/null +++ b/moebius/Data/Convert/loro_ir.hpp @@ -0,0 +1,65 @@ +/** \file loro_ir.hpp + * \copyright GPLv3 + * \details 依赖无关的中层表示(IR):把 moebius 文档树映射到一个中性的 + * Loro 风格节点树,作为 tree <-> Loro CRDT 转换的核心。本层不依赖 + * loro-c,可独立编译与 round-trip 测试;真正调用 loro-c 的 FFI 薄层 + * (公开 tree_to_loro / loro_to_tree)见 loro.hpp / loro.cpp。 + * + * IR 镜像 LoroTree 的「每个节点带 label + 有序子节点」模型,使 + * tree <-> IR 的映射(含标签 int<->name 互转)可在不依赖 loro-c 的 + * 情况下验证忠实性,后续 FFI 层只需把 IR <-> LoroDoc 字节。 + * \author Jim Zhou + * \date 2026 + */ + +#ifndef LORO_IR_H +#define LORO_IR_H + +#include "array.hpp" +#include "tree.hpp" + +/** + * @brief Loro 风格的中性节点表示,对应一个未来的 LoroTree 节点。 + * + * 数值必须与 3rdparty/mogan-loro-ffi/src/lib.rs 的 KIND_* 常量逐字一致 + * (扁平编码里 kind 直接以裸字节跨 FFI 传递)。 + * + * - LORO_ATOMIC:对应 moebius 原子节点(op==0,字符串叶子)。text 存其内容。 + * - LORO_COMPOUND:对应 moebius 复合节点(op>0)。label 存标签名字字符串 + * (如 "document"、"paragraph"),children 为有序子节点。 + * - LORO_GENERIC:对应 moebius generic 节点(op<0,罕见)。label 形如 + * "generic:",children 为有序子节点。 + */ +enum loro_node_kind { LORO_ATOMIC, LORO_COMPOUND, LORO_GENERIC }; + +struct loro_ir_node { + loro_node_kind kind; //!< 节点类别 + string label; //!< COMPOUND: 标签名;GENERIC: "generic:" + string text; //!< ATOMIC: 字符串叶子内容 + array children; //!< COMPOUND / GENERIC: 有序子节点 +}; + +/** @brief moebius tree -> LoroIRNode(深度复制)。 */ +loro_ir_node tree_to_loro_ir (tree t); + +/** @brief LoroIRNode -> moebius tree(深度复制)。按值传递:lolly 的 + * string/array 是 COW 句柄,且其 operator()/operator[] 无 const + * 重载,按值传可让形参可变。 */ +tree loro_ir_to_tree (loro_ir_node node); + +/** + * @name 扁平二进制编解码(与 3rdparty/mogan-loro-ffi/src/lib.rs + * 的格式逐字节一致) + * + * 格式(小端): + * node := kind:u8 label_len:u32 label:bytes text_len:u32 text:bytes + * n_children:u32 node × n_children + * 暴露出来便于在 loro_ir_test 里独立验证 C++ 编解码(不依赖 + * LORO_ENABLED/Rust)。 + */ +///@{ +string loro_ir_encode (loro_ir_node node); +loro_ir_node loro_ir_decode (string bytes); +///@} + +#endif // defined LORO_IR_H diff --git a/moebius/Data/Convert/loro_shadow.cpp b/moebius/Data/Convert/loro_shadow.cpp new file mode 100644 index 0000000000..abab5b9ba9 --- /dev/null +++ b/moebius/Data/Convert/loro_shadow.cpp @@ -0,0 +1,664 @@ +/** \file loro_shadow.cpp + * \copyright GPLv3 + * \details Phase 2 shadow LoroDoc 实现:seed(树->live doc + + * 身份表)、export、to_tree。 + * \author Jim Zhou + * \date 2026 + */ + +#include "loro_shadow.hpp" + +#ifdef LORO_ENABLED + +#include "basic.hpp" // pointer, hash(pointer) +#include "loro_ir.hpp" // loro_ir_decode, loro_ir_to_tree, LORO_* kind +#include "tree_helper.hpp" // L, as_string, inside +#include // make_tree_label + +#include + +// tree_rep* 的 hash(hashmap key 用),仿 observer::hash((pointer)rep) +inline int +hash (tree_rep* p) { + return hash ((pointer) p); +} + +/****************************************************************************** + * 构造与析构 + ******************************************************************************/ + +loro_shadow_rep::loro_shadow_rep () + : doc (mogan_loro_doc_new ()), id_map (mogan_tree_id{0, 0}), + root_id (mogan_tree_id{0, 0}), _update_cb (nullptr), + _update_user_data (nullptr) {} + +loro_shadow_rep::~loro_shadow_rep () { + if (doc) mogan_loro_doc_free (doc); +} + +loro_shadow::loro_shadow () : rep (tm_new ()) {} + +/****************************************************************************** + * seed:递归构建 live doc + 填充身份表 + ******************************************************************************/ + +mogan_tree_id +loro_shadow_rep::seed_node (tree t, mogan_tree_id parent, uint32_t index) { + uint8_t kind; + string label; + if (is_atomic (t)) kind= LORO_ATOMIC; + else if (is_compound (t)) { + kind = LORO_COMPOUND; + label= as_string (L (t)); + } + else { + kind = LORO_GENERIC; + label= "generic:" * as_string ((int) L (t)); + } + + const uint8_t* lp= N (label) > 0 + ? reinterpret_cast (label.begin ()) + : nullptr; + mogan_tree_id id= + mogan_loro_node_create (doc, parent, index, kind, lp, (size_t) N (label)); + id_map (inside (t))= id; // 记录身份 + + if (is_atomic (t)) { + const uint8_t* tp= reinterpret_cast (t->label.begin ()); + size_t tn= (size_t) N (t->label); + // 先尝试 LoroText(合法 UTF-8);失败(非 UTF-8,如图片)回退 Binary + if (mogan_loro_node_text_insert (doc, id, 0, tp, tn) != 0) + mogan_loro_node_set_binary (doc, id, tp, tn); + } + else { + int n= N (t); + for (int i= 0; i < n; i++) + seed_node (t[i], id, (uint32_t) i); + } + return id; +} + +void +loro_shadow_rep::seed (tree root) { + mogan_tree_id root_parent= {UINT64_MAX, 0}; // Root 哨兵 + root_id = seed_node (root, root_parent, 0); +} + +// 并行遍历 buffer 和增强 IR(shadow 的当前状态),把 buffer 的 rep 关联到 +// shadow 的 TreeID。 如果结构匹配 → 填充 id_map + root_id,返回 true。否则返回 +// false(调用方 fallback 到 seed)。 +namespace { +bool +sync_walk (tree t, string& ir, int& pos, mogan_tree_id& root_id, + hashmap& id_map) { + if (pos + 12 > N (ir)) return false; + auto get_u32= [&] () -> uint32_t { + uint32_t v= (uint32_t) (unsigned char) ir[pos] | + ((uint32_t) (unsigned char) ir[pos + 1] << 8) | + ((uint32_t) (unsigned char) ir[pos + 2] << 16) | + ((uint32_t) (unsigned char) ir[pos + 3] << 24); + pos+= 4; + return v; + }; + auto get_str= [&] () -> string { + uint32_t n= get_u32 (); + string r; + for (uint32_t i= 0; i < n; i++) + r << ir[pos + i]; + pos+= n; + return r; + }; + // TreeID + uint64_t peer= 0; + for (int b= 0; b < 8; b++) + peer|= ((uint64_t) (unsigned char) ir[pos + b]) << (8 * b); + pos+= 8; + mogan_tree_id tid{peer, (int32_t) get_u32 ()}; + if (root_id.peer == 0) root_id= tid; + // kind/label/text/n_children + pos++; // kind + get_str (); // label + get_str (); // text + uint32_t n = get_u32 (); // n_children + id_map (inside (t))= tid; + int nc = is_atomic (t) ? 0 : N (t); + if (nc != (int) n) return false; // 结构不匹配 + for (int i= 0; i < nc; i++) + if (!sync_walk (t[i], ir, pos, root_id, id_map)) return false; + return true; +} +} // namespace + +bool +loro_shadow_rep::sync_id_map_from_shadow (tree buffer) { + uint8_t* out = nullptr; + size_t out_len= 0; + if (mogan_loro_doc_to_ir_with_ids (doc, &out, &out_len) != 0 || + out == nullptr || out_len == 0) { + if (out) mogan_loro_free (out, out_len); + return false; // shadow 为空 + } + string ir ((const char*) out, (int) out_len); + mogan_loro_free (out, out_len); + id_map = hashmap (mogan_tree_id{0, 0}); + root_id= mogan_tree_id{0, 0}; + int pos= 0; + if (!sync_walk (buffer, ir, pos, root_id, id_map)) return false; + return root_id.peer != 0; +} + +/****************************************************************************** + * mirror_mod:把 modification 镜像到 live doc(增量 op,消除整树重 seed) + ******************************************************************************/ + +array +loro_shadow_rep::node_children (mogan_tree_id parent) { + array kids; + uint8_t* out = nullptr; + size_t out_len= 0; + if (mogan_loro_node_children (doc, parent, &out, &out_len) == 0 && + out != nullptr) { + size_t n= out_len / 12; + for (size_t i= 0; i < n; i++) { + uint64_t peer= 0; + for (int b= 0; b < 8; b++) + peer|= ((uint64_t) out[i * 12 + b]) << (8 * b); + int32_t counter= (int32_t) ((uint32_t) out[i * 12 + 8] | + ((uint32_t) out[i * 12 + 9] << 8) | + ((uint32_t) out[i * 12 + 10] << 16) | + ((uint32_t) out[i * 12 + 11] << 24)); + kids << mogan_tree_id{peer, counter}; + } + mogan_loro_free (out, out_len); + } + return kids; +} + +void +loro_shadow_rep::mirror_mod (tree doc_root, modification mod) { + bool mirrored= false; + path rp_mod = root (mod); + + if (mod->k == MOD_INSERT || mod->k == MOD_REMOVE) { + if (has_subtree (doc_root, rp_mod)) { + tree& parent= subtree (doc_root, rp_mod); + if (is_atomic (parent) && id_map->contains (inside (parent))) { + // 文本原子 INSERT/REMOVE → LoroText(敲字/退格,rep 稳定) + mogan_tree_id id= id_map (inside (parent)); + if (mod->k == MOD_INSERT) { + string s= mod->t->label; + mogan_loro_node_text_insert ( + doc, id, (uint32_t) index (mod), + reinterpret_cast (s.begin ()), (size_t) N (s)); + } + else { + mogan_loro_node_text_delete (doc, id, (uint32_t) index (mod), + (uint32_t) argument (mod)); + } + mirrored= true; + } + else if (is_compound (parent) && id_map->contains (inside (parent))) { + // 复合子节点 INSERT/REMOVE → node_create / node_delete(块增删) + mogan_tree_id pid= id_map (inside (parent)); + if (mod->k == MOD_INSERT) { + int pos= index (mod); + path child_path= + rp_mod * pos; // 新子节点在 buffer 的位置(post-apply) + if (has_subtree (doc_root, child_path)) { + tree& child= subtree (doc_root, child_path); + seed_node (child, pid, (uint32_t) pos); // 建子树 + 映射身份 + mirrored= true; + } + } + else { + int pos= index (mod); + int nr = argument (mod); + array kids= + node_children (pid); // LoroDoc 当前子节点(镜像前状态) + for (int j= pos + nr - 1; j >= pos; j--) + if (j < N (kids)) mogan_loro_node_delete (doc, kids[j]); + mirrored= true; + } + } + } + } + else if (mod->k == MOD_ASSIGN_NODE) { + // 改标签(格式/样式变更)→ node_set_label(rep 稳定) + if (has_subtree (doc_root, rp_mod)) { + tree& node= subtree (doc_root, rp_mod); + if (id_map->contains (inside (node))) { + mogan_tree_id id = id_map (inside (node)); + string lab= as_string (L (mod)); // 新 op 名 + mogan_loro_node_set_label ( + doc, id, reinterpret_cast (lab.begin ()), + (size_t) N (lab)); + mirrored= true; + } + } + } + else if (mod->k == MOD_SPLIT) { + // 拆分:P=pos 的子节点 X 拆成 t1(P[pos]) + t2(P[pos+1]) + int pos= index (mod); + int at = argument (mod); + if (has_subtree (doc_root, rp_mod) && + id_map->contains (inside (subtree (doc_root, rp_mod)))) { + mogan_tree_id pid = id_map (inside (subtree (doc_root, rp_mod))); + array kids= node_children (pid); + if (pos < N (kids)) { + mogan_tree_id x_id= kids[pos]; + // raw_split 把 ref[pos] 替换成新树 t1(左半)+ t2(右半)。Loro 里 X + // 截断/留前半 即为 t1,故把 buffer 的 t1(在 rp_mod*pos)映射到 X 的 + // TreeID,供后续 mod 解析。 + if (has_subtree (doc_root, rp_mod * pos)) + id_map (inside (subtree (doc_root, rp_mod * pos)))= x_id; + path t2p= rp_mod * (pos + 1); + if (has_subtree (doc_root, t2p)) { + tree& t2= subtree (doc_root, t2p); + if (is_atomic (t2)) { + // 原子拆分:截断 X 的文本到 at,建 t2(原子)放 t2 的文本 + mogan_loro_node_text_delete (doc, x_id, (uint32_t) at, + (uint32_t) N (t2->label)); + mogan_tree_id y_id= mogan_loro_node_create ( + doc, pid, (uint32_t) (pos + 1), LORO_ATOMIC, nullptr, 0); + const uint8_t* tp= + reinterpret_cast (t2->label.begin ()); + mogan_loro_node_text_insert (doc, y_id, 0, tp, + (size_t) N (t2->label)); + id_map (inside (t2))= y_id; + mirrored = true; + } + else { + // 复合拆分:建 t2(复合),把 X 的 [at..] 子节点 mov 到 t2 + string lab = as_string (L (t2)); + mogan_tree_id y_id= mogan_loro_node_create ( + doc, pid, (uint32_t) (pos + 1), LORO_COMPOUND, + reinterpret_cast (lab.begin ()), + (size_t) N (lab)); + int m= N (t2); + for (int i= 0; i < m; i++) { + tree& c= subtree (doc_root, t2p * i); + if (id_map->contains (inside (c))) + mogan_loro_node_mov (doc, id_map (inside (c)), y_id, + (uint32_t) i); + } + id_map (inside (t2))= y_id; + mirrored = true; + } + } + } + } + } + else if (mod->k == MOD_INSERT_NODE) { + // 包一层:rp_mod=包装 W 的路径,argument(mod)=被包节点在 W 内的位置 + int pos= argument (mod); + if (has_subtree (doc_root, rp_mod)) { + tree& W = subtree (doc_root, rp_mod); + path pp= path_up (rp_mod); + int wi= last_item (rp_mod); + // W 的父 TreeID:顶层(path_up 为空)→ root_id;否则 id_map[父] + mogan_tree_id pid; + bool pid_ok= false; + if (is_nil (pp)) { + pid = root_id; + pid_ok= (root_id.peer != 0); + } + else if (has_subtree (doc_root, pp) && + id_map->contains (inside (subtree (doc_root, pp)))) { + pid = id_map (inside (subtree (doc_root, pp))); + pid_ok= true; + } + path rfp= rp_mod * pos; + if (pid_ok && has_subtree (doc_root, rfp) && + id_map->contains (inside (subtree (doc_root, rfp)))) { + mogan_tree_id rf_id= id_map (inside (subtree (doc_root, rfp))); + string lab = as_string (L (W)); + mogan_tree_id w_id = mogan_loro_node_create ( + doc, pid, (uint32_t) wi, LORO_COMPOUND, + reinterpret_cast (lab.begin ()), (size_t) N (lab)); + mogan_loro_node_mov (doc, rf_id, w_id, (uint32_t) pos); + id_map (inside (W))= w_id; + mirrored = true; + } + } + } + else if (mod->k == MOD_REMOVE_NODE) { + // 拆一层:rp_mod=包装 W 的路径,index(mod)=被提升的子节点在 W 内的位置 + int pos= index (mod); + path pp = path_up (rp_mod); + int wi = last_item (rp_mod); + mogan_tree_id pid; + bool pid_ok= false; + if (is_nil (pp)) { + pid = root_id; + pid_ok= (root_id.peer != 0); + } + else if (has_subtree (doc_root, pp) && + id_map->contains (inside (subtree (doc_root, pp)))) { + pid = id_map (inside (subtree (doc_root, pp))); + pid_ok= true; + } + if (pid_ok && has_subtree (doc_root, rp_mod)) { + array kids= node_children (pid); + if (wi < N (kids)) { + mogan_tree_id w_id= kids[wi]; + mogan_tree_id c_id= + id_map->contains (inside (subtree (doc_root, rp_mod))) + ? id_map (inside (subtree (doc_root, rp_mod))) + : mogan_tree_id{0, 0}; + if (c_id.peer != 0) { + mogan_loro_node_mov (doc, c_id, pid, + (uint32_t) wi); // 子提到 W 的位置 + mogan_loro_node_delete (doc, w_id); // 删空包装 + mirrored= true; + } + } + } + } + + else if (mod->k == MOD_ASSIGN) { + // 替换子树:rp_mod=节点路径。删旧节点(Loro 在父/wi 的子)+ 从 buffer + // 当前(=mod->t)重建。 + if (!is_nil (rp_mod) && has_subtree (doc_root, rp_mod)) { + tree& node= subtree (doc_root, rp_mod); // post-assign = mod->t + path pp = path_up (rp_mod); + int wi = last_item (rp_mod); + mogan_tree_id pid; + bool pid_ok= false; + if (is_nil (pp)) { + pid = root_id; + pid_ok= (root_id.peer != 0); + } + else if (has_subtree (doc_root, pp) && + id_map->contains (inside (subtree (doc_root, pp)))) { + pid = id_map (inside (subtree (doc_root, pp))); + pid_ok= true; + } + if (pid_ok) { + array kids= node_children (pid); + if (wi < N (kids)) mogan_loro_node_delete (doc, kids[wi]); // 删旧 + seed_node (node, pid, (uint32_t) wi); // 建新(映射身份) + mirrored= true; + } + } + } + else if (mod->k == MOD_JOIN) { + // 合并 P[pos]+P[pos+1] → P[pos](X 保留合并内容,Y 删除) + int pos= index (mod); + if (has_subtree (doc_root, rp_mod) && + id_map->contains (inside (subtree (doc_root, rp_mod)))) { + mogan_tree_id pid = id_map (inside (subtree (doc_root, rp_mod))); + array kids= node_children (pid); + if (pos + 1 < N (kids)) { + mogan_tree_id x_id= kids[pos]; + mogan_tree_id y_id= kids[pos + 1]; + if (mogan_loro_node_join_text (doc, x_id, y_id) != 0) { + // 复合 join:mov Y 的子节点到 X 末尾 + delete Y + array x_kids= node_children (x_id); + array y_kids= node_children (y_id); + int base = N (x_kids); + int m = N (y_kids); + for (int i= 0; i < m; i++) + mogan_loro_node_mov (doc, y_kids[i], x_id, (uint32_t) (base + i)); + mogan_loro_node_delete (doc, y_id); + } + // 合并后 buffer 的 P*pos(merged)对应 X(X 保留了合并内容) + if (has_subtree (doc_root, rp_mod * pos)) + id_map (inside (subtree (doc_root, rp_mod * pos)))= x_id; + mirrored= true; + } + } + } + if (!mirrored) { + // 兜底(INSERT_NODE/REMOVE_NODE/ASSIGN/SPLIT/JOIN + // 等,回车等复杂结构改动): + // 块级重 seed——只删+重建包含改动的 buffer + // 根直接子节点(段落/块),而非整篇。 回车的 5 个 mod 序列各重 seed + // 一次所在块(块级,远小于整篇),保 peer 血统。 + path p = mod->p; + bool reseeded= false; + if (!is_nil (p) && root_id.peer != 0) { + int block_idx = p->item; + path block_path= path (block_idx); + if (has_subtree (doc_root, block_path) && + id_map->contains (inside (subtree (doc_root, block_path)))) { + mogan_tree_id block_id= + id_map (inside (subtree (doc_root, block_path))); + mogan_loro_node_delete (doc, block_id); + seed_node (subtree (doc_root, block_path), root_id, + (uint32_t) block_idx); + reseeded= true; + } + } + if (!reseeded) { + // 连块级都定位不了(如改动在 buffer 根本身,或身份缺失)→ 整树重 + // seed(保血统)。 + if (root_id.peer != 0 || root_id.counter != 0) + mogan_loro_node_delete (doc, root_id); + id_map= hashmap (mogan_tree_id{0, 0}); + seed (doc_root); + } + } + // 显式提交:同步触发 local-update 事件(增量同步)。 + mogan_loro_doc_commit (doc); +} + +/****************************************************************************** + * 导出 / 读取 + ******************************************************************************/ + +string +loro_shadow_rep::export_snapshot () { + uint8_t* out = nullptr; + size_t out_len= 0; + if (mogan_loro_doc_export (doc, &out, &out_len) != 0 || out == nullptr) { + if (out) mogan_loro_free (out, out_len); + return string (); + } + string s ((const char*) out, (int) out_len); + mogan_loro_free (out, out_len); + return s; +} + +bool +loro_shadow_rep::import_data (string bytes) { + return mogan_loro_doc_import ( + doc, reinterpret_cast (bytes.begin ()), + (size_t) N (bytes)) == 0; +} + +void +loro_shadow_rep::on_local_update (mogan_local_update_cb cb, void* user_data) { + _update_cb = cb; + _update_user_data= user_data; + mogan_loro_doc_on_local_update (doc, cb, user_data); +} + +// 字符级 diff:把 before 文本变到 after 文本所需的 INSERT/REMOVE +// mod(前缀/后缀对齐)。 p 是该原子在 buffer 中的路径;mod 用 +// mod_insert/remove(p, pos, ...)。 +namespace { +void +emit_text_diff (path p, string b, string a, list& mods) { + int bn= N (b), an= N (a); + int pre= 0; + while (pre < bn && pre < an && b[pre] == a[pre]) + pre++; + int suf= 0; + while (suf < bn - pre && suf < an - pre && b[bn - 1 - suf] == a[an - 1 - suf]) + suf++; + int rm_len = bn - pre - suf; // before 中被删的长度 + int ins_len= an - pre - suf; // after 中新增的长度 + if (rm_len > 0) mods= mods * mod_remove (p, pre, rm_len); + if (ins_len > 0) { + string ins= a (pre, pre + ins_len); + mods = mods * mod_insert (p, pre, tree (ins)); + } +} + +// 并行遍历 before/after(假设同结构):文本差异 → 字符 diff mod;结构差异 → +// 返回 false。 +bool +diff_walk (tree b, tree a, path base, list& mods) { + if (b == a) return true; // exactly the same + + if (is_atomic (b) && is_atomic (a)) { + if (b->label != a->label) emit_text_diff (base, b->label, a->label, mods); + return true; + } + + if (is_compound (b) && is_compound (a) && L (b) == L (a)) { + int bn= N (b), an= N (a); + if (bn == an) { + for (int i= 0; i < bn; i++) { + if (!diff_walk (b[i], a[i], base * path (i), mods)) { + // If a child differs structurally in an un-diffable way, we assign + // that child + mods= mods * mod_assign (base * path (i), a[i]); + } + } + return true; + } + else { + // Find common prefix + int pre= 0; + while (pre < bn && pre < an && b[pre] == a[pre]) + pre++; + // Find common suffix + int suf= 0; + while (suf < bn - pre && suf < an - pre && + b[bn - 1 - suf] == a[an - 1 - suf]) + suf++; + + int rm_len = bn - pre - suf; + int ins_len= an - pre - suf; + + if (rm_len > 0) { + mods= mods * mod_remove (base, pre, rm_len); + } + if (ins_len > 0) { + tree ins (L (a), ins_len); + for (int i= 0; i < ins_len; i++) + ins[i]= a[pre + i]; + mods= mods * mod_insert (base, pre, ins); + } + // If there are inner modifications in the non-matching parts, + // replace them via remove+insert is a valid structural edit. + return true; + } + } + + return false; // 结构差异(节点数/op 不同,或类型不同) +} +} // namespace + +list +loro_shadow_rep::diff_from_current (tree buffer) { + list mods; + tree after= to_tree (); + if (!diff_walk (buffer, after, path (), mods)) { + // 结构差异:整树替换(coarse 兜底) + mods= list (); + mods= mods * mod_assign (path (), after); + } + return mods; +} + +list +loro_shadow_rep::remote_diff_mods (string bytes, tree buffer) { + if (!import_data (bytes)) return list (); + return diff_from_current (buffer); +} + +// 解码"带 TreeID 的增强 IR":建 moebius 树 + 填 id_map(rep* -> 导入的 +// TreeID)。 格式:node := peer:u64 counter:i32 kind:u8 label:text text:text +// nchildren:u32 children +namespace { +tree +decode_id_node (string& b, int& pos, + hashmap& id_map) { + auto get_u32= [&] () -> uint32_t { + uint32_t v= (uint32_t) (unsigned char) b[pos] | + ((uint32_t) (unsigned char) b[pos + 1] << 8) | + ((uint32_t) (unsigned char) b[pos + 2] << 16) | + ((uint32_t) (unsigned char) b[pos + 3] << 24); + pos+= 4; + return v; + }; + auto get_str= [&] () -> string { + uint32_t n= get_u32 (); + string r; + for (uint32_t i= 0; i < n; i++) + r << b[pos + i]; + pos+= n; + return r; + }; + // TreeID + uint64_t peer= 0; + for (int i= 0; i < 8; i++) + peer|= ((uint64_t) (unsigned char) b[pos + i]) << (8 * i); + pos+= 8; + mogan_tree_id tid{peer, (int32_t) get_u32 ()}; + + uint8_t kind = (uint8_t) (unsigned char) b[pos++]; + string label= get_str (); + string text = get_str (); + uint32_t n = get_u32 (); + + tree r; + if (kind == LORO_ATOMIC) r= tree (text); + else { + int op= (kind == LORO_COMPOUND) ? (int) moebius::make_tree_label (label) + : as_int (label (8, N (label))); + r = tree (op, (int) n); + for (uint32_t i= 0; i < n; i++) + r[i]= decode_id_node (b, pos, id_map); + } + id_map (inside (r))= tid; + return r; +} +} // namespace + +bool +loro_shadow_rep::import_and_build (string bytes, tree& out_buffer) { + if (!import_data (bytes)) return false; + uint8_t* out = nullptr; + size_t out_len= 0; + if (mogan_loro_doc_to_ir_with_ids (doc, &out, &out_len) != 0 || + out == nullptr) { + if (out) mogan_loro_free (out, out_len); + return false; + } + string ir_bytes ((const char*) out, (int) out_len); + mogan_loro_free (out, out_len); + id_map= hashmap ( + mogan_tree_id{0, 0}); // 重置后由 decode 填充 + int pos = 0; + out_buffer= decode_id_node (ir_bytes, pos, id_map); + return true; +} + +tree +loro_shadow_rep::to_tree () { + uint8_t* out = nullptr; + size_t out_len= 0; + if (mogan_loro_doc_to_ir (doc, &out, &out_len) != 0 || out == nullptr) { + if (out) mogan_loro_free (out, out_len); + return tree (""); + } + string ir_bytes ((const char*) out, (int) out_len); + mogan_loro_free (out, out_len); + return loro_ir_to_tree (loro_ir_decode (ir_bytes)); +} + +bool +loro_shadow_rep::has_id (tree t) { + return id_map->contains (inside (t)); +} + +mogan_tree_id +loro_shadow_rep::get_id (tree t) { + return id_map->contains (inside (t)) ? id_map (inside (t)) + : mogan_tree_id{0, 0}; +} + +#endif // LORO_ENABLED diff --git a/moebius/Data/Convert/loro_shadow.hpp b/moebius/Data/Convert/loro_shadow.hpp new file mode 100644 index 0000000000..fdd27b2f24 --- /dev/null +++ b/moebius/Data/Convert/loro_shadow.hpp @@ -0,0 +1,90 @@ +/** \file loro_shadow.hpp + * \copyright GPLv3 + * \details Phase 2 shadow LoroDoc:每个编辑器挂一个 live + * LoroDoc,随本地编辑逐操作镜像。 seed 用文档树构建 live doc 并填充 rep*→TreeID + * 身份表(供 Step 4 镜像编辑用)。 仅 LORO_ENABLED 下存在;关闭时不编译,无 + * Rust 依赖。 + * \author Jim Zhou + * \date 2026 + */ + +#ifndef LORO_SHADOW_H +#define LORO_SHADOW_H + +#include "array.hpp" +#include "hashmap.hpp" +#include "list.hpp" +#include "modification.hpp" +#include "tree.hpp" + +#include "loro.hpp" // mogan_tree_id + live API(仅 LORO_ENABLED) + +#ifdef LORO_ENABLED + +class loro_shadow_rep : public concrete_struct { +public: + void* doc; // mogan_loro_doc 句柄(不透明) + hashmap id_map; //!< 节点身份:rep* -> Loro TreeID + mogan_tree_id root_id; + + mogan_local_update_cb _update_cb = nullptr; + void* _update_user_data= nullptr; + + loro_shadow_rep (); + ~loro_shadow_rep (); + + /** @brief 用树构建 live doc + 填充身份表。 */ + void seed (tree root); + /** @brief 如果 shadow 已有内容(从远端 import),把 buffer 的 rep 关联到现有 + * TreeID (不创建新根,复用共享血统)。返回 true=成功同步,false=shadow + * 为空。 */ + bool sync_id_map_from_shadow (tree buffer); + /** @brief 把一个 modification 镜像到 live doc。doc_root 是 mod->p + * 所相对的树根。 + * + * 文本原子的 INSERT/REMOVE 精确镜像到 LoroText(原子身份在敲字时稳定,Step 1 + * 确认); 其余 mod(结构、SPLIT/JOIN、ASSIGN 等)暂走整树重 seed 兜底,保证 + * shadow 始终与 buffer 一致——后续逐步把这些也精确化。 */ + void mirror_mod (tree doc_root, modification mod); + /** @brief live doc -> snapshot 字节(包成 lolly string)。 */ + string export_snapshot (); + /** @brief 把 snapshot/update 字节 import(合并)进 live doc。Phase 3 + * 反向同步。 */ + bool import_data (string bytes); + /** @brief import 后用"带 TreeID 的增强 IR"重建 buffer 并把 buffer rep + * 关联到导入的 TreeID(填 id_map),使本端后续编辑能与对端合并。out_buffer + * 即新的 buffer。 */ + bool import_and_build (string bytes, tree& out_buffer); + /** @brief 订阅本地编辑产生的增量 update:每次本地 op + * 提交时回调。事件级增量同步。 */ + void on_local_update (mogan_local_update_cb cb, void* user_data); + /** @brief 把远端 update 导入后,diff buffer(旧)vs shadow(新),生成把 + * buffer 变到 新状态所需的 modification 序列(文本字符级 diff → + * INSERT/REMOVE;结构差异 → 整树 ASSIGN 兜底)。供编辑器在 versioning + * 模式下经 edit_modify 应用到 buffer。 */ + list remote_diff_mods (string bytes, tree buffer); + /** @brief diff buffer(旧)vs shadow 当前状态(新),生成把 buffer 变到 + * shadow 所需的 modification(不 import,用于 debug_loro 本地 round-trip)。 + */ + list diff_from_current (tree buffer); + /** @brief live doc -> tree(经 to_ir + loro_ir_to_tree)。 */ + tree to_tree (); + /** @brief 身份表是否含该节点。 */ + bool has_id (tree t); + /** @brief 取节点的 TreeID(不在表中返回 {0,0})。 */ + mogan_tree_id get_id (tree t); + +private: + mogan_tree_id seed_node (tree t, mogan_tree_id parent, uint32_t index); + /** @brief 取某 LoroTree 节点的子 TreeID 列表(REMOVE 按位置删用)。 */ + array node_children (mogan_tree_id parent); +}; + +class loro_shadow { + CONCRETE (loro_shadow); + loro_shadow (); +}; +CONCRETE_CODE (loro_shadow); + +#endif // LORO_ENABLED +#endif // LORO_SHADOW_H diff --git a/moebius/tests/Data/loro_e2e_test.cpp b/moebius/tests/Data/loro_e2e_test.cpp new file mode 100644 index 0000000000..55ca44e642 --- /dev/null +++ b/moebius/tests/Data/loro_e2e_test.cpp @@ -0,0 +1,58 @@ +/** \file loro_e2e_test.cpp + * \copyright GPLv3 + * \details tree <-> Loro CRDT snapshot 端到端往返测试。 + * + * 仅在 LORO_ENABLED(xmake option + * libloro=y)下编译用例;关闭时本文件为 空二进制(无 + * TEST_CASE),不影响默认构建。 + * \author Jim Zhou + * \date 2026 + */ + +#include "loro.hpp" +#include "moe_doctests.hpp" +#include "tree.hpp" +#include "tree_helper.hpp" +#include +#include + +using namespace moebius; + +#ifdef LORO_ENABLED + +static void +ensure_labels () { + moebius::drd::init_std_drd (); +} + +TEST_CASE ("loro e2e: atomic string round-trips through Loro snapshot") { + tree t ("hello 世界 |\\n"); + tree back= loro_to_tree (tree_to_loro (t)); + CHECK_EQ (back == t, true); +} + +TEST_CASE ("loro e2e: nested document round-trips through Loro") { + ensure_labels (); + // (document (para (concat "Hello " (with "world"))) "trailing text") + tree t (DOCUMENT, 2); + t[0] = tree (PARA, 1); + t[0][0] = tree (CONCAT, 2); + t[0][0][0] = tree ("Hello "); + t[0][0][1] = tree (WITH, 1); + t[0][0][1][0]= tree ("world"); + t[1] = tree ("trailing text"); + + string snap= tree_to_loro (t); + CHECK_EQ (N (snap) > 0, true); // 非空 snapshot + tree back= loro_to_tree (snap); + CHECK_EQ (back == t, true); +} + +TEST_CASE ("loro e2e: empty compound round-trips") { + ensure_labels (); + tree t (DOCUMENT, 0); + tree back= loro_to_tree (tree_to_loro (t)); + CHECK_EQ (back == t, true); +} + +#endif // LORO_ENABLED diff --git a/moebius/tests/Data/loro_ir_test.cpp b/moebius/tests/Data/loro_ir_test.cpp new file mode 100644 index 0000000000..6bbb20e4d3 --- /dev/null +++ b/moebius/tests/Data/loro_ir_test.cpp @@ -0,0 +1,116 @@ +/** \file loro_ir_test.cpp + * \copyright GPLv3 + * \details tree <-> LoroIRNode 往返相等性测试。覆盖 + * atomic、compound(内置与扩展 + * 标签)、generic(op<0)、嵌套与子节点顺序、空复合节点。 + * compound 往复依赖标签 name<->op 表,由 init_std_drd() + * 填充(幂等)。 + * \author Jim Zhou + * \date 2026 + */ + +#include "loro_ir.hpp" +#include "moe_doctests.hpp" +#include "tree.hpp" +#include "tree_helper.hpp" +#include +#include + +using namespace moebius; + +// 填充标签 name<->op 表(init_std_drd 自带幂等 guard),compound 往返依赖它 +static void +ensure_labels () { + moebius::drd::init_std_drd (); +} + +TEST_CASE ("loro_ir: atomic string round-trips with special chars") { + tree t ("hello 世界 |tag\\n"); + tree back= loro_ir_to_tree (tree_to_loro_ir (t)); + CHECK_EQ (back == t, true); +} + +TEST_CASE ("loro_ir: compound with known label round-trips") { + ensure_labels (); + // (document "a" (para "b")) + tree t (DOCUMENT, 2); + t[0] = tree ("a"); + t[1] = tree (PARA, 1); + t[1][0] = tree ("b"); + tree back= loro_ir_to_tree (tree_to_loro_ir (t)); + CHECK_EQ (back == t, true); + CHECK_EQ (as_string (L (back)) == "document", true); +} + +TEST_CASE ("loro_ir: compound with extension label round-trips") { + ensure_labels (); + tree_label ext= make_tree_label ("my-custom-extension-tag"); + tree t (ext, 1); + t[0] = tree ("x"); + tree back= loro_ir_to_tree (tree_to_loro_ir (t)); + CHECK_EQ (back == t, true); + CHECK_EQ (as_string (L (back)) == "my-custom-extension-tag", true); +} + +TEST_CASE ("loro_ir: generic op<0 round-trips") { + tree t (-7, 1); + t[0] = tree ("g"); + loro_ir_node ir= tree_to_loro_ir (t); + CHECK_EQ (ir.kind == LORO_GENERIC, true); + CHECK_EQ (ir.label == "generic:-7", true); + tree back= loro_ir_to_tree (ir); + CHECK_EQ (back == t, true); +} + +TEST_CASE ("loro_ir: nested children and order preserved") { + ensure_labels (); + // (concat "a" (with "b") "c") + tree t (CONCAT, 3); + t[0] = tree ("a"); + t[1] = tree (WITH, 1); + t[1][0] = tree ("b"); + t[2] = tree ("c"); + loro_ir_node ir= tree_to_loro_ir (t); + CHECK_EQ (N (ir.children) == 3, true); + CHECK_EQ (ir.children[0].kind == LORO_ATOMIC, true); + CHECK_EQ (ir.children[0].text == "a", true); + CHECK_EQ (ir.children[1].label == "with", true); + tree back= loro_ir_to_tree (ir); + CHECK_EQ (back == t, true); +} + +TEST_CASE ("loro_ir: empty compound round-trips") { + ensure_labels (); + tree t (DOCUMENT, 0); + tree back= loro_ir_to_tree (tree_to_loro_ir (t)); + CHECK_EQ (back == t, true); +} + +TEST_CASE ("loro_ir codec: flat encode/decode self round-trip + bytes") { + // (compound "document" (atomic "hi") (atomic "x")) + loro_ir_node orig; + orig.kind = LORO_COMPOUND; + orig.label= "document"; + loro_ir_node c0; + c0.kind= LORO_ATOMIC; + c0.text= "hi"; + loro_ir_node c1; + c1.kind= LORO_ATOMIC; + c1.text= "x"; + orig.children << c0; + orig.children << c1; + + string bytes= loro_ir_encode (orig); + // 打印字节(十进制),用于与 Rust 侧 Writer 输出逐字节比对格式 + cout << "[codec] " << N (bytes) << " bytes:"; + for (int i= 0; i < N (bytes); i++) + cout << " " << (int) (unsigned char) bytes[i]; + cout << LF; + + loro_ir_node back= loro_ir_decode (bytes); + CHECK_EQ (back.kind == LORO_COMPOUND, true); + CHECK_EQ (back.label == "document", true); + CHECK_EQ (N (back.children) == 2, true); + CHECK_EQ (back.children[0].text == "hi", true); + CHECK_EQ (back.children[1].text == "x", true); +} diff --git a/moebius/tests/Data/loro_shadow_test.cpp b/moebius/tests/Data/loro_shadow_test.cpp new file mode 100644 index 0000000000..4797662f2e --- /dev/null +++ b/moebius/tests/Data/loro_shadow_test.cpp @@ -0,0 +1,324 @@ +/** \file loro_shadow_test.cpp + * \copyright GPLv3 + * \details Phase 2 Step 3 gate:seed 一棵树到 shadow live doc,验证 + * to_tree / export_snapshot(→loro_to_tree) + * 都与原树深度相等,且身份表覆盖各节点。 仅 LORO_ENABLED 下编译用例。 + * \author Jim Zhou + * \date 2026 + */ + +#include "loro.hpp" +#include "loro_shadow.hpp" +#include "moe_doctests.hpp" +#include "tree.hpp" +#include "tree_helper.hpp" +#include +#include + +using namespace moebius; + +#ifdef LORO_ENABLED + +static void +ensure_labels () { + moebius::drd::init_std_drd (); +} + +TEST_CASE ("loro_shadow: seed round-trips a document") { + ensure_labels (); + // (document (para (concat "hello")) (para "world")) + tree t (DOCUMENT, 2); + t[0] = tree (PARA, 1); + t[0][0] = tree (CONCAT, 1); + t[0][0][0]= tree ("hello"); + t[1] = tree (PARA, 1); + t[1][0] = tree ("world"); + + loro_shadow sh; + sh->seed (t); + + // live doc -> tree(经 to_ir + loro_ir_to_tree) + tree back= sh->to_tree (); + CHECK_EQ (back == t, true); + + // live doc -> snapshot -> loro_to_tree(Phase 1 路径) + string snap = sh->export_snapshot (); + tree back2= loro_to_tree (snap); + CHECK_EQ (back2 == t, true); + + // 身份表覆盖根与各层节点 + CHECK_EQ (sh->has_id (t), true); + CHECK_EQ (sh->has_id (t[0]), true); + CHECK_EQ (sh->has_id (t[0][0][0]), true); +} + +TEST_CASE ("loro_shadow: atomic text (UTF-8) seeded via LoroText") { + ensure_labels (); + tree t (PARA, 1); + t[0]= tree ("hello 世界"); + loro_shadow sh; + sh->seed (t); + CHECK_EQ (sh->to_tree () == t, true); + CHECK_EQ (sh->has_id (t[0]), true); +} + +TEST_CASE ("loro_shadow: mirror text typing and backspace") { + ensure_labels (); + // (document (paragraph (concat ""))) + tree t (DOCUMENT, 1); + t[0] = tree (PARA, 1); + t[0][0] = tree (CONCAT, 1); + t[0][0][0]= tree (""); // 空原子 + loro_shadow sh; + sh->seed (t); + + path atom= path (0) * 0 * 0; // [0,0,0] + mogan_tree_id id0 = sh->get_id (t[0][0][0]); // 原子身份(seed 后) + // 敲 "abc":只喂 mod 给 mirror_mod(它读 mod + // 不读树状态),再手工把期望结果赋给 t + sh->mirror_mod (t, mod_insert (atom, 0, tree ("a"))); + sh->mirror_mod (t, mod_insert (atom, 1, tree ("b"))); + sh->mirror_mod (t, mod_insert (atom, 2, tree ("c"))); + t[0][0][0]->label= string ("abc"); + CHECK_EQ (sh->to_tree () == t, true); + + // 退格删 "c" + sh->mirror_mod (t, mod_remove (atom, 2, 1)); + t[0][0][0]->label= string ("ab"); + CHECK_EQ (sh->to_tree () == t, true); + + // 原子身份逐字不变 → 走的是精确 LoroText 路径(非兜底重 seed) + mogan_tree_id id1= sh->get_id (t[0][0][0]); + CHECK_EQ (id1.peer == id0.peer && id1.counter == id0.counter, true); +} + +TEST_CASE ("loro_shadow: structural edit falls back to re-seed") { + ensure_labels (); + // (document (paragraph "hi")) + tree t (DOCUMENT, 1); + t[0] = tree (PARA, 1); + t[0][0]= tree ("hi"); + loro_shadow sh; + sh->seed (t); + CHECK_EQ (sh->to_tree () == t, true); + + // 结构改动:document 末尾插入新段落(root=doc 复合 → 兜底整树重 seed)。 + // 手工把 t 改成 post-edit 状态(不用 raw_apply,避免拉入 libmogan 符号), + // mirror_mod 兜底从 t 重新 seed。 + tree extra (PARA, 1); + extra[0]= tree ("new"); + tree mutated (DOCUMENT, 2); + mutated[0]= t[0]; + mutated[1]= extra; + t = mutated; + sh->mirror_mod (t, mod_insert (path (), 1, extra)); + CHECK_EQ (sh->to_tree () == t, true); + CHECK_EQ (N (t) == 2, true); +} + +// 模拟一段真实编辑会话:打字 → 退格 → 结构改动(兜底重 seed)→ 再打字。 +// 每步都用期望的真值 buffer 比对 shadow.to_tree(),证明 shadow 始终与 buffer +// 一致。 +TEST_CASE ("loro_shadow: consistent across a mixed edit session") { + ensure_labels (); + // (document (paragraph (concat ""))) + tree t (DOCUMENT, 1); + t[0] = tree (PARA, 1); + t[0][0] = tree (CONCAT, 1); + t[0][0][0]= tree (""); + loro_shadow sh; + sh->seed (t); + path atom= path (0) * 0 * 0; // [0,0,0] + + // 1. 打 "hello"(精确 LoroText) + const char* word= "hello"; + for (int i= 0; word[i] != 0; i++) { + char s[2]= {word[i], 0}; + sh->mirror_mod (t, mod_insert (atom, i, tree (s))); + } + t[0][0][0]->label= string ("hello"); + CHECK_EQ (sh->to_tree () == t, true); + + // 2. 退格两次 → "hel" + sh->mirror_mod (t, mod_remove (atom, 4, 1)); + sh->mirror_mod (t, mod_remove (atom, 3, 1)); + t[0][0][0]->label= string ("hel"); + CHECK_EQ (sh->to_tree () == t, true); + + // 3. 结构改动:插入第二段(root=doc 复合 → 兜底整树重 seed) + tree p2 (PARA, 1); + p2[0]= tree ("world"); + tree mutated (DOCUMENT, 2); + mutated[0]= t[0]; + mutated[1]= p2; + t = mutated; + sh->mirror_mod (t, mod_insert (path (), 1, p2)); + CHECK_EQ (sh->to_tree () == t, true); + + // snapshot 可导出且非空 + CHECK_EQ (N (sh->export_snapshot ()) > 0, true); +} + +// 正向 e2e:A 本地编辑(mirror)→ snapshot → B 导入 → B 的树 == A 的 buffer。 +TEST_CASE ("loro e2e: local edit on A propagates to B via snapshot") { + ensure_labels (); + // A 的 buffer: (document (paragraph (concat "x"))) + tree tA (DOCUMENT, 1); + tA[0] = tree (PARA, 1); + tA[0][0] = tree (CONCAT, 1); + tA[0][0][0]= tree ("x"); + loro_shadow a; + a->seed (tA); + + // A 敲 "y" → "xy"(精确 LoroText 镜像) + a->mirror_mod (tA, mod_insert (path (0) * 0 * 0, 1, tree ("y"))); + tA[0][0][0]->label= string ("xy"); + CHECK_EQ (a->to_tree () == tA, true); + + // 导出 snapshot + string snap= a->export_snapshot (); + CHECK_EQ (N (snap) > 0, true); + + // B 导入 snapshot(被动),to_tree 应等于 A 的编辑后 buffer + loro_shadow b; + CHECK_EQ (b->import_data (snap), true); + CHECK_EQ (b->to_tree () == tA, true); +} + +// 双向合并 e2e:A/B 从公共初始 snapshot +// 各自并发编辑不同节点,互导后收敛到同一棵树。 关键:B 用 import_and_build 把 +// buffer rep 关联到 A 的 TreeID,二者身份对齐才能合并。 +TEST_CASE ("loro e2e: bidirectional merge (concurrent edits converge)") { + ensure_labels (); + // 公共初始文档 (document (para "a") (para "b")) + tree init (DOCUMENT, 2); + init[0] = tree (PARA, 1); + init[0][0]= tree ("a"); + init[1] = tree (PARA, 1); + init[1][0]= tree ("b"); + + loro_shadow a; + a->seed (init); + string sa0= a->export_snapshot (); // 公共初始 snapshot(A 编辑前导出) + + // A 编辑 para0:"a" -> "aA"(tA 共享 init 的 rep) + tree tA= init; + a->mirror_mod (tA, mod_insert (path (0) * 0, 1, tree ("A"))); + tA[0][0]->label= string ("aA"); + string sa = a->export_snapshot (); + + // B 从 sa0 导入并构建 buffer(id_map 关联到 A 的 TreeID) + loro_shadow b; + tree tB; + CHECK_EQ (b->import_and_build (sa0, tB), true); + // B 编辑 para1:"b" -> "bB" + b->mirror_mod (tB, mod_insert (path (1) * 0, 1, tree ("B"))); + tB[1][0]->label= string ("bB"); + string sb = b->export_snapshot (); + + // 互导 snapshot → CRDT 合并 + CHECK_EQ (a->import_data (sb), true); + CHECK_EQ (b->import_data (sa), true); + + // 收敛:两端都 == (para "aA") (para "bB") + tree expected (DOCUMENT, 2); + expected[0] = tree (PARA, 1); + expected[0][0]= tree ("aA"); + expected[1] = tree (PARA, 1); + expected[1][0]= tree ("bB"); + CHECK_EQ (a->to_tree () == expected, true); + CHECK_EQ (b->to_tree () == expected, true); +} + +// 事件级增量同步:A 订阅 local-update,编辑产生 delta(仅新增 op)→ B 导入 +// delta 收敛。 +extern "C" void +loro_test_capture_update (void* ud, const uint8_t* bytes, size_t len) { + auto* s= static_cast (ud); + *s = string ((const char*) bytes, (int) len); +} + +TEST_CASE ("loro event: incremental sync via local-update events") { + ensure_labels (); + tree init (DOCUMENT, 1); + init[0] = tree (PARA, 1); + init[0][0] = tree (CONCAT, 1); + init[0][0][0] = tree ("x"); + tree tA= init; // 持久 buffer(共享 init 的 rep) + loro_shadow a; + a->seed (tA); + string s0= a->export_snapshot (); + loro_shadow b; + CHECK_EQ (b->import_data (s0), true); // B 起点 == init,共享 TreeID + + // A 订阅 local-update(捕获 seed 之后的增量 delta) + string captured; + a->on_local_update (loro_test_capture_update, &captured); + + // A 编辑:"x" -> "xy" + a->mirror_mod (tA, mod_insert (path (0) * 0 * 0, 1, tree ("y"))); + tA[0][0][0]->label= string ("xy"); + + CHECK_EQ (N (captured) > 0, true); // 事件触发,捕获到 delta(非整 snapshot) + + // B 导入 delta(仅新增 op)→ 收敛到 A 的编辑后状态 + CHECK_EQ (b->import_data (captured), true); + CHECK_EQ (b->to_tree () == tA, true); +} + +// 远端 delta → modification:B 收到 A 的编辑后,生成把 buffer 变到新状态所需的 +// mod, 手工应用到 buf(模拟 edit_modify),验证 buf 与 A 的编辑结果一致。 +TEST_CASE ("loro reverse: remote edit -> modification (apply to buffer)") { + ensure_labels (); + tree init (DOCUMENT, 1); + init[0] = tree (PARA, 1); + init[0][0] = tree (CONCAT, 1); + init[0][0][0]= tree ("x"); + + // A: seed、编辑 "x"->"xy"、导出 + loro_shadow a; + tree tA= init; + a->seed (tA); + string s0= a->export_snapshot (); + a->mirror_mod (tA, mod_insert (path (0) * 0 * 0, 1, tree ("y"))); + tA[0][0][0]->label= string ("xy"); + string sa = a->export_snapshot (); + + // B: 从 s0 建共享身份的 buffer + loro_shadow b; + tree buf; + CHECK_EQ (b->import_and_build (s0, buf), true); + // 生成把 buf(init) 变到 A 编辑后状态的 mods + list mods= b->remote_diff_mods (sa, buf); + CHECK_EQ (N (mods) >= 1, true); + + // 手工把 mods 应用到 buf(模拟编辑器 edit_modify,不依赖 libmogan) + for (list l= mods; !is_nil (l); l= l->next) { + modification m= l->item; + if (m->k == MOD_INSERT) { + tree& atom= subtree (buf, root (m)); + string s = m->t->label; + int pos = index (m); + atom->label= + atom->label (0, pos) * s * atom->label (pos, N (atom->label)); + } + else if (m->k == MOD_REMOVE) { + tree& atom= subtree (buf, root (m)); + int pos = index (m); + int nr = argument (m); + atom->label= + atom->label (0, pos) * atom->label (pos + nr, N (atom->label)); + } + else if (m->k == MOD_ASSIGN) { + buf= m->t; + } + } + + tree expected (DOCUMENT, 1); + expected[0] = tree (PARA, 1); + expected[0][0] = tree (CONCAT, 1); + expected[0][0][0]= tree ("xy"); + CHECK_EQ (buf == expected, true); +} + +#endif // LORO_ENABLED diff --git a/moebius/tests/Data/loro_tmu_test.cpp b/moebius/tests/Data/loro_tmu_test.cpp new file mode 100644 index 0000000000..d305e21da8 --- /dev/null +++ b/moebius/tests/Data/loro_tmu_test.cpp @@ -0,0 +1,108 @@ +/** \file loro_tmu_test.cpp + * \copyright GPLv3 + * \details 真实文档保真测试:遍历 TeXmacs/tests/tmu/,对每个夹具做 + * tmu_document_to_tree -> tree_to_loro -> + * loro_to_tree,断言与原树深度相等。 + * 覆盖真实文档里的数学、表格、图片(raw-data)、样式等结构,暴露映射盲区。 + * + * 仅在 LORO_ENABLED(xmake option + * libloro=y)下编译用例;关闭时为空二进制。 + * \author Jim Zhou + * \date 2026 + */ + +#include "file.hpp" +#include "loro.hpp" +#include "moe_doctests.hpp" +#include "tmu.hpp" +#include "tree.hpp" +#include "tree_helper.hpp" +#include "url.hpp" +#include +#include + +using namespace moebius; + +#ifdef LORO_ENABLED + +static void +ensure_labels () { + moebius::drd::init_std_drd (); +} + +// 定位 tmu 夹具目录:尝试常见 CWD 候选(xmake r 默认在项目根) +static string +resolve_fixture_dir () { + const char* cands[]= { + "TeXmacs/tests/tmu/", + "../TeXmacs/tests/tmu/", + "../../TeXmacs/tests/tmu/", + "../../../TeXmacs/tests/tmu/", + "../../../../TeXmacs/tests/tmu/", + }; + int n= sizeof (cands) / sizeof (cands[0]); + for (int i= 0; i < n; i++) { + string c= cands[i]; + if (is_directory (c)) return c; + } + return string (); +} + +static bool +ends_with_tmu (string name) { + int n= N (name); + return n >= 4 && name (n - 4, n) == ".tmu"; +} + +TEST_CASE ("loro real: round-trip all tmu fixtures") { + ensure_labels (); + string dir= resolve_fixture_dir (); + if (N (dir) == 0) { + MESSAGE ("找不到 TeXmacs/tests/tmu 夹具目录,跳过(请从项目根目录运行)"); + return; + } + + bool err = false; + array names = read_directory (dir, err); + int tested= 0, passed= 0, failed= 0, skipped= 0, shown= 0; + for (int i= 0; i < N (names); i++) { + string name= names[i]; + if (!ends_with_tmu (name)) continue; + string content= string_load (dir * name); + if (N (content) == 0) continue; + + tree doc= tmu_document_to_tree (content); + // 解析失败会返回 (ERROR "bad format or data");跳过不计,避免把 error + // 树的恒等 round-trip 误当通过。若全部都 ERROR,下方 tested>0 会兜住(提示 + // init 不足)。 + if (is_compound (doc) && L (doc) == ERROR) { + skipped++; + continue; + } + tree back= loro_to_tree (tree_to_loro (doc)); + tested++; + if (back == doc) { + passed++; + continue; + } + failed++; + if (shown < 3) { + // 打印前 600 字符的 tmu 表示,定位分歧起点 + string ot = tree_to_tmu (doc); + string bt = tree_to_tmu (back); + int upto= N (ot) < 600 ? N (ot) : 600; + cout << "MISMATCH: " << name << " (arity orig=" << N (doc) + << " back=" << N (back) << ")" << LF; + cout << " orig: " << ot (0, upto) << (N (ot) > upto ? "..." : "") << LF; + cout << " back: " << bt (0, upto) << (N (bt) > upto ? "..." : "") << LF; + shown++; + } + } + + cout << "loro real: tested=" << tested << " passed=" << passed + << " failed=" << failed << " skipped(parse-error)=" << skipped << LF; + CHECK_EQ (tested > 0, true); + CHECK_EQ (failed == 0, true); +} + +#endif // LORO_ENABLED diff --git a/moebius/xmake.lua b/moebius/xmake.lua index e837c944aa..280fe37c7a 100644 --- a/moebius/xmake.lua +++ b/moebius/xmake.lua @@ -31,6 +31,10 @@ local moe_includedirs = { add_requires("liii-doctest", {system=false}) add_requires("nanobench", {system=false}) +if has_config("loro") then + includes("../xmake/targets/loro.lua") +end + target("libmoebius") do set_kind ("static") set_languages("c++17") @@ -42,6 +46,10 @@ target("libmoebius") do add_deps("liblolly") add_deps("goldfish") + if has_config("loro") then + add_deps("loro") + add_defines("LORO_ENABLED") + end add_headerfiles("Data/Convert/(*.hpp)") add_headerfiles("Data/History/(*.hpp)") @@ -65,6 +73,9 @@ target("moebius_tests") do set_default (false) add_deps("libmoebius") + if has_config("loro") then + add_defines("LORO_ENABLED") + end add_includedirs(moe_includedirs) add_includedirs("tests") diff --git a/src/Edit/Modify/edit_modify.cpp b/src/Edit/Modify/edit_modify.cpp index 6ae5842fe0..a2ccc56f58 100644 --- a/src/Edit/Modify/edit_modify.cpp +++ b/src/Edit/Modify/edit_modify.cpp @@ -19,11 +19,135 @@ * Constructors and destructors ******************************************************************************/ +#ifdef LORO_ENABLED +extern void (*g_loro_broadcast_update) (string bytes); + +static void +local_update_cb (void* user_data, const uint8_t* bytes, size_t len) { + if (g_loro_broadcast_update) { + cout << "[Loro] Local update generated, size: " << len + << " bytes. Broadcasting...\n"; + string data ((const char*) bytes, len); + g_loro_broadcast_update (data); + } + else { + cout << "[Loro] Local update generated, but no broadcast handler " + "registered.\n"; + } +} +#endif + edit_modify_rep::edit_modify_rep () : editor_rep (), // NOTE: ignored by the compiler, but suppresses warning - author (new_author ()), arch (author, rp) {} + author (new_author ()), arch (author, rp) +#ifdef LORO_ENABLED + , + loro_doc (), loro_seeded (false), loro_applying_remote (false), + loro_routing (false) +#endif +{ +#ifdef LORO_ENABLED + loro_doc->on_local_update (local_update_cb, this); +#endif +} edit_modify_rep::~edit_modify_rep () {} +#ifdef LORO_ENABLED +// Phase 2:把一个 modification 镜像到 shadow LoroDoc。 +// mod->p 是 the_et 根路径,用 mod/rp 转成 buffer 相对;首次镜像时 lazy seed。 + +void +edit_modify_rep::ensure_loro_seeded () { + if (loro_applying_remote) return; + if (!loro_seeded) { + // 如果 shadow 已有内容(从远端 import 而来),复用其 TreeID + // 血统(不创建新根), 避免 A/B 各自 seed 产生两个根 → to_tree 只读 + // roots[0] → 对端编辑不可见。 + if (!loro_doc->sync_id_map_from_shadow (the_buffer ())) + loro_doc->seed (the_buffer ()); // shadow 为空 → 本端是创建者 → seed + loro_seeded= true; + } +} + +void +edit_modify_rep::mirror_loro (const modification& mod) { + if (loro_applying_remote) return; // 远端应用期间不回灌镜像,避免循环 + if (const_cast (mod)->k == MOD_SET_CURSOR) return; + ensure_loro_seeded (); // Fallback in case not called from edit_announce + loro_doc->mirror_mod (the_buffer (), mod / rp); +#ifdef LORO_DEBUG + // debug_loro 验证:镜像后 buffer 应与 Loro + // 状态一致。不一致则告警(说明镜像链路有 bug)。 这是安全模式——mirror_loro 在 + // edit_done(post-apply) 里,不调 edit_announce,无递归风险。 + tree lt= loro_doc->to_tree (); + if (!(lt == the_buffer ())) + cout << "[loro-verify] MISMATCH: buffer != Loro after mirror\n"; +#endif +} + +// Phase 3:导入远端 update,把 diff 出的 mods 经 edit_announce 应用到 buffer。 +// loro_applying_remote 守卫使这些应用的 edit_done→mirror_loro +// 被跳过(versioning)。 +void +edit_modify_rep::apply_remote (string bytes) { + cout << "[Loro] Applying remote update of size " << N (bytes) << "\n"; + loro_applying_remote = true; + list mods= loro_doc->remote_diff_mods (bytes, the_buffer ()); + cout << "[Loro] Diff produced " << N (mods) << " modifications.\n"; + for (list l= mods; !is_nil (l); l= l->next) { + cout << "[Loro] Applying remote mod: " << l->item << "\n"; + apply (et, rp * l->item); + } + loro_applying_remote= false; + + // 关键:apply_remote 通过 edit_announce 改了 buffer(新 tree_rep*), + // 但这些新 rep 不在 id_map 里,因此下一次本地编辑 mirror_mod 会 id_map miss + // -> 块级重 seed + // -> TreeID 被重洗 -> 远端 update 引用旧 TreeID -> 永久 Diff 0。 + // 因此,这里重建 id_map,把 buffer 当前状态关联到 shadow 的 TreeID。 + if (!is_nil (mods)) loro_doc->sync_id_map_from_shadow (the_buffer ()); + + if (!is_nil (mods)) { + cout << "[Loro] Forcing typeset invalidation and repaint...\n"; + notify_change (THE_TREE); + typeset_invalidate_all (); + send_invalidate_all (this); + } +} + +// debug_loro:把 mod 经 Loro round-trip 后再应用(不直接应用原 mod)。 +// - 文本精确路径:mirror 后 Loro 已含 op → diff 回译出 mod → 应用时跳过 +// mirror_loro(防重复)。 +// - 结构兜底:mirror 未改变 Loro(重 seed 自 before)→ diff 为空 → 正常应用原 +// mod +// (mirror_loro 会从 post-edit buffer 重 seed,使 Loro 与 buffer 一致)。 +bool +edit_modify_rep::route_through_loro (const modification& mod) { + if (loro_routing) return false; // 重入(应用回译/原 mod)→ 正常路径 + loro_routing= true; + if (!loro_seeded) { + loro_doc->seed (the_buffer ()); + loro_seeded= true; + } + tree before= the_buffer (); + loro_doc->mirror_mod (before, mod / rp); + list mods= loro_doc->diff_from_current (before); + if (is_nil (mods)) { + edit_announce ( + this, mod); // 结构兜底:正常应用(mirror_loro 重 seed 自 post-edit) + } + else { + loro_applying_remote= + true; // 文本精确:Loro 已有 op,应用回译 mod 时跳过 mirror_loro + for (list l= mods; !is_nil (l); l= l->next) + edit_announce (this, rp * l->item); + loro_applying_remote= false; + } + loro_routing= false; + return true; +} +#endif + /****************************************************************************** * Notification of changes in document ******************************************************************************/ @@ -210,6 +334,15 @@ edit_set_cursor (editor_rep* ed, path pp, tree data) { void edit_announce (editor_rep* ed, modification mod) { + // NOTE: 曾尝试在此拦截做 debug_loro round-trip,但 edit_observer 对每个 mod + // 配对调用 edit_announce(应用)+edit_done(mirror),在 announce 里 mirror + // 后外层 done 会重复 mirror (文本路径插两次)。盲写难以安全区分内/外层 + // done,故暂不激活。debug_loro 改用验证模式。 +#ifdef LIII_DEBUG + if (mod->k != MOD_SET_CURSOR) cout << "[loro-mod] " << mod << "\n"; +#endif + if (mod->k != MOD_SET_CURSOR) ed->ensure_loro_seeded (); + switch (mod->k) { case MOD_ASSIGN: edit_assign (ed, mod->p, mod->t); @@ -247,7 +380,10 @@ void edit_done (editor_rep* ed, modification mod) { path p= copy (mod->p); ASSERT (ed->the_buffer_path () <= p, "invalid modification"); - if (mod->k != MOD_SET_CURSOR) ed->post_notify (p); + if (mod->k != MOD_SET_CURSOR) { + ed->post_notify (p); + ed->mirror_loro (mod); // Phase 2:镜像到 shadow LoroDoc + } } void diff --git a/src/Edit/Modify/edit_modify.hpp b/src/Edit/Modify/edit_modify.hpp index 057631bb8b..80f6bc86bb 100644 --- a/src/Edit/Modify/edit_modify.hpp +++ b/src/Edit/Modify/edit_modify.hpp @@ -13,6 +13,9 @@ #define EDIT_MODIFY_H #include "archiver.hpp" #include "editor.hpp" +#ifdef LORO_ENABLED +#include "loro_shadow.hpp" +#endif path inner_paragraph (tree t, path p); @@ -21,22 +24,38 @@ class edit_modify_rep : virtual public editor_rep { observer cur_pos; // tree_position corresponding to tp double author; // the author identifier associated to this view archiver arch; // archiver attached to the editor +#ifdef LORO_ENABLED + loro_shadow loro_doc; // Phase 2 shadow LoroDoc(随本地编辑镜像) + bool loro_seeded = false; // 是否已 seed 当前 buffer + bool loro_applying_remote= false; // 远端应用期间,跳过镜像回灌 + bool loro_routing = false; // debug_loro round-trip 中,防递归 +#endif public: edit_modify_rep (); ~edit_modify_rep (); - double this_author (); + double this_author () override; - void notify_assign (path p, tree u); - void notify_insert (path p, tree u); - void notify_remove (path p, int nr); - void notify_split (path p); + void notify_assign (path p, tree u) override; + void notify_insert (path p, tree u) override; + void notify_remove (path p, int nr) override; + void notify_split (path p) override; void notify_join (path p); void notify_assign_node (path p, tree_label op); void notify_insert_node (path p, tree t); void notify_remove_node (path p); void notify_set_cursor (path p, tree data); void post_notify (path p); +#ifdef LORO_ENABLED + void ensure_loro_seeded () override; + void mirror_loro (const modification& mod) override; + // Phase 3:导入远端 update,diff 出把 buffer 变到新状态所需的 mods,经 + // edit_announce 应用到 buffer(versioning:loro_applying_remote 守卫使 + // mirror_loro 跳过,避免回灌)。 + void apply_remote (string bytes) override; + // debug_loro:把 mod 经 Loro round-trip(mirror→diff_from_current)后再应用。 + bool route_through_loro (const modification& mod) override; +#endif void clear_undo_history (); void archive_state (); diff --git a/src/Edit/editor.hpp b/src/Edit/editor.hpp index 36ea9c4da1..df652b0dbf 100644 --- a/src/Edit/editor.hpp +++ b/src/Edit/editor.hpp @@ -623,6 +623,18 @@ class editor_rep : public simple_widget_rep { virtual void show_meminfo () = 0; virtual void edit_special () = 0; + // Phase 2:把 modification 镜像到 shadow LoroDoc(默认 no-op;edit_modify_rep + // 在 LORO_ENABLED 下 override)。在 edit_done 里调用。 + virtual void mirror_loro (const modification& mod) { (void) mod; } + virtual void apply_remote (string bytes) {} + virtual void ensure_loro_seeded () {} + // debug_loro:把 mod 经 Loro round-trip 后再应用(返回 true + // 表示已处理,跳过直接应用)。 + virtual bool route_through_loro (const modification& mod) { + (void) mod; + return false; + } + friend class tm_window_rep; friend class tm_server_rep; friend class server_command_rep; diff --git a/src/Plugins/ImGui/im_gui.cpp b/src/Plugins/ImGui/im_gui.cpp index 12fb2f7183..1e540d33ea 100644 --- a/src/Plugins/ImGui/im_gui.cpp +++ b/src/Plugins/ImGui/im_gui.cpp @@ -500,6 +500,35 @@ gui_open (int& argc, char** argv) { // "Invalid situation" 而提前返回。将其保存在这里,让 ImGui 驱动这一回调。 static void (*g_interpose_fn) (void)= nullptr; +#ifdef LORO_ENABLED +#include "editor.hpp" +#include "im_websocket.hpp" +#include "new_view.hpp" // get_current_editor + +void (*g_loro_broadcast_update) (string bytes)= nullptr; + +class test_websocket_client : public im_websocket_client { +public: + void on_connect () override { + cout << "Connected to Loro relay server!\n"; + send ("JOIN test-room", false); + } + void on_message (string data, bool is_binary) override { + if (is_binary) { + editor ed= get_current_editor (); + if (!is_nil (ed)) { + ed->apply_remote (data); + } + } + else { + cout << "Received msg: text size: " << N (data) << "\n"; + } + } + void on_error (string msg) override { cout << "WS Error: " << msg << "\n"; } + void on_disconnect () override { cout << "WS Disconnected\n"; } +}; +#endif + void gui_interpose (void (*fn) (void)) { g_interpose_fn= fn; @@ -508,6 +537,26 @@ gui_interpose (void (*fn) (void)) { void im_interpose () { if (g_interpose_fn != nullptr) g_interpose_fn (); + +#ifdef LORO_ENABLED + static test_websocket_client* g_ws_client= nullptr; + if (!g_ws_client) { + g_ws_client= new test_websocket_client (); + g_ws_client->connect ("ws://127.0.0.1:8765"); + + g_loro_broadcast_update= [] (string bytes) { + if (g_ws_client && g_ws_client->connected ()) { + cout << "[WS] Sending " << N (bytes) + << " bytes of Loro update to server.\n"; + g_ws_client->send (bytes, true); // send binary + } + else { + cout << "[WS] Cannot send Loro update, WebSocket is not connected.\n"; + } + }; + } + g_ws_client->poll (); +#endif } void diff --git a/src/Plugins/ImGui/im_websocket.cpp b/src/Plugins/ImGui/im_websocket.cpp new file mode 100644 index 0000000000..079789e0c7 --- /dev/null +++ b/src/Plugins/ImGui/im_websocket.cpp @@ -0,0 +1,232 @@ +/****************************************************************************** + * MODULE : im_websocket.cpp + * DESCRIPTION: A single-threaded, non-blocking WebSocket client using libcurl + ******************************************************************************/ + +#include "im_websocket.hpp" +#include "basic.hpp" // for cout, debug_std + +#if defined(CURL_VERSION_MAJOR) +#if LIBCURL_VERSION_NUM >= 0x075600 // 7.86.0 for websocket +#define MOGAN_HAS_CURL_WS 1 +#endif +#endif + +// Fallback in case header defines CURLWS_BINARY directly +#if !defined(MOGAN_HAS_CURL_WS) && defined(CURLWS_BINARY) +#define MOGAN_HAS_CURL_WS 1 +#endif + +im_websocket_client::im_websocket_client () + : multi_handle (NULL), easy_handle (NULL), is_connected (false) { + multi_handle= curl_multi_init (); +} + +im_websocket_client::~im_websocket_client () { + cleanup (); + if (multi_handle) { + curl_multi_cleanup (multi_handle); + multi_handle= NULL; + } +} + +void +im_websocket_client::cleanup () { + if (easy_handle) { + if (multi_handle) { + curl_multi_remove_handle (multi_handle, easy_handle); + } + curl_easy_cleanup (easy_handle); + easy_handle= NULL; + } + is_connected= false; + rx_buffer = ""; +} + +void +im_websocket_client::connect (string url) { + disconnect (); // cleanup any existing connection + + current_url= url; + easy_handle= curl_easy_init (); + if (!easy_handle) { + on_error ("Failed to initialize libcurl easy handle"); + return; + } + + c_string curl_url (url); + curl_easy_setopt (easy_handle, CURLOPT_URL, (const char*) curl_url); + curl_easy_setopt (easy_handle, CURLOPT_CONNECT_ONLY, 2L); // 2 = WebSocket + + curl_multi_add_handle (multi_handle, easy_handle); +} + +void +im_websocket_client::disconnect () { + if (is_connected || easy_handle) { + cleanup (); + on_disconnect (); + } +} + +void +im_websocket_client::send (string data, bool is_binary) { + if (!is_connected || !easy_handle) return; + im_ws_msg msg; + msg.data = data; + msg.is_binary= is_binary; + msg.offset = 0; + tx_queue = tx_queue * list (msg); + process_tx_queue (); +} + +void +im_websocket_client::process_tx_queue () { + if (!is_connected || !easy_handle) return; + +#ifdef MOGAN_HAS_CURL_WS + while (!is_nil (tx_queue)) { + im_ws_msg& msg = tx_queue->item; + size_t total_len= (size_t) N (msg.data); + size_t remaining= total_len - msg.offset; + size_t chunk_size= + remaining > 65536 + ? 65536 + : remaining; // Send in 64KB chunks to avoid large buffer issues + + unsigned int flags = msg.is_binary ? CURLWS_BINARY : CURLWS_TEXT; + curl_off_t fragsize= 0; + + if (remaining > chunk_size) { + flags|= CURLWS_OFFSET; + } + + if (msg.offset == 0 && total_len > chunk_size) { + fragsize= total_len; + } + else { + fragsize= 0; + } + + size_t sent= 0; + const char* ptr = &msg.data[(int) msg.offset]; + CURLcode res = curl_ws_send (easy_handle, (const void*) ptr, chunk_size, + &sent, fragsize, flags); + + if (res == CURLE_AGAIN) { + // Socket full, try again later + break; + } + else if (res != CURLE_OK) { + string err_msg= "Failed to send WebSocket message: "; + err_msg << curl_easy_strerror (res); + on_error (err_msg); + disconnect (); + break; + } + + msg.offset+= sent; + if (msg.offset >= total_len) { + // Finished this message + tx_queue= tx_queue->next; + } + else { + // Partial send, wait for next poll + break; + } + } +#else + if (!is_nil (tx_queue)) { + on_error ("libcurl version too old, websocket not supported"); + tx_queue= list (); + } +#endif +} + +void +im_websocket_client::poll () { + if (!multi_handle || !easy_handle) return; + + int still_running= 0; + CURLMcode mc = curl_multi_perform (multi_handle, &still_running); + + if (mc != CURLM_OK) { + on_error ("curl_multi_perform failed"); + disconnect (); + return; + } + + // Check connection status + int msgs_in_queue= 0; + CURLMsg* msg; + while ((msg= curl_multi_info_read (multi_handle, &msgs_in_queue))) { + if (msg->msg == CURLMSG_DONE) { + if (msg->data.result == CURLE_OK) { + if (!is_connected) { + is_connected= true; + on_connect (); + } + } + else { + string err_msg= "WebSocket connection failed: "; + err_msg << curl_easy_strerror (msg->data.result); + on_error (err_msg); + disconnect (); + return; // handle is cleaned up + } + } + } + + // If connected, try to receive data + if (is_connected) { + process_tx_queue (); + +#ifdef MOGAN_HAS_CURL_WS + size_t recv_len= 0; + const struct curl_ws_frame* meta = NULL; + char buffer[4096]; + + // We might receive multiple frames or partial frames + while (true) { + CURLcode res= + curl_ws_recv (easy_handle, buffer, sizeof (buffer), &recv_len, &meta); + + if (res == CURLE_OK && recv_len > 0) { + // Append to our buffer + string chunk (buffer, (int) recv_len); + rx_buffer << chunk; + + if (meta && meta->bytesleft == 0 && !(meta->flags & CURLWS_CONT)) { + // Full message received + bool is_binary= true; + if (meta->flags & CURLWS_TEXT) { + is_binary= false; + } + + string complete_msg= rx_buffer; + rx_buffer = ""; + + if (meta->flags & CURLWS_CLOSE) { + disconnect (); + return; + } + + if (!(meta->flags & CURLWS_PING) && !(meta->flags & CURLWS_PONG)) { + on_message (complete_msg, is_binary); + } + } + } + else if (res == CURLE_AGAIN) { + // No data available right now + break; + } + else { + // Error or closed + on_error ("WebSocket receive error or connection closed"); + disconnect (); + break; + } + } +#endif + } +} diff --git a/src/Plugins/ImGui/im_websocket.hpp b/src/Plugins/ImGui/im_websocket.hpp new file mode 100644 index 0000000000..aa5e81097c --- /dev/null +++ b/src/Plugins/ImGui/im_websocket.hpp @@ -0,0 +1,53 @@ +/****************************************************************************** + * MODULE : im_websocket.hpp + * DESCRIPTION: A single-threaded, non-blocking WebSocket client using libcurl + ******************************************************************************/ + +#ifndef IM_WEBSOCKET_HPP +#define IM_WEBSOCKET_HPP + +#include "list.hpp" +#include "string.hpp" +#include + +struct im_ws_msg { + string data; + bool is_binary; + size_t offset; +}; + +class im_websocket_client { +private: + CURLM* multi_handle; + CURL* easy_handle; + bool is_connected; + string current_url; + string rx_buffer; // Buffer for partial frames + + list tx_queue; + + // Internal helpers + void cleanup (); + void process_tx_queue (); + +public: + im_websocket_client (); + virtual ~im_websocket_client (); + + void connect (string url); + void disconnect (); + void send (string data, bool is_binary= true); + + // Must be called in the event loop (e.g. im_interpose or im_main_loop) + void poll (); + + bool connected () const { return is_connected; } + + // Virtual callbacks (override to handle) + virtual void on_message (string data, bool is_binary) {} + virtual void on_connect () {} + virtual void on_disconnect () {} + virtual void on_error (string msg) {} +}; + +#endif // IM_WEBSOCKET_HPP diff --git a/tools/loro-relay/README.md b/tools/loro-relay/README.md new file mode 100644 index 0000000000..c8931fec53 --- /dev/null +++ b/tools/loro-relay/README.md @@ -0,0 +1,58 @@ +# Mogan Loro 协作编辑转发服务器 + +这是一个仅用于本地测试的 WebSocket 转发服务器。它把每个编辑器发来的 Loro 增量 update 按房间(room)广播给其他编辑器,从而可以用两个 Mogan 编辑器打开同一文件来验证协作编辑。 + +## 环境要求 + +- Node.js 18+ +- `ws` 包(运行 `npm install` 安装) + +## 启动 + +```bash +cd tools/loro-relay +npm install +node server.js +``` + +默认监听 `ws://127.0.0.1:8765`。 + +环境变量: +- `MOGAN_LORO_RELAY_HOST`:绑定地址,默认 `127.0.0.1` +- `MOGAN_LORO_RELAY_PORT`:监听端口,默认 `8765` + +## 使用 + +1. 先启动服务器。 +2. 用 Mogan 打开一个文件,并启用协作编辑(见下文)。 +3. 在另一个 Mogan 窗口打开同一文件。 +4. 在任一窗口编辑,另一窗口应自动同步。 + +## 编辑器启用方式 + +构建时打开 `loro_relay` 开关(依赖 `libloro`): + +```bash +xmake f --libloro=y --loro_relay=y --qt_frontend=n +xmake b stem +``` + +运行编辑器: + +```bash +MOGAN_LORO_RELAY=ws://127.0.0.1:8765 ./MoganSTEM .../your_file.tmu +``` + +若未设置 `MOGAN_LORO_RELAY`,默认尝试连接 `ws://127.0.0.1:8765`。连接失败时编辑器会自动降级为单机编辑。 + +## 协议 + +- 客户端连接后先发文本帧:`JOIN `,其中 `room_id` 为文档路径。 +- 之后所有二进制帧都被视为 Loro update,服务器会原样广播给同房间的其他客户端。 +- 服务器会保存每个房间最后一条消息,新加入者会收到它作为基线。 + +## 注意 + +- 这是**本地测试/开发工具**,不是生产级协作后端。 +- 目前仅支持 ImGui 后端(Qt 后端尚未适配)。 +- Loro 更新是二进制帧,不会回发给发送者自己。 diff --git a/tools/loro-relay/package-lock.json b/tools/loro-relay/package-lock.json new file mode 100644 index 0000000000..8c0c410b88 --- /dev/null +++ b/tools/loro-relay/package-lock.json @@ -0,0 +1,37 @@ +{ + "name": "mogan-loro-relay", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "mogan-loro-relay", + "version": "0.1.0", + "license": "GPL-3.0-or-later", + "dependencies": { + "ws": "^8.18.0" + } + }, + "node_modules/ws": { + "version": "8.21.1", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.1.tgz", + "integrity": "sha512-+0NTnW77fFN/DjQi6k/Sq/Yvk4Sgajw7urW8V+asjXnRgDs9gyGkdb7EzgfhA4goXsRIZKE28fzIXBHEzhuiWw==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + } + } +} diff --git a/tools/loro-relay/package.json b/tools/loro-relay/package.json new file mode 100644 index 0000000000..098a0a62b6 --- /dev/null +++ b/tools/loro-relay/package.json @@ -0,0 +1,9 @@ +{ + "name": "mogan-loro-relay", + "version": "0.1.0", + "description": "Mogan Loro CRDT collaborative editing WebSocket relay server", + "license": "GPL-3.0-or-later", + "dependencies": { + "ws": "^8.18.0" + } +} diff --git a/tools/loro-relay/server.js b/tools/loro-relay/server.js new file mode 100644 index 0000000000..2decd24c0a --- /dev/null +++ b/tools/loro-relay/server.js @@ -0,0 +1,154 @@ +#!/usr/bin/env node + +// Loro 协作编辑中继服务器(转发型)。 +// 编辑器作为 WebSocket 客户端连接,把本地 Loro update(二进制)发来; +// 本服务器按“房间(room)”把每条消息广播给除发送者外的所有连接, +// 从而让多个编辑器在同一文件上协作编辑。 +// +// 运行:npm install 然后 node server.js +// 默认监听 ws://127.0.0.1:8765 +// 环境变量:MOGAN_LORO_RELAY_HOST, MOGAN_LORO_RELAY_PORT + +const WebSocket = require('ws'); + +const HOST = process.env.MOGAN_LORO_RELAY_HOST || '0.0.0.0'; +const PORT = parseInt(process.env.MOGAN_LORO_RELAY_PORT || '8765', 10); + +const wss = new WebSocket.Server({ host: HOST, port: PORT }); + +// room -> { clients: Set, state: Buffer|null } +const rooms = new Map(); + +// 用于为每个连接分配一个递增的 ID,便于日志区分 +let nextClientId = 1; + +function getRoom(roomId) { + if (!rooms.has(roomId)) { + rooms.set(roomId, { clients: new Set(), state: null }); + console.log(`${ts()} [room:${roomId}] 房间创建`); + } + return rooms.get(roomId); +} + +function leaveRoom(ws) { + if (!ws.roomId) return; + const room = rooms.get(ws.roomId); + if (room) { + room.clients.delete(ws); + console.log( + `${ts()} [client:${ws.clientId}] 离开房间 [room:${ws.roomId}],房间当前人数: ${room.clients.size}` + ); + if (room.clients.size === 0) { + rooms.delete(ws.roomId); + console.log(`${ts()} [room:${ws.roomId}] 房间已空,销毁`); + } + } + ws.roomId = null; +} + +function joinRoom(ws, roomId) { + leaveRoom(ws); + const room = getRoom(roomId); + room.clients.add(ws); + ws.roomId = roomId; + console.log( + `${ts()} [client:${ws.clientId}] 加入房间 [room:${roomId}],房间当前人数: ${room.clients.size}` + ); + + // 如果房间里已经有状态,把最新状态发给新加入者, + // 保证后加入的编辑器能从同一基线开始同步。 + if (room.state && room.state.length > 0) { + if (ws.readyState === WebSocket.OPEN) { + ws.send(room.state); + console.log( + `${ts()} [client:${ws.clientId}] 发送房间快照,大小 ${room.state.length} 字节` + ); + } + } +} + +const LATENCY_MS = 0; + +function broadcast(ws, data) { + if (!ws.roomId) return; + const room = rooms.get(ws.roomId); + if (!room) return; + + room.state = data; + + let sentCount = 0; + + for (const client of room.clients) { + if (client === ws) continue; + if (client.readyState !== WebSocket.OPEN) continue; + + setTimeout(() => { + if (client.readyState === WebSocket.OPEN) + client.send(data); + }, LATENCY_MS); + + sentCount++; + } + + console.log( + `${ts()} 延迟 ${LATENCY_MS}ms 后广播给 ${sentCount} 个客户端` + ); +} + +wss.on('connection', (ws) => { + ws.roomId = null; + ws.clientId = nextClientId++; + console.log( + `${ts()} [client:${ws.clientId}] 新连接建立,当前总连接数: ${wss.clients.size}` + ); + + ws.on('message', (data, isBinary) => { + if (!isBinary) { + const text = data.toString('utf8').trim(); + console.log( + `${ts()} [client:${ws.clientId}] 收到文本消息: ${text.length > 100 ? text.slice(0, 100) + '...' : text}` + ); + if (text.startsWith('JOIN ')) { + const roomId = text.slice(5).trim(); + joinRoom(ws, roomId); + } + return; + } + + if (ws.roomId) { + console.log( + `${ts()} [client:${ws.clientId}] 收到二进制数据,大小 ${data.length} 字节,准备广播` + ); + broadcast(ws, data); + } else { + console.log( + `${ts()} [client:${ws.clientId}] 收到二进制数据但未加入任何房间,忽略` + ); + } + }); + + ws.on('close', (code, reason) => { + console.log( + `${ts()} [client:${ws.clientId}] 连接关闭 (code: ${code}, reason: ${reason ? reason.toString() : '无'})` + ); + leaveRoom(ws); + }); + + ws.on('error', (err) => { + console.error(`${ts()} [client:${ws.clientId}] WebSocket error:`, err); + }); +}); + +wss.on('listening', () => { + console.log(`[relay] Mogan Loro relay listening on ws://${HOST}:${PORT}`); +}); + +wss.on('error', (err) => { + console.error('Relay server error:', err); + process.exit(1); +}); + +// 简易时间戳生成 +function ts() { + return new Date().toISOString(); +} diff --git a/xmake.lua b/xmake.lua index a0ff6a2046..0b7cf5b338 100644 --- a/xmake.lua +++ b/xmake.lua @@ -102,6 +102,10 @@ else includes("xmake/targets/imgui.lua") end +if has_config("loro") then + includes("xmake/targets/loro.lua") +end + if is_plat("windows") then includes("xmake/targets/liii_windows_icon.lua") end diff --git a/xmake/options.lua b/xmake/options.lua index 8b9237d487..da3a2908ac 100644 --- a/xmake/options.lua +++ b/xmake/options.lua @@ -61,4 +61,18 @@ option_end() option("debug_with_timestamp") set_default(true) set_description("Enable timestamps in debug messages") +option_end() + +-- Loro CRDT FFI:用 cargo 构建 3rdparty/mogan-loro-ffi,并定义 LORO_ENABLED 启用 +-- tree <-> Loro snapshot 转换(loro.cpp)。默认关闭,避免拖累主构建。 +option("loro") + set_default(false) + set_description("Enable Loro CRDT FFI (builds 3rdparty/mogan-loro-ffi via cargo)") +option_end() + +-- debug_loro:编辑器的每次操作不直接应用,而是经 Loro round-trip(mirror→回译)后再应用, +-- 用于在真实编辑器里验证 Loro 全链路。需配合 libloro。默认关闭。 +option("debug_loro") + set_default(false) + set_description("Route every editor op through Loro (mirror->round-trip) to verify the full chain; needs libloro") option_end() \ No newline at end of file diff --git a/xmake/requires.lua b/xmake/requires.lua index e573cf614a..736ef4927a 100644 --- a/xmake/requires.lua +++ b/xmake/requires.lua @@ -9,9 +9,6 @@ -- in the root directory or . add_requires("liii-tbox", {system=false}) -if not is_plat("wasm") then - add_requires("cpr", {system=false}) -end -- QWK is built locally from 3rdparty/qwindowkitty, no external package needed if is_plat ("windows") then add_requires("libiconv "..LIBICONV_VERSION, {system=false}) @@ -31,6 +28,13 @@ else add_requires("libcurl", {system=false}) end end +if not is_plat("wasm") then + add_requires("cpr", {system=false}) +end + +if has_config("loro") then + add_requires("rust") +end if has_config("pdfhummus") then add_requires("liii-pdfhummus", {system=false,configs={libpng=true,libjpeg=true}}) diff --git a/xmake/targets/imgui.lua b/xmake/targets/imgui.lua index 83f8e9a81e..2d89c97d28 100644 --- a/xmake/targets/imgui.lua +++ b/xmake/targets/imgui.lua @@ -15,6 +15,13 @@ target("imgui") do set_encodings("utf-8") add_includedirs({"$(projectdir)/3rdparty/imgui", "$(projectdir)/3rdparty/imgui/backends"}, {public = true}) + if has_config("loro") then + if is_plat("wasm") then + -- no WASM support for now + else + add_packages("libcurl", {public = true}) + end + end -- The GLFW backend needs glfw; propagate it so libmogan/stem link it too. add_packages("glfw", {public = true}) diff --git a/xmake/targets/libmogan.lua b/xmake/targets/libmogan.lua index 99c380088c..3f4c9303a0 100644 --- a/xmake/targets/libmogan.lua +++ b/xmake/targets/libmogan.lua @@ -28,6 +28,11 @@ target("libmogan") do add_deps("libmoebius") add_deps("liblolly") add_deps("goldfish") + -- Loro 编辑器挂载:libloro 开启时编译 mirror_loro/apply_remote/route_through_loro + -- (libmoebius 已 public 链接 mogan_loro_ffi,这里继承)。debug_loro 再开 round-trip 路由。 + if has_config("loro") then + add_defines("LORO_ENABLED") + end if has_config("qt_frontend") then add_deps("QWKCore", "QWKWidgets") add_rules("qt.static") diff --git a/xmake/targets/loro.lua b/xmake/targets/loro.lua new file mode 100644 index 0000000000..62a2481ae3 --- /dev/null +++ b/xmake/targets/loro.lua @@ -0,0 +1,57 @@ +-- +-- MODULE : loro.lua +-- DESCRIPTION : Second-party target for the Rust Loro FFI. +-- It invokes Cargo to build libmogan_loro_ffi.a and exports all +-- required link information to dependent targets. +-- COPYRIGHT : (C) 2026 JimZhouZZY +-- +-- This software falls under the GNU general public license version 3 or later. +-- It comes WITHOUT ANY WARRANTY WHATSOEVER. For details, see the file LICENSE +-- in the root directory or . + +target("loro") + set_kind("phony") + add_packages("rust") + local ffi_dir = path.join(os.projectdir(), "3rdparty", "mogan-loro-ffi") + local profile = (is_mode("release") or is_mode("releasedbg")) + and "release" + or "dev" + local subdir = (profile == "release") + and "release" + or "debug" + + -- 导出库搜索路径与库名 + add_linkdirs(path.join(ffi_dir, "target", subdir), {public = true}) + add_links("mogan_loro_ffi", {public = true}) + + -- 跨平台系统库导出 + if is_plat("linux") then + add_syslinks("pthread", "dl", "m", "util", {public = true}) + elseif is_plat("macosx") then + add_syslinks("iconv", "resolv", "System", {public = true}) + add_frameworks("Security", "Foundation", {public = true}) + elseif is_plat("windows") then + -- 补充:Rust 静态库在 Windows 上必需的底层系统链接 + add_syslinks("userenv", "ws2_32", "bcrypt", {public = true}) + end + + before_build(function (target) + import("core.base.option") + + local args = {"build", "--manifest-path", path.join(ffi_dir, "Cargo.toml")} + table.join2(args, {"--profile", profile}) + + if option.get("verbose") then + table.insert(args, "-v") + end + + os.vrunv("cargo", args) + end) + +on_clean(function (target) + local ffi_dir = path.join(os.projectdir(), "3rdparty", "mogan-loro-ffi") + local target_dir = path.join(ffi_dir, "target") + cprint("${yellow}clean cargo target: %s", target_dir) + os.rm(target_dir) +end) +target_end()