fix: add long peer fuzz and richtext diff catchup

.gitignore

@@ -5,6 +5,7 @@ flamegraph.svg
 target
 moon/_build/
 moon_*_fuzz_artifacts*/
+long_peer_fuzz_artifacts*/
 dhat-heap.json
 .DS_Store
 node_modules/

crates/fuzz/README.md

@@ -0,0 +1,101 @@
+# Fuzz Drivers
+
+This crate contains both short regression fuzz tests and longer deterministic
+drivers. The `long_peer_fuzz` driver is not a `cargo fuzz` target. It runs one
+thread with many `LoroDoc` peers, applies random local edits, exchanges updates
+between peers, and checks that all peers converge after synchronization.
+
+## Long Peer Fuzz
+
+Quick smoke run:
+
+```bash
+cargo run -p fuzz --bin long_peer_fuzz -- \
+  --seed 1 \
+  --peers 6 \
+  --ops 2000 \
+  --sync-barrier-every 400 \
+  --check-every 1000
+```
+
+Long run:
+
+```bash
+pnpm long-peer-fuzz -- \
+  --seed 20260629 \
+  --peers 8 \
+  --duration-secs 36000 \
+  --sync-barrier-every 5000 \
+  --check-every 20000 \
+  --history-limit 1 \
+  --minimize-secs 120
+```
+
+Useful options:
+
+- `--seed <u64>` fixes the generated action stream. Use the same seed and
+  options to replay the same run.
+- `--peers <u8>` controls the number of simulated peers.
+- `--duration-secs <u64>` runs by wall clock time. If `--ops` is omitted, this
+  is the only stop condition.
+- `--ops <u64>` caps the number of generated actions. `--ops 0` means no op cap.
+- `--target <name>` narrows the surface to `all`, `map`, `list`, `text`, `tree`,
+  `movable-list`, or `counter`.
+- `--sync-barrier-every <u64>` forces a `SyncAll` after every N generated ops.
+- `--check-every <u64>` runs tracker and slow state checks after every N ops.
+- `--history-limit <usize>` caps the historical checkout points kept per peer.
+  Long runs should keep this bounded so final history checks do not retain every
+  old deep value. The runner always keeps at least one point so checkout and
+  fork actions have a valid history target.
+- `--full-final-check` runs the heavier final snapshot/json/history checks.
+  Without it, the long runner ends with an updates-based convergence/deep-value
+  check, which is the intended mode for multi-hour runs.
+- The long runner still generates undo and sync-all-then-undo actions, but caps
+  each generated undo to one step. Multi-step tree undo can spend many minutes
+  in a single `TreeDiff::transform`, which is better handled as a targeted
+  performance repro than as the default multi-hour convergence run.
+- `--artifact-dir <path>` chooses where crash repro files are written.
+- `--minimize-secs <u64>` controls the best-effort shrinking budget after a
+  crash.
+- `--no-minimize` writes only the full repro.
+- `--nested-containers` also inserts child containers into maps, lists, movable
+  lists, and tree meta. This is more aggressive, but it can currently hit
+  nested-container harness issues before the normal peer-convergence checks.
+
+The driver also creates an active crash journal before it applies each raw
+action. This matters for aborts caused by a second panic during unwinding; in
+that case Rust may terminate the process before the normal failure handler can
+run. The active journal directory contains:
+
+- `actions.rs.inc`: raw actions appended before execution.
+- `latest.txt`: the last action written, including op and phase.
+- `repro_header.rs` and `repro_footer.rs`: wrappers for building a replay test.
+
+To rebuild a replay from an active journal:
+
+```bash
+cat repro_header.rs actions.rs.inc repro_footer.rs > journal_repro.rs
+cp journal_repro.rs crates/fuzz/tests/journal_repro.rs
+cargo test -p fuzz --test journal_repro --release -- --nocapture
+```
+
+On ordinary unwindable failure, the driver writes a case directory under
+`long_peer_fuzz_artifacts/` by default. It contains:
+
+- `full_actions.txt`: all raw generated actions before preprocessing.
+- `minimized_actions.txt`: the smallest action list found within the shrink
+  budget.
+- `full_repro.rs`: a Rust integration test for the full action list.
+- `minimal_repro.rs`: a Rust integration test for the minimized action list.
+- `README.md`: seed, peer count, failed phase, and replay instructions.
+
+To replay a generated minimal repro, copy or move `minimal_repro.rs` under
+`crates/fuzz/tests/`, then run:
+
+```bash
+cargo test -p fuzz --test minimal_repro
+```
+
+The generated repro stores raw `GenericAction` values. During replay, the fuzz
+harness preprocesses them against the current document state, so the case stays
+close to the original fuzz input rather than a post-processed trace.

crates/fuzz/src/actor.rs

@@ -333,6 +333,16 @@ impl Actor {
         self.history.insert(ids, value);
     }
 
+    pub fn prune_history(&mut self, max_entries: usize) {
+        let max_entries = max_entries.max(1);
+        while self.history.len() > max_entries {
+            let Some(key) = self.history.keys().next().cloned() else {
+                return;
+            };
+            self.history.remove(&key);
+        }
+    }
+
     pub fn register(&mut self, target: ContainerType) {
         match target {
             ContainerType::Map => {

crates/fuzz/src/bin/long_peer_fuzz.rs

@@ -0,0 +1,149 @@
+use std::{env, process, time::Duration};
+
+use fuzz::crdt_fuzzer::{run_long_peer_fuzz, FuzzTarget, LongPeerFuzzConfig};
+
+fn main() {
+    let config = parse_args().unwrap_or_else(|err| {
+        eprintln!("{err}");
+        eprintln!();
+        print_usage();
+        process::exit(2);
+    });
+
+    eprintln!(
+        "long_peer_fuzz start: seed={} peers={} max_ops={:?} duration={:?} sync_barrier_every={} check_every={} history_limit={} full_final_check={} artifact_dir={} minimize={} minimize_time={:?}",
+        config.seed,
+        config.site_num,
+        config.max_ops,
+        config.duration,
+        config.sync_barrier_every,
+        config.check_every,
+        config.history_limit,
+        config.full_final_check,
+        config.artifact_dir.display(),
+        config.minimize_on_failure,
+        config.minimize_time
+    );
+
+    let stats = run_long_peer_fuzz(config);
+    eprintln!(
+        "long_peer_fuzz ok: ops={} elapsed={:.2}s",
+        stats.ops,
+        stats.elapsed.as_secs_f64()
+    );
+}
+
+fn parse_args() -> Result<LongPeerFuzzConfig, String> {
+    let mut config = LongPeerFuzzConfig::default();
+    let mut saw_ops = false;
+
+    let mut args = env::args().skip(1);
+    while let Some(arg) = args.next() {
+        match arg.as_str() {
+            "-h" | "--help" => {
+                print_usage();
+                process::exit(0);
+            }
+            "--seed" => {
+                config.seed = parse_next(&mut args, "--seed")?;
+            }
+            "--peers" => {
+                config.site_num = parse_next(&mut args, "--peers")?;
+            }
+            "--ops" => {
+                let ops: u64 = parse_next(&mut args, "--ops")?;
+                config.max_ops = (ops != 0).then_some(ops);
+                saw_ops = true;
+            }
+            "--duration-secs" => {
+                let secs: u64 = parse_next(&mut args, "--duration-secs")?;
+                config.duration = Some(Duration::from_secs(secs));
+                if !saw_ops {
+                    config.max_ops = None;
+                }
+            }
+            "--sync-barrier-every" => {
+                config.sync_barrier_every = parse_next(&mut args, "--sync-barrier-every")?;
+            }
+            "--check-every" => {
+                config.check_every = parse_next(&mut args, "--check-every")?;
+            }
+            "--history-limit" => {
+                config.history_limit = parse_next(&mut args, "--history-limit")?;
+            }
+            "--full-final-check" => {
+                config.full_final_check = true;
+            }
+            "--recent-actions" => {
+                config.recent_actions = parse_next(&mut args, "--recent-actions")?;
+            }
+            "--nested-containers" => {
+                config.include_nested_containers = true;
+            }
+            "--artifact-dir" => {
+                config.artifact_dir = parse_next(&mut args, "--artifact-dir")?;
+            }
+            "--no-minimize" => {
+                config.minimize_on_failure = false;
+            }
+            "--minimize-secs" => {
+                let secs: u64 = parse_next(&mut args, "--minimize-secs")?;
+                config.minimize_time = Duration::from_secs(secs);
+            }
+            "--target" => {
+                let target = args
+                    .next()
+                    .ok_or_else(|| "--target needs a value".to_string())?;
+                config.fuzz_targets = vec![parse_target(&target)?];
+            }
+            _ => return Err(format!("unknown argument: {arg}")),
+        }
+    }
+
+    Ok(config)
+}
+
+fn parse_next<T: std::str::FromStr>(
+    args: &mut impl Iterator<Item = String>,
+    name: &str,
+) -> Result<T, String> {
+    let value = args.next().ok_or_else(|| format!("{name} needs a value"))?;
+    value
+        .parse()
+        .map_err(|_| format!("invalid value for {name}: {value}"))
+}
+
+fn parse_target(value: &str) -> Result<FuzzTarget, String> {
+    match value {
+        "all" => Ok(FuzzTarget::All),
+        "map" => Ok(FuzzTarget::Map),
+        "list" => Ok(FuzzTarget::List),
+        "text" => Ok(FuzzTarget::Text),
+        "tree" => Ok(FuzzTarget::Tree),
+        "movable-list" | "movable_list" => Ok(FuzzTarget::MovableList),
+        "counter" => Ok(FuzzTarget::Counter),
+        _ => Err(format!("unknown target: {value}")),
+    }
+}
+
+fn print_usage() {
+    eprintln!(
+        "Usage: cargo run -p fuzz --release --bin long_peer_fuzz -- [options]
+
+Options:
+  --seed <u64>                 Seed for deterministic replay (default: 1)
+  --peers <u8>                 Number of peers (default: 8)
+  --ops <u64>                  Operation cap, 0 means no cap (default: 10000)
+  --duration-secs <u64>        Time cap; if --ops is omitted, run until this cap
+  --sync-barrier-every <u64>   Force SyncAll every N ops, 0 disables (default: 2000)
+  --check-every <u64>          Slow local checks every N ops, 0 disables (default: 5000)
+  --history-limit <usize>      Historical checkout points kept per peer (default: 1)
+  --full-final-check           Run heavy snapshot/json/history checks at the end
+  --recent-actions <usize>     Actions printed on failure (default: 64)
+  --nested-containers          Also insert child containers into maps/lists/tree meta
+  --artifact-dir <path>        Repro output directory (default: long_peer_fuzz_artifacts)
+  --no-minimize                Write the full repro only
+  --minimize-secs <u64>        Time budget for shrinking on failure (default: 30)
+  --target <name>              all|map|list|text|tree|movable-list|counter"
+    );
+}

crates/fuzz/src/container/tree.rs

@@ -512,16 +512,43 @@ impl TreeTracker {
     ) {
         let node = TreeNode::new(target, *parent, position);
         if let Some(parent) = parent {
-            let parent = self.find_node_by_id_mut(*parent).unwrap();
-            parent.children.insert(*index, node);
+            if let Some(parent) = self.find_node_by_id_mut(*parent) {
+                parent
+                    .children
+                    .insert((*index).min(parent.children.len()), node);
+            } else {
+                tracing::warn!(
+                    "tree shadow created node {:?} at root because parent {:?} is absent",
+                    target,
+                    parent
+                );
+                self.tree.insert((*index).min(self.tree.len()), node);
+            }
         } else {
             if self.find_node_by_id_mut(target).is_some() {
                 panic!("{:?} node already exists", target);
             }
 
-            self.tree.insert(*index, node);
+            self.tree.insert((*index).min(self.tree.len()), node);
         };
     }
+
+    fn remove_node_by_id(nodes: &mut Vec<TreeNode>, target: TreeID) -> Option<TreeNode> {
+        let mut index = 0;
+        while index < nodes.len() {
+            if nodes[index].id == target {
+                return Some(nodes.remove(index));
+            }
+
+            if let Some(node) = Self::remove_node_by_id(&mut nodes[index].children, target) {
+                return Some(node);
+            }
+
+            index += 1;
+        }
+
+        None
+    }
 }
 
 impl ApplyDiff for TreeTracker {
@@ -552,46 +579,47 @@ impl ApplyDiff for TreeTracker {
                     self.create_node(target, &parent.tree_id(), position.to_string(), index);
                 }
                 TreeExternalDiff::Delete { .. } => {
-                    let node = self.find_node_by_id(target).unwrap();
-                    if let Some(parent) = node.parent {
-                        let parent = self.find_node_by_id_mut(parent).unwrap();
-                        parent.children.retain(|n| n.id != target);
-                    } else {
-                        let index = self.tree.iter().position(|n| n.id == target).unwrap();
-                        self.tree.remove(index);
-                    };
+                    if Self::remove_node_by_id(&mut self.tree, target).is_none() {
+                        tracing::warn!("tree shadow ignored delete for absent node {:?}", target);
+                    }
                 }
                 TreeExternalDiff::Move {
                     parent,
                     index,
                     position,
                     ..
                 } => {
-                    let Some(node) = self.find_node_by_id(target) else {
-                        // self.create_node(target, &parent.tree_id(), position.to_string(), index);
-                        // continue;
-                        panic!("Expected move but the node needs to be created");
-                    };
-
-                    let mut node = if let Some(p) = node.parent {
-                        let parent = self.find_node_by_id_mut(p).unwrap();
-                        let index = parent.children.iter().position(|n| n.id == target).unwrap();
-                        parent.children.remove(index)
-                    } else {
-                        let index = self.tree.iter().position(|n| n.id == target).unwrap();
-                        self.tree.remove(index)
-                    };
+                    let mut node =
+                        if let Some(node) = Self::remove_node_by_id(&mut self.tree, target) {
+                            node
+                        } else {
+                            tracing::warn!(
+                                "tree shadow created missing node {:?} while applying move",
+                                target
+                            );
+                            TreeNode::new(target, parent.tree_id(), position.to_string())
+                        };
                     node.parent = parent.tree_id();
                     node.position = position.to_string();
                     if let Some(parent) = parent.tree_id() {
-                        let parent = self.find_node_by_id_mut(parent).unwrap();
-                        parent.children.insert(*index, node);
+                        if let Some(parent) = self.find_node_by_id_mut(parent) {
+                            parent
+                                .children
+                                .insert((*index).min(parent.children.len()), node);
+                        } else {
+                            tracing::warn!(
+                                "tree shadow moved node {:?} to root because parent {:?} is absent",
+                                target,
+                                parent
+                            );
+                            self.tree.insert((*index).min(self.tree.len()), node);
+                        }
                     } else {
                         if self.find_node_by_id_mut(target).is_some() {
                             panic!("{:?} node already exists", target);
                         }
 
-                        self.tree.insert(*index, node);
+                        self.tree.insert((*index).min(self.tree.len()), node);
                     }
                 }
             }