diff --git a/cli/src/services/agent_trace_dwh_replica/replica.rs b/cli/src/services/agent_trace_dwh_replica/replica.rs index 4f6597f2..576dcc00 100644 --- a/cli/src/services/agent_trace_dwh_replica/replica.rs +++ b/cli/src/services/agent_trace_dwh_replica/replica.rs @@ -28,7 +28,7 @@ use crate::services::{ /// /// The replica never discovers or persists these values itself: callers /// resolve `local_path` from the canonical -/// `agent_trace_dwh_replica_path_for_repository` helper and are responsible +/// `agent_trace_dwh_replica_path_for_dwh` helper and are responsible /// for acquiring `database_url`/`auth_token` themselves. pub struct AgentTraceDwhReplicaConfig { pub local_path: PathBuf, @@ -249,8 +249,9 @@ fn initialize_empty_schema( } /// Derive the bridge-lock path for an explicit replica path: the replica -/// path's file name with `.bridge-lock` appended, mirroring -/// `default_paths::agent_trace_dwh_bridge_lock_path_for_repository`. +/// path's file name with `.bridge-lock` appended. This is the sole source of +/// the replica's bridge-lock identity; `default_paths` derives no lock path +/// of its own. fn bridge_lock_path_for_replica(local_path: &Path) -> PathBuf { let mut file_name = local_path .file_name() diff --git a/cli/src/services/default_paths.rs b/cli/src/services/default_paths.rs index f4419fa6..ffb52cc8 100644 --- a/cli/src/services/default_paths.rs +++ b/cli/src/services/default_paths.rs @@ -307,89 +307,52 @@ pub fn agent_trace_db_path_for_repository_at( .join("agent-trace.db")) } -/// Returns the canonical repository-scoped Agent Trace DWH sync replica -/// database file path. +/// Returns the canonical DWH-scoped Agent Trace DWH sync replica database +/// file path. /// -/// The path is `/sce/repos//agent-trace-sync.db`, -/// where `state_root` comes from the shared default-path catalog -/// (`XDG_STATE_HOME` or platform equivalent) and `repository_id` is the -/// stable repository identity hash from `services::repository_identity`. -/// This path is distinct from the source-capture -/// `agent_trace_db_path_for_repository` path: the replica is a disposable, -/// single-owner Turso Sync database, never a multiprocess-WAL capture store. +/// The path is `/sce/dwh//agent-trace-sync.db`, where +/// `state_root` comes from the shared default-path catalog (`XDG_STATE_HOME` +/// or platform equivalent) and `dwh_id` is an opaque, repository-independent +/// identifier for the remote workspace Agent Trace DWH. One replica exists +/// per remote DWH; many repositories may ETL into the same replica. This +/// path is distinct from the source-capture `agent_trace_db_path_for_repository` +/// path: the replica is a disposable, single-owner Turso Sync database, +/// never a multiprocess-WAL capture store. #[allow(dead_code)] -pub fn agent_trace_dwh_replica_path_for_repository(repository_id: &str) -> anyhow::Result { +pub fn agent_trace_dwh_replica_path_for_dwh(dwh_id: &str) -> anyhow::Result { let state_root = resolve_sce_default_locations()? .roots() .state_root() .to_path_buf(); - agent_trace_dwh_replica_path_for_repository_at(&state_root, repository_id) + agent_trace_dwh_replica_path_for_dwh_at(&state_root, dwh_id) } -/// Builds the repository-scoped Agent Trace DWH sync replica database path -/// under an explicit state root: -/// `/sce/repos//agent-trace-sync.db`. +/// Builds the DWH-scoped Agent Trace DWH sync replica database path under an +/// explicit state root: `/sce/dwh//agent-trace-sync.db`. #[allow(dead_code)] -pub fn agent_trace_dwh_replica_path_for_repository_at( +pub fn agent_trace_dwh_replica_path_for_dwh_at( state_root: &std::path::Path, - repository_id: &str, -) -> anyhow::Result { - Ok( - agent_trace_dwh_replica_dir_for_repository_at(state_root, repository_id)? - .join("agent-trace-sync.db"), - ) -} - -/// Returns the canonical repository-scoped Agent Trace DWH sync replica -/// bridge-lock file path: the replica path with a `.bridge-lock` suffix. -/// -/// Exactly one process may hold this lock at a time; it is the sole -/// synchronization mechanism proving single ownership of the replica. -#[allow(dead_code)] -pub fn agent_trace_dwh_bridge_lock_path_for_repository( - repository_id: &str, -) -> anyhow::Result { - let state_root = resolve_sce_default_locations()? - .roots() - .state_root() - .to_path_buf(); - agent_trace_dwh_bridge_lock_path_for_repository_at(&state_root, repository_id) -} - -/// Builds the repository-scoped Agent Trace DWH sync replica bridge-lock -/// path under an explicit state root. -#[allow(dead_code)] -pub fn agent_trace_dwh_bridge_lock_path_for_repository_at( - state_root: &std::path::Path, - repository_id: &str, + dwh_id: &str, ) -> anyhow::Result { - let replica_path = agent_trace_dwh_replica_path_for_repository_at(state_root, repository_id)?; - let mut file_name = replica_path - .file_name() - .ok_or_else(|| anyhow::anyhow!("replica path must have a file name"))? - .to_os_string(); - file_name.push(".bridge-lock"); - Ok(replica_path.with_file_name(file_name)) + Ok(agent_trace_dwh_replica_dir_for_dwh_at(state_root, dwh_id)?.join("agent-trace-sync.db")) } #[allow(dead_code)] -fn agent_trace_dwh_replica_dir_for_repository_at( +fn agent_trace_dwh_replica_dir_for_dwh_at( state_root: &std::path::Path, - repository_id: &str, + dwh_id: &str, ) -> anyhow::Result { - let repository_id = repository_id.trim(); - if repository_id.is_empty() { - anyhow::bail!( - "repository ID must not be empty when resolving Agent Trace DWH replica path" - ); + let dwh_id = dwh_id.trim(); + if dwh_id.is_empty() { + anyhow::bail!("DWH ID must not be empty when resolving Agent Trace DWH replica path"); } - // The repository ID becomes a single path segment; reject anything that - // could escape the `repos/` directory. - if repository_id.contains(['/', '\\']) || repository_id == "." || repository_id == ".." { - anyhow::bail!("repository ID '{repository_id}' is not a valid path segment"); + // The DWH ID becomes a single path segment; reject anything that could + // escape the `dwh/` directory. + if dwh_id.contains(['/', '\\']) || dwh_id == "." || dwh_id == ".." { + anyhow::bail!("DWH ID '{dwh_id}' is not a valid path segment"); } - Ok(state_root.join("sce").join("repos").join(repository_id)) + Ok(state_root.join("sce").join("dwh").join(dwh_id)) } /// Returns the canonical default observability log directory. @@ -672,10 +635,10 @@ mod tests { use super::*; #[test] - fn agent_trace_dwh_replica_path_resolves_under_repos_and_is_distinct_from_source_db() { + fn agent_trace_dwh_replica_path_for_dwh_resolves_under_dwh_and_is_distinct_from_source_db() { let state_root = Path::new("/tmp/state"); - let replica_path = agent_trace_dwh_replica_path_for_repository_at(state_root, "repo-abc") + let replica_path = agent_trace_dwh_replica_path_for_dwh_at(state_root, "dwh-abc") .expect("replica path should resolve"); let source_path = agent_trace_db_path_for_repository_at(state_root, "repo-abc") .expect("source path should resolve"); @@ -684,8 +647,8 @@ mod tests { replica_path, state_root .join("sce") - .join("repos") - .join("repo-abc") + .join("dwh") + .join("dwh-abc") .join("agent-trace-sync.db") ); assert_ne!( @@ -699,48 +662,59 @@ mod tests { } #[test] - fn agent_trace_dwh_replica_path_bridge_lock_path_is_the_replica_path_with_a_bridge_lock_suffix() - { + fn agent_trace_dwh_replica_path_for_dwh_is_stable_and_distinct_across_dwh_ids() { let state_root = Path::new("/tmp/state"); - let replica_path = agent_trace_dwh_replica_path_for_repository_at(state_root, "repo-abc") - .expect("replica path should resolve"); - let lock_path = agent_trace_dwh_bridge_lock_path_for_repository_at(state_root, "repo-abc") - .expect("lock path should resolve"); + let dwh_a_first = agent_trace_dwh_replica_path_for_dwh_at(state_root, "dwh-A") + .expect("dwh-A replica path should resolve"); + let dwh_a_second = agent_trace_dwh_replica_path_for_dwh_at(state_root, "dwh-A") + .expect("dwh-A replica path should resolve again"); + let dwh_b = agent_trace_dwh_replica_path_for_dwh_at(state_root, "dwh-B") + .expect("dwh-B replica path should resolve"); assert_eq!( - lock_path, - replica_path.with_file_name("agent-trace-sync.db.bridge-lock") + dwh_a_first, dwh_a_second, + "the same dwh_id must always resolve to the same path" + ); + assert_ne!( + dwh_a_first, dwh_b, + "distinct dwh_id values must resolve to distinct replica paths" ); - assert_ne!(lock_path, replica_path); } #[test] - fn agent_trace_dwh_replica_path_rejects_empty_repository_id() { - let error = agent_trace_dwh_replica_path_for_repository_at(Path::new("/tmp/state"), " ") - .expect_err("empty repository ID should fail"); - assert!(error.to_string().contains("must not be empty")); + fn agent_trace_dwh_replica_path_for_dwh_never_shares_a_parent_with_the_source_path() { + let state_root = Path::new("/tmp/state"); + + for repository_id in ["repo-A", "dwh-X"] { + for dwh_id in ["dwh-X", "repo-A"] { + let source_path = agent_trace_db_path_for_repository_at(state_root, repository_id) + .expect("source path should resolve"); + let replica_path = agent_trace_dwh_replica_path_for_dwh_at(state_root, dwh_id) + .expect("replica path should resolve"); + + assert_ne!( + source_path.parent(), + replica_path.parent(), + "source and DWH replica paths must never share a parent directory" + ); + } + } } #[test] - fn agent_trace_dwh_replica_path_rejects_escaping_repository_ids() { - for bad in ["../escape", "a/b", "a\\b", ".", ".."] { - let error = - agent_trace_dwh_replica_path_for_repository_at(Path::new("/tmp/state"), bad) - .expect_err("escaping repository ID should fail"); - assert!(error.to_string().contains("not a valid path segment")); - } + fn agent_trace_dwh_replica_path_for_dwh_rejects_empty_dwh_id() { + let error = agent_trace_dwh_replica_path_for_dwh_at(Path::new("/tmp/state"), " ") + .expect_err("empty DWH ID should fail"); + assert!(error.to_string().contains("must not be empty")); } #[test] - fn agent_trace_dwh_replica_path_bridge_lock_path_rejects_the_same_invalid_repository_ids_as_the_replica_path( - ) { - for bad in ["", " ", "../escape", "a/b", "a\\b", ".", ".."] { - let replica_result = - agent_trace_dwh_replica_path_for_repository_at(Path::new("/tmp/state"), bad); - let lock_result = - agent_trace_dwh_bridge_lock_path_for_repository_at(Path::new("/tmp/state"), bad); - assert_eq!(replica_result.is_err(), lock_result.is_err()); + fn agent_trace_dwh_replica_path_for_dwh_rejects_escaping_dwh_ids() { + for bad in ["../escape", "a/b", "a\\b", ".", ".."] { + let error = agent_trace_dwh_replica_path_for_dwh_at(Path::new("/tmp/state"), bad) + .expect_err("escaping DWH ID should fail"); + assert!(error.to_string().contains("not a valid path segment")); } } } diff --git a/context/architecture.md b/context/architecture.md index 4c78c9e2..bb8e95b6 100644 --- a/context/architecture.md +++ b/context/architecture.md @@ -124,7 +124,7 @@ The repository includes a new placeholder Rust binary crate at `cli/`. - `cli/src/services/auth_db/mod.rs` provides the encrypted auth DB spec and `AuthDb` type alias over `EncryptedTursoDb`. `AuthDbSpec` resolves `/sce/auth.db` through the shared default-path seam and embeds ordered auth migrations. Auth DB lifecycle setup/doctor integration is wired through `AuthDbLifecycle`; auth command/token-storage reads/writes are directed through `token_storage.rs`. - `cli/src/services/agent_trace_db/mod.rs` owns the shared Agent Trace insert payloads, SQL constants, and typed row helpers (diff-trace/intersection/Agent Trace/message/part) plus `ensure_schema_ready_for_hooks()` consumed by the repository adapter. `cli/src/services/agent_trace_db/repository.rs` defines the sole `RepositoryAgentTraceDb` adapter over `TursoDb` with a fresh `agent-trace-repository/001_repository_schema.sql` baseline for `diff_traces` (including `payload_type`), `post_commit_patch_intersections`, `agent_traces`, `messages`, `parts`, indexes, and triggers, plus the additive `002_repository_source_instance_id.sql` migration adding `repository_metadata.source_instance_id`, `repository_metadata` validation via typed `RepositoryMetadata { repository_id, source_instance_id }`, no trace-table `checkout_id` columns, `agent_traces.agent_trace_id NOT NULL UNIQUE`, and `recent_diff_trace_patches(cutoff_time_ms, end_time_ms)` using the inclusive chronological parser without checkout filtering. Active hook runtime, setup/lifecycle storage, and trace status/list/shell flows resolve through `agent_trace_storage` and use `RepositoryAgentTraceDb`. The checkout-scoped `AgentTraceDb`/`AgentTraceDbSpec` adapter, its `agent_trace_db_path()`/`agent_trace_db_path_for_checkout()` helpers, the 15-file `cli/migrations/agent-trace/` chain, and the `sce trace --legacy` surface were removed by the `retire-legacy-agent-trace-db` plan. - `cli/src/services/agent_trace_dwh_db/mod.rs` defines `AgentTraceDwhDb = TursoDb`, a separate append-oriented destination-schema adapter for the CLI-independent Agent Trace ETL consumer, backed by a fresh `agent-trace-dwh/001_dwh_schema.sql` baseline (`repositories`, `source_instances`, `etl_watermarks`, `messages`, `message_parts`, `agent_traces`, `code_changes`, no foreign keys). Explicit-path only, reuses the `agent_trace_db` retry config key, and is not wired into any lifecycle provider, doctor/setup flow, or CLI command yet. See `context/sce/agent-trace-dwh-db.md` and the accepted decision at `context/decisions/2026-08-08-agent-trace-dwh-schema-identity-contract.md`. -- `cli/src/services/agent_trace_dwh_replica/mod.rs` defines `AgentTraceDwhReplica`, the sole owner of a Turso Sync connection to the repository-scoped, single-owner `agent-trace-sync.db` replica at `/sce/repos//agent-trace-sync.db` (distinct from both the multiprocess-WAL source `agent-trace.db` and the DWH's own explicit-path `AgentTraceDwhDbSpec`). `open()` acquires a non-blocking `BridgeLock` (`lock.rs`, over the sibling `.bridge-lock` file) before any Turso access, opens the local file through `turso::sync::Builder` without ever enabling `experimental_multiprocess_wal`, and wraps the result as an `AgentTraceDwhDb` via a narrow `TursoDb::from_connection`/`block_on` seam in `cli/src/services/db/mod.rs` to classify DWH schema state (`AgentTraceDwhDb::classify_schema_state()`): a `Ready` schema is left untouched, a genuinely `Empty` schema is initialized locally with `run_migrations()` and published with `push()`, and an `Incompatible` schema fails the open loudly without repair. It exposes lock-lifetime-bound SQL access plus `run_agent_trace_etl()`, `run_code_changes_etl()`, `pull()`, and `push()`; the ETL methods delegate bounded source extraction and atomic fact/watermark loading while preserving lock ownership, and pull/push remain separate explicit operations. It also redacts the caller-supplied auth token from every error. Callers provide `local_path`/`database_url`/`auth_token` explicitly; no credential discovery/persistence or CLI/lifecycle wiring exists yet. See `context/sce/agent-trace-dwh-replica.md` and the accepted decision at `context/decisions/2026-08-08-agent-trace-dwh-empty-remote-auto-initialization.md` (superseding `context/decisions/2026-08-08-agent-trace-dwh-turso-sync-replica-ownership.md`). +- `cli/src/services/agent_trace_dwh_replica/mod.rs` defines `AgentTraceDwhReplica`, the sole owner of a Turso Sync connection to the DWH-scoped, single-owner `agent-trace-sync.db` replica at `/sce/dwh//agent-trace-sync.db` (distinct from both the multiprocess-WAL source `agent-trace.db` and the DWH's own explicit-path `AgentTraceDwhDbSpec`) — one replica per remote Agent Trace DWH (a workspace), contributed to by many repositories, keyed by an opaque `dwh_id` rather than any `repository_id`. `open()` acquires a non-blocking `BridgeLock` (`lock.rs`, over the sibling `.bridge-lock` file) before any Turso access, opens the local file through `turso::sync::Builder` without ever enabling `experimental_multiprocess_wal`, and wraps the result as an `AgentTraceDwhDb` via a narrow `TursoDb::from_connection`/`block_on` seam in `cli/src/services/db/mod.rs` to classify DWH schema state (`AgentTraceDwhDb::classify_schema_state()`): a `Ready` schema is left untouched, a genuinely `Empty` schema is initialized locally with `run_migrations()` and published with `push()`, and an `Incompatible` schema fails the open loudly without repair. It exposes lock-lifetime-bound SQL access plus `run_agent_trace_etl()`, `run_code_changes_etl()`, `pull()`, and `push()`; the ETL methods delegate bounded source extraction and atomic fact/watermark loading while preserving lock ownership, and pull/push remain separate explicit operations. It also redacts the caller-supplied auth token from every error. Callers provide `local_path`/`database_url`/`auth_token` explicitly; no credential discovery/persistence or CLI/lifecycle wiring exists yet. See `context/sce/agent-trace-dwh-replica.md` and the accepted decision at `context/decisions/2026-08-08-agent-trace-dwh-empty-remote-auto-initialization.md` (superseding `context/decisions/2026-08-08-agent-trace-dwh-turso-sync-replica-ownership.md`). - `cli/src/services/agent_trace_etl/mod.rs` defines `AgentTraceEtl`, the CLI-independent incremental bridge from the repository source of truth to the DWH replica. It validates source metadata, uses short non-blocking `agent_traces` snapshots and bounded source-contention retry, then commits every fact/dimension batch and its `(repository_id, source_instance_id, agent_traces)` watermark in one local destination transaction. The sibling `conversation_messages_etl.rs`, `conversation_parts_etl.rs`, and `conversation_etl.rs` reuse those mechanics for logical messages and source-lineage-scoped parts, with separate `messages`/`parts` watermarks, exact text hashing, conflict checks, and no parent-message foreign key. `code_changes_etl.rs` adds the strict, bounded `diff_traces` to `code_changes` bridge: it preserves source metadata, hashes the exact original payload, derives checked `ParsedPatch` metrics, validates source-lineage identity, and atomically advances the independent watermark. Its only conversation relationship is `session_id`; no `message_id` or message causality is inferred. The `CodeChangesEtl` and conversation runners preserve destination lock ownership. Remote pull/push is separate, and the local replica is reconstructible through replay. The source audit found only schema-maintenance `updated_at` trigger bodies issuing SQL `UPDATE messages`/`UPDATE parts`; active production capture writes those tables through append-oriented insert helpers, so synchronized fields are treated as immutable after insertion and have no update CDC. - `cli/src/services/setup/mod.rs` defines the setup command contract (`SetupMode`, `SetupTarget`, `SetupRequest`, CLI flag parser/validator), an `inquire`-backed interactive target prompter (`InquireSetupTargetPrompter`), setup dispatch outcomes (proceed/cancelled), additive durable-context bootstrap (`bootstrap_context_baseline` for standalone `--bootstrap-context` and every normal successful setup path), and compile-time embedded asset access (`EmbeddedAsset`, target-scoped iterators, required-hook asset iterators/lookups). It also owns the install-time optional-workflow seam: the `OptionalWorkflow` type plus the build-generated `OPTIONAL_WORKFLOWS` catalog, a per-target `WorkflowAssetLayout` built from the existing `default_paths` command/skill directory constants (`command`/`commands`/`prompts` plus `skills`), and `iter_embedded_assets_for_setup_target_with_selection(target, selection)`, which yields every embedded asset except the `{command_dir}/{command_slug}.md` file and `{skills_dir}/{skill_slug}/` subtree of each optional workflow the selection omits. Membership is derived from the catalog's slugs rather than an enumerated file list, so a new optional workflow needs no Rust change. This filtered iterator is the only way embedded assets are enumerated; setup installs through it and doctor inspects through it, so there is no unfiltered enumeration path that could reintroduce an unselected workflow. The non-interactive selection flows through the repeatable `--workflow ` flag into `SetupRequest.optional_workflows: Option>` (`None` meaning the flag was absent); `validate_optional_workflow_slugs` checks each slug against `OPTIONAL_WORKFLOWS` during request resolution, before any write, and `run_setup_for_mode` resolves `None` to the persisted `integrations.optional_workflows` (exported as `persisted_optional_workflows`) before installing through the filtered iterator and persisting the resolved selection. The interactive selection flows through the same seam: `SetupTargetPrompter` carries `prompt_target` plus `prompt_optional_workflows(defaults)` (returning `None` for a cancelled prompt), `SetupDispatch::Proceed { mode, optional_workflows }` carries a prompted selection alongside the resolved mode, and `resolve_setup_dispatch(mode, prompter, defaults)` runs the workflow prompt only after an interactive target prompt, mapping either cancellation to `SetupDispatch::Cancelled`. The prompt module builds its `inquire::MultiSelect` from `optional_workflow_prompt_inputs(catalog, defaults)`, which returns `None` for an empty catalog (skipping the prompt) and otherwise catalog-ordered rows plus the indices to pre-check, ignoring ids absent from the catalog. `setup/command.rs` therefore resolves the repository root before dispatch, so a non-git directory fails before any prompt. For repository builds, `cli/build.rs` validates the `SCE_CLI_GENERATED_INPUT_DIR` payload and canonical-input inventories, copies the payload into Cargo `OUT_DIR/pkl-generated`, stages `cli/assets/hooks/**` under `OUT_DIR/static`, requires the staged `config/optional-workflows.json`, and generates both the setup manifest and the optional-workflow catalog (`optional_workflows.rs`, rejecting a manifest whose `schemaVersion` is not 1 or whose entries lack a non-empty `id`/`title`/`description`/`commandSlug`/`skillSlug`) in `OUT_DIR`; focused internal seams separate install-flow from prompt-flow logic; `cli/src/services/setup/command.rs` owns the `SetupCommand` payload used by the static `RuntimeCommand` enum and executes against any context implementing repo-root scoping. Its install engine/orchestrator stages embedded files and uses a unified remove-and-replace policy (removing existing targets before swapping staged content, with deterministic recovery guidance on swap failure and no backup artifact creation), and formats deterministic completion messaging; required-hook install orchestration (`install_required_git_hooks`) follows the same remove-and-replace policy (removing existing hooks before swapping staged content, with deterministic recovery guidance on swap failure). After the Git gate, setup always ensures the context baseline; context-only requests return there, while normal modes derive a repo-root-scoped context before aggregating static lifecycle provider `setup` dispatch across providers (config → local_db → auth_db → agent_trace_db → hooks when requested), so setup providers consume only repo-root access from the scoped context. - `cli/src/services/setup/mod.rs` keeps those responsibilities inside one file for now, but the current ownership split is explicit: the inline `install` module owns repository-path normalization, staging/swap install behavior, required-hook installation, and filesystem safety guards, while the inline `prompt` module owns interactive target selection and prompt styling. diff --git a/context/cli/default-path-catalog.md b/context/cli/default-path-catalog.md index e69e2949..6e77dab5 100644 --- a/context/cli/default-path-catalog.md +++ b/context/cli/default-path-catalog.md @@ -18,7 +18,7 @@ - local DB: `/sce/local.db` - default observability log directory accessor: `/sce/logs` (Linux: `$XDG_STATE_HOME/sce/logs`, or `~/.local/state/sce/logs` when `XDG_STATE_HOME` is unset) - agent trace DB (only helper): `/sce/repos/{repository_id}/agent-trace.db` via `agent_trace_db_path_for_repository(repository_id)` (plus `_at(state_root, repository_id)` for explicit roots); rejects empty or path-unsafe repository IDs. The former global-sentinel `agent_trace_db_path()` and per-checkout `agent_trace_db_path_for_checkout(checkout_id)` helpers were removed by the `retire-legacy-agent-trace-db` plan. -- Agent Trace DWH sync replica and its bridge lock: `/sce/repos/{repository_id}/agent-trace-sync.db` and `agent-trace-sync.db.bridge-lock` via `agent_trace_dwh_replica_path_for_repository(repository_id)` / `agent_trace_dwh_bridge_lock_path_for_repository(repository_id)` (plus `_at(state_root, repository_id)` explicit-root variants); same repository-ID validation as the source Agent Trace DB path, and always distinct from it. `cli/src/services/agent_trace_dwh_replica/lock.rs` defines `BridgeLock`, a non-blocking `fs4`-backed exclusive-file-lock guard over the `.bridge-lock` path: `BridgeLock::acquire` never blocks, rejects a concurrent owner with actionable guidance, and is released only by guard drop or process exit (the OS releases the underlying `flock` on file-descriptor close either way); the lock file itself is created if missing and is never deleted. Introduced by the `agent-trace-dwh-turso-sync-replica` plan; the Turso Sync open/pull/push boundary that uses this lock is `AgentTraceDwhReplica` in `cli/src/services/agent_trace_dwh_replica/replica.rs`, which acquires the lock before opening any Turso Sync connection (see `context/sce/agent-trace-dwh-replica.md`). +- Agent Trace DWH sync replica: `/sce/dwh/{dwh_id}/agent-trace-sync.db` via `agent_trace_dwh_replica_path_for_dwh(dwh_id)` (plus `_at(state_root, dwh_id)` for an explicit root). `dwh_id` is an opaque, repository-independent identifier for the remote workspace Agent Trace DWH — one local replica per remote DWH, contributed to by many repositories — validated the same way `repository_id` is (rejects empty, `.`, `..`, and path separators), and the resulting path never shares a parent with any `agent_trace_db_path_for_repository(repository_id)` source path. No CLI/lifecycle caller resolves `dwh_id` yet, so the helper carries `#[allow(dead_code)]`. `cli/src/services/agent_trace_dwh_replica/lock.rs` defines `BridgeLock`, a non-blocking `fs4`-backed exclusive-file-lock guard over the sibling `agent-trace-sync.db.bridge-lock` path; `AgentTraceDwhReplica::open` derives that lock path directly from its caller-supplied `local_path` (`bridge_lock_path_for_replica` in `replica.rs`), so `default_paths.rs` defines no separate bridge-lock path helper. `BridgeLock::acquire` never blocks, rejects a concurrent owner with actionable guidance, and is released only by guard drop or process exit (the OS releases the underlying `flock` on file-descriptor close either way); the lock file itself is created if missing and is never deleted. Introduced by the `agent-trace-dwh-turso-sync-replica` plan and re-scoped from repository to DWH identity by the `dwh-scoped-agent-trace-replica-identity` plan; the Turso Sync open/pull/push boundary that uses this lock is `AgentTraceDwhReplica` in `cli/src/services/agent_trace_dwh_replica/replica.rs`, which acquires the lock before opening any Turso Sync connection (see `context/sce/agent-trace-dwh-replica.md`). ### Repo-relative paths diff --git a/context/context-map.md b/context/context-map.md index 0db1d978..5c6a5c73 100644 --- a/context/context-map.md +++ b/context/context-map.md @@ -10,7 +10,7 @@ Primary context files: Feature/domain context: - `context/cli/cli-command-surface.md` (CLI command surface including top-level help with ASCII art banner and gradient rendering, setup install flow with the repeatable `sce setup --workflow ` optional-workflow selection and its interactive post-target multi-select, WorkOS device authorization flow + token storage behavior, attribution-only hook routing with validated post-commit `--remote-url` plumbing plus DB-backed `diff-trace` dual persistence and post-commit Agent Trace payload persistence including range `content_hash`, setup-owned local DB + repository-scoped Agent Trace DB bootstrap plus doctor DB health coverage with credential-safe repository identity diagnostics, centralized Rust SCE web URL helpers in `services::agent_trace`, nested flake release package/app installability, Cargo local install + crates.io readiness policy, and hidden `sce policy bash` command adapter for bash-policy hook callers; `sce sync` command wiring is deferred to `0.4.0`; static `RuntimeCommand` enum dispatch lives in `services/command_registry.rs`, command payload structs for help/version/completion/auth/config/setup/doctor/hooks/policy are owned by their respective `services/{name}/command.rs` files, and clap-to-runtime conversion lives in `services/parse/command_runtime.rs`) -- `context/cli/default-path-catalog.md` (canonical production CLI path-ownership contract centered on `cli/src/services/default_paths.rs`, including persisted auth/config files, named DB paths for auth/local/repository-scoped Agent Trace databases, the repository-scoped Agent Trace DWH sync replica path and its non-blocking single-owner `BridgeLock` guard, the default observability log-dir accessor consumed by config resolution with Linux `${XDG_STATE_HOME:-~/.local/state}/sce/logs` fallback semantics, repo-relative, embedded-asset, install, hook, and context-path families plus the regression guard that keeps production path ownership centralized) +- `context/cli/default-path-catalog.md` (canonical production CLI path-ownership contract centered on `cli/src/services/default_paths.rs`, including persisted auth/config files, named DB paths for auth/local/repository-scoped Agent Trace databases, the DWH-scoped Agent Trace DWH sync replica path (one replica per remote workspace DWH, contributed to by many repositories) and its non-blocking single-owner `BridgeLock` guard, the default observability log-dir accessor consumed by config resolution with Linux `${XDG_STATE_HOME:-~/.local/state}/sce/logs` fallback semantics, repo-relative, embedded-asset, install, hook, and context-path families plus the regression guard that keeps production path ownership centralized) - `context/cli/repository-identity.md` (repository identity module in `cli/src/services/repository_identity/`: pure scheme-neutral `host[:port]/path` canonicalization for SCP/`ssh://`/HTTPS/`git://` remote URLs with credential stripping, hostname lowercasing, default-port removal, and query/fragment/trailing-`.git` cleanup, trim-only explicit-identity handling, `sha256("sce-repository-id-v1\0" + canonical_identity)` repository IDs, credential-safe fieldless errors, plus the `resolve` runtime submodule applying explicit-config-then-configured-remote precedence with `git config --get remote..url` lookup, `RepositoryIdentitySource` provenance, and `.sce/config.json`-guidance resolution errors that never echo URLs; consumed by the T04 `agent_trace_storage` resolver) - `context/cli/agent-trace-storage.md` (repository-scoped Agent Trace storage resolver in `cli/src/services/agent_trace_storage/`: `AgentTraceStorageContext` inputs mirroring the `agent_trace.*` config keys, `ResolvedAgentTraceStorage` carrying repository identity + checkout ID + `/sce/repos//agent-trace.db` path + open `RepositoryAgentTraceDb` + typed `RepositoryMetadata`, two resolver families sharing one internal open core: `resolve_agent_trace_storage{,_at_state_root}` (setup/lifecycle and `sce trace status` callers) with idempotent concurrent-safe first open via bounded fast-path-then-migrate retry plus narrow one-file schema migration-metadata repair and repository metadata validation, and `resolve_agent_trace_storage_for_hook_runtime{,_at_state_root}` (hook runtime only) which never creates, migrates, or repairs schema/migration metadata and fails with `sce setup` guidance on a missing/incomplete database, path-unsafe repository ID rejection in `default_paths::agent_trace_db_path_for_repository{,_at}`, and strict never-touch boundary for any pre-migration checkout-scoped/global DB files) - `context/cli/checkout-identity.md` (current checkout identity infrastructure in `cli/src/services/checkout/`, including `/sce/checkout-id` UUIDv7 storage, setup/hook integration that creates/reuses checkout identity as repository-scoped Agent Trace diagnostic metadata, the removed per-checkout DB opener/path helper, `sce doctor` checkout identity display, and the never-touch on-disk handling of pre-migration checkout-scoped DB files that are no longer inspectable via the CLI) @@ -62,7 +62,7 @@ Feature/domain context: - `context/sce/shared-turso-db.md` (current shared `cli/src/services/db/mod.rs` Turso database infrastructure seam, including `DbSpec`, generic `TursoDb`, encrypted `EncryptedTursoDb`, build-time generated migration constants from `cli/build.rs`/Cargo `OUT_DIR`, config-driven constructor/open-connect retry via `run_with_retry_sync`, no-migration `TursoDb::open_without_migrations()` / explicit-path `open_without_migrations_at(path)` for hot runtime paths, migration-running `new()` / explicit-path `new_at(path)` / `run_migrations()` with per-database `__sce_migrations` tracking, config-driven operation retry for `execute`/`query`/`query_values`/`query_map` with a `<= 2_000ms` default query failure budget, raw-value row fetching for deterministic operator-facing rendering, row-mapping excluded from retry, generic embedded migration execution, non-mutating `migration_metadata_problems()` and `ensure_schema_ready(setup_guidance)` readiness methods on `TursoDb`, one-level explicit `transaction()`/`read_transaction()`/`TursoTransaction` seam (`transaction()` issues `BEGIN IMMEDIATE`, `read_transaction()` issues a plain non-blocking `BEGIN`; both commit-on-`Ok` with best-effort rollback on closure error or failed commit, no nested transactions/retry), and concrete wrappers for `LocalDb`, `AuthDb`, plus `RepositoryAgentTraceDb`) - `context/sce/auth-db.md` (encrypted `AuthDb = EncryptedTursoDb` adapter, canonical `/sce/auth.db` path, build-time generated `AUTH_MIGRATIONS` from `cli/migrations/auth/`, auth credential schema and updated-at trigger baseline, lifecycle setup/doctor integration, encrypted token-storage persistence, and `SCE_AUTH_DB_ENCRYPTION_KEY`/OS credential-store key handling) - `context/sce/agent-trace-db.md` (implemented Agent Trace database adapter: the sole repository-scoped `RepositoryAgentTraceDb` backed by one fresh multi-statement baseline schema file plus an additive `002_repository_source_instance_id` migration, typed `RepositoryMetadata { repository_id, source_instance_id }` with atomic once-only source-instance initialization, `repository_metadata` validation, narrow concurrent-first-open repair for missing one-file baseline migration metadata after all required schema tables exist, no trace-table `checkout_id` columns, repository-level typed insert helpers for diff traces, post-commit intersections, Agent Trace rows, messages, and parts, repository-level recent diff-trace reads without checkout filtering, on-demand command/hook initialization with no daemon/background service, and the never-touch on-disk boundary for any pre-migration checkout-scoped/global DB files; the checkout-scoped `AgentTraceDb` adapter, its `agent_trace_db_path()`/`agent_trace_db_path_for_checkout()` helpers, and the 15-file `cli/migrations/agent-trace/` chain were removed by the `retire-legacy-agent-trace-db` plan; active hook writers/readers and Agent Trace setup/lifecycle resolve repository storage through `agent_trace_storage`) -- `context/sce/agent-trace-dwh-replica.md` (Agent Trace DWH Turso Sync replica boundary: `AgentTraceDwhReplica` in `cli/src/services/agent_trace_dwh_replica/replica.rs`, the sole owner of a Turso Sync connection to the repository-scoped `agent-trace-sync.db`; acquires the `BridgeLock` before any Turso access, opens via `turso::sync::Builder` without enabling multiprocess WAL, then classifies the DWH schema via `AgentTraceDwhDb::classify_schema_state()` — a `Ready` schema is left untouched, a genuinely `Empty` schema is initialized locally with `run_migrations()` and published with `push()`, and an `Incompatible` schema fails loudly without repair — exposes lock-lifetime-bound `run_agent_trace_etl()` and `run_code_changes_etl()` plus explicit `pull()`/`push()`, redacts the caller-supplied auth token from every error, and reuses a new `TursoDb::from_connection`/`block_on` seam; credential discovery/persistence and CLI/lifecycle wiring remain deferred) +- `context/sce/agent-trace-dwh-replica.md` (Agent Trace DWH Turso Sync replica boundary: `AgentTraceDwhReplica` in `cli/src/services/agent_trace_dwh_replica/replica.rs`, the sole owner of a Turso Sync connection to the DWH-scoped `agent-trace-sync.db` (one replica per remote Agent Trace DWH, contributed to by many repositories); acquires the `BridgeLock` before any Turso access, opens via `turso::sync::Builder` without enabling multiprocess WAL, then classifies the DWH schema via `AgentTraceDwhDb::classify_schema_state()` — a `Ready` schema is left untouched, a genuinely `Empty` schema is initialized locally with `run_migrations()` and published with `push()`, and an `Incompatible` schema fails loudly without repair — exposes lock-lifetime-bound `run_agent_trace_etl()` and `run_code_changes_etl()` plus explicit `pull()`/`push()`, redacts the caller-supplied auth token from every error, and reuses a new `TursoDb::from_connection`/`block_on` seam; credential discovery/persistence and CLI/lifecycle wiring remain deferred) - `context/sce/agent-trace-dwh-sync.md` (`AgentTraceDwhSync` orchestration boundary in `cli/src/services/agent_trace_dwh_sync.rs` composing `AgentTraceDwhReplica` with `AgentTraceEtl`, `ConversationEtl`, and `CodeChangesEtl` into one `run()` call — open replica, pull once, run the three ETLs in order, push once on full success — returning combined `AgentTraceDwhSyncStats` and a stage-tagged `AgentTraceDwhSyncError`; documents the observed Turso Sync `pulled_changes` reconciliation-echo behavior after any fresh open following a push; core service and empty-remote/no-op proof only so far, full failure/recovery/convergence semantics land with later `agent-trace-dwh-sync` plan tasks) - `context/sce/agent-trace-dwh-db.md` (Agent Trace DWH: a separate append-oriented destination schema for the CLI-independent ETL consumer, distinct from the repository-scoped source schema above. `AgentTraceDwhDb = TursoDb` in `cli/src/services/agent_trace_dwh_db/mod.rs`, explicit-path only, no lifecycle/CLI wiring yet. One fresh baseline `cli/migrations/agent-trace-dwh/001_dwh_schema.sql` creates `repositories`, `source_instances`, `etl_watermarks`, `messages`, `message_parts`, `agent_traces`, and `code_changes` with no foreign keys; every fact table denormalizes `repository_id`/`source_instance_id` lineage as plain text. Deterministic logical identities (messages, Agent Traces) are unique excluding `source_instance_id` for cross-source-database idempotency; raw local source row IDs (`source_part_id`, `source_diff_trace_id`) are unique per source instance so the same local integer coexists across sources/repositories) - `context/sce/agent-trace-etl.md` (shared Agent Trace ETL mechanics and the `agent_traces`, `messages`, `parts`, and `code_changes` bridges between the `agent-trace.db` source and DWH replica; covers bounded extraction, exact-content transformation/hashing, table-specific identity validation, atomic per-lineage watermark advancement, source contention handling, stats, replica-owned orchestration, and the code-change session-only relationship) diff --git a/context/glossary.md b/context/glossary.md index 6c8a6d28..f6fb3820 100644 --- a/context/glossary.md +++ b/context/glossary.md @@ -75,8 +75,8 @@ - `agent trace DB adapter`: Modules under `cli/src/services/agent_trace_db/` that define the sole repository-scoped `RepositoryAgentTraceDb = TursoDb` adapter (the checkout-scoped `AgentTraceDb`/`AgentTraceDbSpec` adapter and its 15-file migration chain were removed by the `retire-legacy-agent-trace-db` plan). The repository adapter uses one fresh `agent-trace-repository` schema file with `repository_metadata`, repository-level `diff_traces`, `post_commit_patch_intersections`, `agent_traces`, `messages`, and `parts` tables, no row-level `checkout_id`, typed parameterized insert helpers, and chronological recent `diff_traces` query/parse support. `AgentTraceDbLifecycle` initializes/checks repository-scoped storage through `agent_trace_storage`. - `Agent Trace DWH`: Separate, append-oriented destination schema for the CLI-independent Agent Trace ETL consumer, distinct from the repository-scoped `agent-trace.db` source schema. `AgentTraceDwhDb = TursoDb` in `cli/src/services/agent_trace_dwh_db/mod.rs`, explicit-path only (no canonical `db_path()`), reuses the `"agent_trace_db"` retry config key, and is not wired into lifecycle, doctor/setup, or CLI commands. One fresh baseline `cli/migrations/agent-trace-dwh/001_dwh_schema.sql` creates the lineage dimensions, extensible watermarks, and facts with no foreign keys; every fact denormalizes repository/source lineage so out-of-order or partial ingestion is not blocked. `AgentTraceEtl` loads `agent_traces` facts and watermarks atomically through the lock-owned replica. See `context/sce/agent-trace-etl.md` and `context/sce/agent-trace-dwh-db.md`. - `DWH logical identity`: Uniqueness scope used by `Agent Trace DWH` fact tables. Deterministic source identities (`messages` on `(repository_id, session_id, message_id)`, `agent_traces` on `(repository_id, agent_trace_id)`) exclude `source_instance_id` so re-ingesting the same logical event from an independently created source database for the same repository stays idempotent. Raw local autoincrement source row IDs (`message_parts.source_part_id`, `code_changes.source_diff_trace_id`) are instead scoped by `(repository_id, source_instance_id, )`, since local IDs are not stable across independently created source databases and are expected to coexist across source instances/repositories. -- `Agent Trace DWH sync replica`: The repository-scoped local `/sce/repos/{repository_id}/agent-trace-sync.db` path resolved by `agent_trace_dwh_replica_path_for_repository(repository_id)` in `cli/src/services/default_paths.rs`, distinct from both the source `agent-trace.db` path and the DWH's own explicit-path `AgentTraceDwhDbSpec`. `AgentTraceDwhReplica` is the disposable single-owner local database boundary: it acquires the `bridge lock` first, then bootstraps/pulls/pushes through Turso Sync. Its `run_agent_trace_etl()` delegates local fact/watermark loading to `AgentTraceEtl` without invoking pull/push, and the local state is replay/reconstruction-safe. See [sce/agent-trace-etl.md](sce/agent-trace-etl.md), [sce/agent-trace-dwh-replica.md](sce/agent-trace-dwh-replica.md), [cli/default-path-catalog.md](cli/default-path-catalog.md). -- `bridge lock`: The non-blocking, single-owner OS file lock guarding an `Agent Trace DWH sync replica` path, implemented as `BridgeLock` in `cli/src/services/agent_trace_dwh_replica/lock.rs` over the sibling `agent-trace-sync.db.bridge-lock` file (`agent_trace_dwh_bridge_lock_path_for_repository(repository_id)`). Backed by the `fs4` crate's non-blocking exclusive file lock; acquisition never blocks, a concurrently held lock is rejected with actionable guidance, and the OS releases the lock only on guard drop or owning-process exit — the lock file itself is created if missing and is never deleted. Holding a bridge lock never blocks the multiprocess-WAL source `agent-trace.db`. See [cli/default-path-catalog.md](cli/default-path-catalog.md). +- `Agent Trace DWH sync replica`: The DWH-scoped local `/sce/dwh/{dwh_id}/agent-trace-sync.db` path resolved by `agent_trace_dwh_replica_path_for_dwh(dwh_id)` in `cli/src/services/default_paths.rs`, distinct from both the source `agent-trace.db` path and the DWH's own explicit-path `AgentTraceDwhDbSpec`. One replica exists per remote Agent Trace DWH (a workspace), and many repositories may ETL into the same replica; `dwh_id` is an opaque, repository-independent identifier and never a `repository_id`. `AgentTraceDwhReplica` is the disposable single-owner local database boundary: it acquires the `bridge lock` first, then bootstraps/pulls/pushes through Turso Sync. Its `run_agent_trace_etl()` delegates local fact/watermark loading to `AgentTraceEtl` without invoking pull/push, and the local state is replay/reconstruction-safe. See [sce/agent-trace-etl.md](sce/agent-trace-etl.md), [sce/agent-trace-dwh-replica.md](sce/agent-trace-dwh-replica.md), [cli/default-path-catalog.md](cli/default-path-catalog.md). +- `bridge lock`: The non-blocking, single-owner OS file lock guarding an `Agent Trace DWH sync replica` path, implemented as `BridgeLock` in `cli/src/services/agent_trace_dwh_replica/lock.rs` over the sibling `agent-trace-sync.db.bridge-lock` file. `AgentTraceDwhReplica::open` derives this lock path directly from its caller-supplied `local_path` (`bridge_lock_path_for_replica` in `replica.rs`); `default_paths.rs` defines no separate bridge-lock path helper. Backed by the `fs4` crate's non-blocking exclusive file lock; acquisition never blocks, a concurrently held lock is rejected with actionable guidance, and the OS releases the lock only on guard drop or owning-process exit — the lock file itself is created if missing and is never deleted. Holding a bridge lock never blocks the multiprocess-WAL source `agent-trace.db`. See [cli/default-path-catalog.md](cli/default-path-catalog.md). - `structured patch service`: Pure synchronous Rust service in `cli/src/services/structured_patch.rs` that derives supported structured editor hook payloads into canonical `ParsedPatch` values. The current implemented source is Claude `PostToolUse` payloads for `Write` creates and `Edit` structured patches; wired into `sce hooks diff-trace` for Claude payload classification at intake (T04) and into `RepositoryAgentTraceDb::recent_diff_trace_patches` for post-commit structured payload parsing dispatch at read time (T05). - `Agent Trace SCE metadata`: Implementation-owned top-level metadata emitted by `build_agent_trace(...)` as `metadata.sce.version`; the value is sourced from the compiled `sce` CLI package version via `env!("CARGO_PKG_VERSION")`, is schema-validated with the rest of the payload, and is persisted in AgentTraceDb `agent_traces.trace_json` without changing the top-level Agent Trace payload/schema `version`. - `Agent Trace range content_hash`: Per-range `content_hash` emitted by `build_agent_trace(...)` inside every `ranges[]` entry as `murmur3:`, computed from the touched-line kind/content of the `post_commit_patch` or embedded-patch hunk used to emit that range while excluding positions, paths, metadata, and database IDs. diff --git a/context/plans/dwh-scoped-agent-trace-replica-identity.md b/context/plans/dwh-scoped-agent-trace-replica-identity.md new file mode 100644 index 00000000..554d5f57 --- /dev/null +++ b/context/plans/dwh-scoped-agent-trace-replica-identity.md @@ -0,0 +1,113 @@ +# Plan: dwh-scoped-agent-trace-replica-identity + +## Change summary + +Corrects the storage identity of the Agent Trace DWH Turso Sync replica (`agent-trace-sync.db`). It currently resolves under `/sce/repos//agent-trace-sync.db`, which wrongly ties one local replica to one repository. The architecture is one Agent Trace DWH per workspace, contributed to by many repositories, so the replica must resolve under `/sce/dwh//agent-trace-sync.db` instead — an opaque, repository-independent identifier for the remote workspace DWH. + +This replaces the repository-scoped default-path helpers in `cli/src/services/default_paths.rs` with DWH-scoped equivalents, removes the now-redundant repository-scoped bridge-lock path helpers (dead code today — `AgentTraceDwhReplica` already derives its lock path from the caller-supplied `local_path`), and updates durable context describing the old identity. The source Agent Trace DB path (`agent_trace_db_path_for_repository`), the DWH schema, `AgentTraceDwhReplicaConfig`, and `AgentTraceDwhSync`'s orchestration API are all unaffected — this plan is a persistence-path correction only, not a behavior or schema change. No control-plane, CLI, or credential-discovery work is added; a future caller resolves `dwh_id` and passes it to the helpers this plan adds. + +## Acceptance criteria + +- [x] AC1: `agent_trace_dwh_replica_path_for_dwh_at(state_root, dwh_id)` resolves `/sce/dwh//agent-trace-sync.db` for any `dwh_id`, independent of any repository ID — the function signature does not accept one. + - Validate: `cargo test --manifest-path cli/Cargo.toml default_paths::tests -- dwh` +- [x] AC2: Two distinct `dwh_id` values resolve to distinct replica paths and distinct bridge-lock paths, and the same `dwh_id` always resolves to the same path. + - Validate: `cargo test --manifest-path cli/Cargo.toml default_paths::tests -- dwh` +- [x] AC3: The repository source path (`agent_trace_db_path_for_repository`) is unchanged and never shares a parent directory with any DWH replica path, for any `repository_id`/`dwh_id` pair. + - Validate: `cargo test --manifest-path cli/Cargo.toml default_paths::tests -- dwh` +- [x] AC4: Invalid `dwh_id` path segments (`""`, `"."`, `".."`, values containing `/` or `\`) are rejected by the DWH replica path resolver, matching the existing repository-ID validation behavior. + - Validate: `cargo test --manifest-path cli/Cargo.toml default_paths::tests -- dwh` +- [x] AC5: No repository-scoped DWH replica or bridge-lock path helper remains in the codebase. + - Validate: `grep -rn "agent_trace_dwh_replica_path_for_repository\|agent_trace_dwh_bridge_lock_path_for_repository" cli/src` returns nothing +- [x] AC6: `AgentTraceDwhReplicaConfig`, `AgentTraceDwhReplica::open`, and `AgentTraceDwhSync::run` keep their current signatures (`local_path`/`database_url`/`auth_token`; `repository_id`/`source`/`replica_config`) — no `dwh_id`/`workspace_id` field or parameter is added to any of them. + - Validate: `git diff --stat cli/src/services/agent_trace_dwh_replica cli/src/services/agent_trace_dwh_sync.rs` shows no signature changes outside doc comments +- [x] AC7: Durable context describes the replica as workspace/DWH-scoped, with many repositories able to ETL into the same replica, and no longer describes it as repository-scoped. + - Validate: `grep -rn "repository-scoped.*agent-trace-sync\|repos/{repository_id}/agent-trace-sync\|repos//agent-trace-sync" context/` returns nothing + +### Full validation + +- `cargo test --manifest-path cli/Cargo.toml` +- `nix flake check` + +### Context sync + +- `context/cli/default-path-catalog.md` +- `context/glossary.md` +- `context/architecture.md` +- `context/sce/agent-trace-dwh-replica.md` +- `context/sce/agent-trace-dwh-sync.md` + +## Constraints and non-goals + +- **In scope:** `cli/src/services/default_paths.rs` (path helpers and their tests); doc-comment references to the renamed helpers in `cli/src/services/agent_trace_dwh_replica/replica.rs`; the five context files listed under Context sync. +- **Out of scope:** `AgentTraceDwhReplica`, `AgentTraceDwhReplicaConfig`, `AgentTraceDwhSync`, any ETL module, the Agent Trace source DB schema, the DWH schema, watermark identities, credential handling, control-plane APIs, CLI commands (`sce trace sync` or otherwise), and migration/import of any existing repository-scoped replica file on disk. +- **Constraints:** `dwh_id` is validated only as an opaque path segment (same rejection rules already used for `repository_id`: empty, `.`, `..`, `/`, `\`) — its contents are never interpreted, and it is not required to be a UUID. +- **Non-goal:** Do not add a repository-scoped-to-DWH-scoped migration path for on-disk replica files. The old `sce/repos//agent-trace-sync.db` file (pre-production, never shipped) is left untouched — not copied, renamed, or deleted. A new DWH-scoped replica bootstraps normally from the remote. +- **Non-goal:** Do not add a `workspace_id` concept to the path or to any type. `dwh_id` is the sole storage identity for this plan. + +## Assumptions + +- New DWH-scoped helpers keep the same `#[allow(dead_code)]` posture the repository-scoped helpers currently carry: no CLI/lifecycle caller is wired yet (that remains explicitly out of scope, per non-goals), so the lint attribute stays until a future caller resolves `dwh_id` and calls them. +- The removed bridge-lock helpers (`agent_trace_dwh_bridge_lock_path_for_repository[_at]`) are deleted outright rather than renamed to a DWH-scoped equivalent: they are `#[allow(dead_code)]` today with no real caller, and `AgentTraceDwhReplica::open` already derives its lock path from the caller-supplied `local_path` (see `bridge_lock_path_for_replica` in `replica.rs`), so a parallel default-path lock helper would be a second, unused source of the same identity. +- The internal-only helper `agent_trace_dwh_replica_dir_for_repository_at` is renamed to `agent_trace_dwh_replica_dir_for_dwh_at` alongside the public rename, since it exists solely to build the public path. + +## Task stack + +- [x] T01: `Replace repository-scoped DWH replica path helpers with DWH-scoped equivalents` (status:done) + - Task ID: T01 + - Goal: In `cli/src/services/default_paths.rs`, replace `agent_trace_dwh_replica_path_for_repository`/`_at` with `agent_trace_dwh_replica_path_for_dwh`/`_at` resolving `/sce/dwh//agent-trace-sync.db`, remove `agent_trace_dwh_bridge_lock_path_for_repository`/`_at` and their doc-comment references, rename the internal directory helper, and add/replace tests proving path identity, cross-DWH isolation, source/destination separation, and path-traversal rejection for `dwh_id`. + - Boundaries (in/out of scope): In — `default_paths.rs` public/internal helpers and their unit tests; the stale doc-comment reference to the old helper name in `cli/src/services/agent_trace_dwh_replica/replica.rs`. Out — any change to `AgentTraceDwhReplica`, `AgentTraceDwhReplicaConfig`, `AgentTraceDwhSync`, or `agent_trace_db_path_for_repository`. + - Dependencies: none + - Done when: `agent_trace_dwh_replica_path_for_dwh_at(state_root, "dwh-A")` returns `/sce/dwh/dwh-A/agent-trace-sync.db`; `dwh-A` and `dwh-B` resolve to distinct replica paths; `repository_id = "repo-A"` and `dwh_id = "dwh-X"` resolve to `sce/repos/repo-A/agent-trace.db` and `sce/dwh/dwh-X/agent-trace-sync.db` respectively with no shared parent; `""`, `"."`, `".."`, `"a/b"`, `"a\b"` are rejected as `dwh_id`; no repository-scoped DWH path/lock helper remains in `cli/src`. + - Verification notes (commands or checks): `cargo test --manifest-path cli/Cargo.toml default_paths::tests`; `grep -rn "agent_trace_dwh_replica_path_for_repository\|agent_trace_dwh_bridge_lock_path_for_repository" cli/src` + - Evidence: Renamed `agent_trace_dwh_replica_path_for_repository[_at]` → `agent_trace_dwh_replica_path_for_dwh[_at]` and the internal `agent_trace_dwh_replica_dir_for_repository_at` → `agent_trace_dwh_replica_dir_for_dwh_at` in `cli/src/services/default_paths.rs`, switching the resolved segment from `repos/` to `dwh/` with the same empty/`.`/`..`/`/`/`\` validation. Deleted `agent_trace_dwh_bridge_lock_path_for_repository`/`_at` outright (per plan assumption — `bridge_lock_path_for_replica` in `replica.rs` already derives the lock path from the caller-supplied `local_path`). Updated the two stale doc-comment references in `cli/src/services/agent_trace_dwh_replica/replica.rs` (lines 31 and the `bridge_lock_path_for_replica` doc comment) to stop naming the removed/renamed helpers. Rewrote the `default_paths.rs` test module: renamed tests to `dwh_id` terminology, added a same-ID-stability/cross-DWH-distinctness test and a source/replica no-shared-parent test, kept empty/escaping-ID rejection tests, and dropped the two bridge-lock tests (helper removed). `agent_trace_db_path_for_repository[_at]`, `AgentTraceDwhReplica`, `AgentTraceDwhReplicaConfig`, and `AgentTraceDwhSync` were not touched. + - Verification: `nix flake check` (the repo's bash policy blocks direct `cargo test`/`cargo fmt --check` invocations and requires this instead) — passed on a clean rerun (an initial run hit 3 pre-existing, unrelated flaky SQLite-lock failures in `agent_trace_db`/`agent_trace_dwh_db` tests that reproduce with or without this change; a second `nix flake check` run passed with all tests green, confirming this change caused no regression). All 5 new/renamed `default_paths::tests::agent_trace_dwh_replica_path_for_dwh_*` tests passed. `grep -rn "agent_trace_dwh_replica_path_for_repository\|agent_trace_dwh_bridge_lock_path_for_repository" cli/src` returns no matches (AC5 satisfied). + +- [x] T02: `Update durable context for DWH-scoped replica identity` (status:done) + - Task ID: T02 + - Goal: Update `context/cli/default-path-catalog.md`, `context/glossary.md`, `context/architecture.md`, `context/sce/agent-trace-dwh-replica.md`, and `context/sce/agent-trace-dwh-sync.md` so the replica is described as one local replica per remote Agent Trace DWH (workspace-scoped), with many repositories able to ETL into the same replica, referencing the new `agent_trace_dwh_replica_path_for_dwh`/`_at` helpers and the `sce/dwh//` path instead of the old repository-scoped description. + - Boundaries (in/out of scope): In — the five listed context files. Out — historical/completed plan files under `context/plans/` (`agent-trace-dwh-turso-sync-replica.md`, `agent-trace-dwh-sync.md`), which stay as historical records of what was built at the time; out — any decision-record file (superseding language belongs in a new decision only if a reader would otherwise be misled about current identity, which this task's target files already resolve). + - Dependencies: T01 + - Done when: none of the five files describe the replica or its bridge lock as repository-scoped; each instead states one replica per remote DWH, keyed by `dwh_id`, with many repositories contributing to it. + - Verification notes (commands or checks): `grep -rn "repository-scoped.*agent-trace-sync\|repos/{repository_id}/agent-trace-sync\|repos//agent-trace-sync" context/` + - Evidence: No new edits were required — commit `3050c9e` (T01's own implementation commit) already performed this synchronization as part of the same change: it updated `context/architecture.md`, `context/cli/default-path-catalog.md`, `context/glossary.md`, and `context/sce/agent-trace-dwh-replica.md` to describe the replica as one per remote Agent Trace DWH keyed by `dwh_id` with many repositories contributing, referencing `agent_trace_dwh_replica_path_for_dwh`/`_at` and the `sce/dwh//` path, and also updated `context/context-map.md`'s corresponding entry. `context/sce/agent-trace-dwh-sync.md` never described replica storage scoping (it documents orchestration only, not path identity), so it required no change to satisfy the done check. Confirmed all repository-scoped language and paths are gone from the five target files. + - Verification: `grep -rn "repository-scoped.*agent-trace-sync\|repos/{repository_id}/agent-trace-sync\|repos//agent-trace-sync" context/cli/default-path-catalog.md context/glossary.md context/architecture.md context/sce/agent-trace-dwh-replica.md context/sce/agent-trace-dwh-sync.md` — no matches (AC7 satisfied). + +## Open questions + +None. The change request fully specifies the target path shape, validation rules, API boundaries, non-goals, and documentation scope; no scope, criterion, or ordering decision was left open. + +## Validation Report + +**Status:** validated +**Date:** 2026-08-10 + +### Commands run + +- `cargo test --manifest-path cli/Cargo.toml` -> blocked by repo bash policy (`use-nix-flake-check-over-cargo-test`); substituted with `nix flake check` per repository convention (see T01 evidence for the same substitution). +- `nix flake check` -> exit 0 (all checks passed; `cli-tests` derivation cached from unchanged inputs since the last successful build). +- `nix log .#checks.x86_64-linux.cli-tests` (inspection of the cached build log) -> `test result: ok. 294 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out`, including all 5 `default_paths::tests::agent_trace_dwh_replica_path_for_dwh_*` tests (AC1–AC4 evidence). +- `grep -rn "agent_trace_dwh_replica_path_for_repository\|agent_trace_dwh_bridge_lock_path_for_repository" cli/src` -> exit 1, no matches (AC5). +- `git diff --stat cli/src/services/agent_trace_dwh_replica cli/src/services/agent_trace_dwh_sync.rs` -> empty (no uncommitted changes); `git diff --stat e3d8031 3050c9e -- cli/src/services/agent_trace_dwh_replica cli/src/services/agent_trace_dwh_sync.rs` -> only `replica.rs` changed (4 insertions, 3 deletions, doc comments only per inspection); `agent_trace_dwh_sync.rs` untouched by this plan (AC6). +- `grep -rn "repository-scoped.*agent-trace-sync\|repos/{repository_id}/agent-trace-sync\|repos//agent-trace-sync" context/` -> returns 3 matches outside this plan's five target files, all pre-existing and unrelated to replica storage scoping (see Residual risks). The same command scoped to the five target files (`context/cli/default-path-catalog.md context/glossary.md context/architecture.md context/sce/agent-trace-dwh-replica.md context/sce/agent-trace-dwh-sync.md`) returns no matches (AC7). + +### Scaffolding removed + +None. + +### Success-criteria verification + +- [x] AC1: `agent_trace_dwh_replica_path_for_dwh_at` resolves `/sce/dwh//agent-trace-sync.db`, no repository ID accepted -> `agent_trace_dwh_replica_path_for_dwh_resolves_under_dwh_and_is_distinct_from_source_db` passed. +- [x] AC2: distinct/stable `dwh_id` resolution -> `agent_trace_dwh_replica_path_for_dwh_is_stable_and_distinct_across_dwh_ids` passed. +- [x] AC3: no shared parent with source path -> `agent_trace_dwh_replica_path_for_dwh_never_shares_a_parent_with_the_source_path` passed. +- [x] AC4: invalid `dwh_id` segments rejected -> `agent_trace_dwh_replica_path_for_dwh_rejects_empty_dwh_id` and `agent_trace_dwh_replica_path_for_dwh_rejects_escaping_dwh_ids` passed. +- [x] AC5: no repository-scoped DWH helper remains -> grep returns nothing. +- [x] AC6: `AgentTraceDwhReplicaConfig`/`AgentTraceDwhReplica::open`/`AgentTraceDwhSync::run` signatures unchanged -> diff since the plan's baseline commit touches only doc comments in `replica.rs`; `agent_trace_dwh_sync.rs` untouched. +- [x] AC7: durable context no longer describes the replica as repository-scoped -> the five target files are clean; the plan's own change summary and T02 evidence confirm scope. + +### Failed checks and follow-ups + +None. + +### Residual risks + +- The AC7 `Validate:` grep, run unscoped across all of `context/`, also matches `context/sce/agent-trace-etl.md` and two historical plan files (`context/plans/incremental-agent-trace-etl-transactional-watermarks.md`, `context/plans/agent-trace-dwh-turso-sync-replica.md`). These are pre-existing, incidental regex collisions (the sentence structure places "repository-scoped" near a different noun — the source `agent-trace.db` — not the replica), and the two plan files are explicitly out of scope per T02's boundaries as historical records. None of these describe the DWH sync replica itself as repository-scoped. Not a defect introduced by this plan. diff --git a/context/sce/agent-trace-dwh-replica.md b/context/sce/agent-trace-dwh-replica.md index 934b72cd..1515031c 100644 --- a/context/sce/agent-trace-dwh-replica.md +++ b/context/sce/agent-trace-dwh-replica.md @@ -1,12 +1,12 @@ # Agent Trace DWH Turso Sync Replica -`AgentTraceDwhReplica` is the sole owner of a Turso Sync connection to a repository's `agent-trace-sync.db` — a disposable, single-owner local database distinct from both the multiprocess-WAL source `agent-trace.db` (see [agent-trace-db.md](agent-trace-db.md)) and the `Agent Trace DWH`'s own explicit-path adapter (see [agent-trace-dwh-db.md](agent-trace-dwh-db.md)). It is the boundary used by the CLI-independent `AgentTraceEtl`, `ConversationEtl`, and `CodeChangesEtl` bridges for local fact/watermark loading; ETL never performs pull/push, credential discovery/persistence, or background sync. +`AgentTraceDwhReplica` is the sole owner of a Turso Sync connection to a remote Agent Trace DWH's (a workspace's) `agent-trace-sync.db` — a disposable, single-owner local database distinct from both the multiprocess-WAL source `agent-trace.db` (see [agent-trace-db.md](agent-trace-db.md)) and the `Agent Trace DWH`'s own explicit-path adapter (see [agent-trace-dwh-db.md](agent-trace-dwh-db.md)). One replica exists per remote DWH; many repositories may ETL into the same replica, since the replica's identity is keyed by an opaque `dwh_id` rather than any single repository. It is the boundary used by the CLI-independent `AgentTraceEtl`, `ConversationEtl`, and `CodeChangesEtl` bridges for local fact/watermark loading; ETL never performs pull/push, credential discovery/persistence, or background sync. ## Ownership and lock-before-open `cli/src/services/agent_trace_dwh_replica/replica.rs` defines `AgentTraceDwhReplica::open(config: AgentTraceDwhReplicaConfig)`, where `AgentTraceDwhReplicaConfig { local_path, database_url, auth_token }` are all caller-supplied explicit values — the replica never discovers, stores, or persists credentials itself. `open()`: -1. Derives the sibling `.bridge-lock` path from `local_path` (the same suffix convention as `agent_trace_dwh_bridge_lock_path_for_repository`, see [../cli/default-path-catalog.md](../cli/default-path-catalog.md)) and acquires a `BridgeLock` **before** any Turso access. A concurrently held lock fails the whole call with `AgentTraceDwhReplicaError::Lock` before a Turso Sync builder, the local file, or the network is ever touched. +1. Derives the sibling `.bridge-lock` path directly from `local_path` (`bridge_lock_path_for_replica` in `replica.rs`; `default_paths.rs` defines no separate bridge-lock path helper, see [../cli/default-path-catalog.md](../cli/default-path-catalog.md)) and acquires a `BridgeLock` **before** any Turso access. A concurrently held lock fails the whole call with `AgentTraceDwhReplicaError::Lock` before a Turso Sync builder, the local file, or the network is ever touched. 2. Opens the local file through `turso::sync::Builder::new_remote(local_path).with_remote_url(..).with_auth_token(..)`, never calling `.experimental_multiprocess_wal(true)` — that flag is reserved for the source capture database this replica never opens. 3. Wraps the resulting connection into `AgentTraceDwhDb` via a narrow `TursoDb::from_connection(conn, runtime)` seam (see [shared-turso-db.md](shared-turso-db.md)) and classifies its schema state via `AgentTraceDwhDb::classify_schema_state()` (see [agent-trace-dwh-db.md](agent-trace-dwh-db.md)). A `Ready` schema is left untouched. A genuinely `Empty` schema is initialized locally with `AgentTraceDwhDb::run_migrations()` and published with a single `push()`, narrowly recovering from a push conflict with one best-effort `pull()` plus a readiness re-verification (the *original* push failure is returned unless that re-verification now reports ready, in which case another initializer is treated as having won the race). An `Incompatible` schema — an unrelated schema, a partial DWH schema, or a migration ledger with unexpected entries — fails the whole call as `AgentTraceDwhReplicaError::IncompatibleSchema`, without repairing or partially completing it. diff --git a/context/sce/shared-turso-db.md b/context/sce/shared-turso-db.md index 6fad2518..5c331e46 100644 --- a/context/sce/shared-turso-db.md +++ b/context/sce/shared-turso-db.md @@ -65,7 +65,7 @@ The shared module is exported from `cli/src/services/mod.rs` and compile-checked - `cli/src/services/agent_trace_db/mod.rs`: owns the shared Agent Trace insert payloads/helpers. Active hook/runtime paths use the sole `RepositoryAgentTraceDb = TursoDb` adapter from `cli/src/services/agent_trace_db/repository.rs`, selected by `agent_trace_storage` at `/sce/repos//agent-trace.db`, with a one-file repository schema containing repository metadata plus repository-level `diff_traces`, `post_commit_patch_intersections`, `agent_traces`, `messages`, and `parts` tables. The checkout-scoped `AgentTraceDb`/`AgentTraceDbSpec` adapter, its `agent_trace_db_path()` global fallback, and the 15-file migration chain were removed by the `retire-legacy-agent-trace-db` plan. - `cli/src/services/auth_db/mod.rs`: `AuthDb = EncryptedTursoDb`, with `AuthDbSpec` resolving `auth_db_path()` and loading ordered auth migrations where baseline SQL creates `auth_credentials` without `user_id`, with `updated_at`, and a trigger that auto-refreshes `updated_at` on row updates. - `cli/src/services/agent_trace_dwh_db/mod.rs`: `AgentTraceDwhDb = TursoDb`, a separate append-oriented destination-schema adapter for the CLI-independent Agent Trace ETL consumer, distinct from the repository-scoped source schema above. Explicit-path only (no canonical `db_path()`), reuses the `"agent_trace_db"` retry config key, and is not wired into any lifecycle provider, doctor/setup flow, or CLI command. See [agent-trace-dwh-db.md](agent-trace-dwh-db.md). -- `cli/src/services/agent_trace_dwh_replica/mod.rs`: `AgentTraceDwhReplica`, the sole owner of a Turso Sync connection to the repository-scoped `agent-trace-sync.db` replica. Built by opening a connection through `turso::sync::Builder` (the `sync` Cargo feature) and wrapping it as an `AgentTraceDwhDb` via the `from_connection`/`block_on` seam above, rather than through `TursoDb::new`/`new_at`. See [agent-trace-dwh-replica.md](agent-trace-dwh-replica.md). +- `cli/src/services/agent_trace_dwh_replica/mod.rs`: `AgentTraceDwhReplica`, the sole owner of a Turso Sync connection to the DWH-scoped `agent-trace-sync.db` replica (one replica per remote Agent Trace DWH, contributed to by many repositories). Built by opening a connection through `turso::sync::Builder` (the `sync` Cargo feature) and wrapping it as an `AgentTraceDwhDb` via the `from_connection`/`block_on` seam above, rather than through `TursoDb::new`/`new_at`. See [agent-trace-dwh-replica.md](agent-trace-dwh-replica.md). - `cli/src/services/agent_trace_etl/mod.rs`: the incremental `AgentTraceEtl` bridge. It extracts bounded, ordered `agent_traces` batches without reserving the source write lock, then loads facts and the lineage watermark atomically through `AgentTraceDwhReplica`. `cli/src/services/conversation_messages_etl.rs` uses the same `read_transaction()` and shared retry seam for logical `messages` batches with an independent watermark. See [agent-trace-etl.md](agent-trace-etl.md) and [conversation-messages-etl.md](conversation-messages-etl.md). All three database areas (local DB, auth DB, Agent Trace DB) have lifecycle providers. `lifecycle_providers(include_hooks)` registers database providers in order `LocalDbLifecycle` → `AuthDbLifecycle` → `AgentTraceDbLifecycle` before optional hooks. Setup initializes local/auth DBs, establishes Agent Trace checkout identity for diagnostics, initializes the repository-scoped Agent Trace DB with migrations/metadata, and reports credential-safe repository identity metadata. Hook runtime (`open_repository_db_for_hook_runtime` in `cli/src/services/agent_trace_storage/mod.rs`) never creates, migrates, or repairs schema/migration metadata: it opens through `open_without_migrations_at` and calls `ensure_schema_ready_for_hooks()`, a non-mutating readiness check that fails with `sce setup` guidance when the schema is missing or migration-incomplete; only source-instance metadata initialization (an atomic, race-safe `UPDATE` of an existing column) still runs on the hook path. Doctor diagnoses/fixes DB parent/path readiness through lifecycle providers.