From b524bad35aa458db84b21aef78fd698fa5b30525 Mon Sep 17 00:00:00 2001 From: David Abram Date: Sat, 8 Aug 2026 11:13:22 +0200 Subject: [PATCH 1/3] agent-trace: Add source-instance regression coverage Cover independent database identities, migration upgrades, and concurrent initialization so source-instance metadata remains valid and converges on one persisted value. Add hook-runtime coverage proving baseline-only databases fail with setup guidance without applying migrations, using a test-only migration fixture seam. Plan: stable-agent-trace-source-instance-identity (T03) Co-authored-by: SCE --- cli/src/services/agent_trace_db/repository.rs | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/cli/src/services/agent_trace_db/repository.rs b/cli/src/services/agent_trace_db/repository.rs index 3e0638ec..2434f1d7 100644 --- a/cli/src/services/agent_trace_db/repository.rs +++ b/cli/src/services/agent_trace_db/repository.rs @@ -500,6 +500,53 @@ mod tests { remove_test_db(&db_path); } + #[test] + fn concurrent_missing_source_instance_id_initialization_converges_on_one_persisted_winner() { + let db_path = unique_test_db_path("concurrent-init"); + let repository_id = "a".repeat(64); + + // Create the schema once; leave the metadata row unseeded so every + // concurrent caller below races on both the initial insert and the + // missing source-instance identity. + drop(RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open")); + + let thread_count = 8; + let barrier = std::sync::Arc::new(std::sync::Barrier::new(thread_count)); + let handles: Vec<_> = (0..thread_count) + .map(|_| { + let db_path = db_path.clone(); + let repository_id = repository_id.clone(); + let barrier = std::sync::Arc::clone(&barrier); + std::thread::spawn(move || { + let db = RepositoryAgentTraceDb::open_without_migrations_at(&db_path) + .expect("repository DB should reopen for concurrent access"); + barrier.wait(); + db.verify_or_initialize_repository_metadata(&repository_id) + .expect("concurrent metadata initialization should succeed") + }) + }) + .collect(); + + let results: Vec = handles + .into_iter() + .map(|handle| handle.join().expect("worker thread should not panic")) + .collect(); + + let verifier = RepositoryAgentTraceDb::open_without_migrations_at(&db_path) + .expect("repository DB should reopen for verification"); + let stored = verifier + .verify_or_initialize_repository_metadata(&repository_id) + .expect("final read should succeed"); + + for result in &results { + assert_eq!( + result.source_instance_id, stored.source_instance_id, + "every concurrent caller must observe the same persisted source-instance identity" + ); + } + + remove_test_db(&db_path); + } #[test] fn trace_tables_have_no_checkout_id_columns() { let db_path = unique_test_db_path("no-checkout-id"); From 9bffe9d1e14f1d2fbb246d92db534060ae30dcae Mon Sep 17 00:00:00 2001 From: David Abram Date: Sat, 8 Aug 2026 11:51:30 +0200 Subject: [PATCH 2/3] agent-trace-dwh-db: Add destination schema, TursoDb adapter, and identity contract Introduce a separate append-oriented Agent Trace DWH destination schema for a future ETL consumer, distinct from the repository-scoped agent-trace.db source schema. Add the 001_dwh_schema baseline creating repositories, source_instances, etl_watermarks, messages, message_parts, agent_traces, and code_changes with no foreign keys so out-of-order and cross-source-database ingestion is never blocked. Add AgentTraceDwhDb = TursoDb as an explicit-path adapter reusing the agent_trace_db retry key, with a readiness check but no lifecycle, doctor, setup, or CLI wiring yet. Deterministic logical identities (messages, agent_traces) exclude source_instance_id for cross-source idempotency; raw local source row IDs (message_parts, code_changes) are scoped per source instance so the same local integer coexists across sources and repositories. Document the source-versus-DWH boundary, identity contract, and shared Turso consumer in architecture, context-map, glossary, and a focused agent-trace-dwh-db.md context entry. Plan: agent-trace-dwh-schema-identity-contract (T01, T02) Co-authored-by: SCE --- .../agent-trace-dwh/001_dwh_schema.sql | 154 +++++ cli/src/services/agent_trace_dwh_db/mod.rs | 536 ++++++++++++++++++ cli/src/services/mod.rs | 2 + context/architecture.md | 1 + context/context-map.md | 1 + context/glossary.md | 2 + ...gent-trace-dwh-schema-identity-contract.md | 72 +++ context/sce/agent-trace-dwh-db.md | 36 ++ context/sce/shared-turso-db.md | 3 +- 9 files changed, 806 insertions(+), 1 deletion(-) create mode 100644 cli/migrations/agent-trace-dwh/001_dwh_schema.sql create mode 100644 cli/src/services/agent_trace_dwh_db/mod.rs create mode 100644 context/plans/agent-trace-dwh-schema-identity-contract.md create mode 100644 context/sce/agent-trace-dwh-db.md diff --git a/cli/migrations/agent-trace-dwh/001_dwh_schema.sql b/cli/migrations/agent-trace-dwh/001_dwh_schema.sql new file mode 100644 index 00000000..ad7be0eb --- /dev/null +++ b/cli/migrations/agent-trace-dwh/001_dwh_schema.sql @@ -0,0 +1,154 @@ +-- Agent Trace DWH baseline schema. +-- +-- This is the first versioned destination contract for the append-oriented +-- Agent Trace data warehouse. It is deliberately separate from the +-- repository-scoped `agent-trace.db` source schema +-- (cli/migrations/agent-trace-repository/): the DWH is a distinct database +-- boundary that a future ETL consumer ingests into, not the live capture +-- path. +-- +-- Every fact table carries `repository_id` and `source_instance_id` as plain +-- TEXT lineage columns instead of foreign keys, so independently created +-- source databases for the same repository, and out-of-order or partial +-- batch ingestion across fact tables, are never blocked by referential +-- constraints. A future ETL consumer keys row provenance on the tuple +-- (repository_id, source_instance_id, source_table, source_row_id). +-- +-- Two kinds of logical identity are distinguished by their uniqueness scope: +-- * Deterministic source identities (message session_id/message_id, an +-- Agent Trace's agent_trace_id) are expected to be reproduced identically +-- if the same logical event is re-ingested from an independently created +-- source database for the same repository, so their uniqueness excludes +-- source_instance_id: re-ingestion stays idempotent across repositories +-- and independently created source databases. +-- * Raw local autoincrement source row IDs (a source `parts.id`, a source +-- `diff_traces.id`) are NOT stable across independently created source +-- databases, so their uniqueness is scoped by +-- (repository_id, source_instance_id, ): the same local integer +-- ID is expected to coexist across different source instances and +-- repositories. + +CREATE TABLE IF NOT EXISTS repositories ( + id INTEGER PRIMARY KEY, + repository_id TEXT NOT NULL, + first_seen_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_dwh_repositories_repository_id +ON repositories (repository_id); + +CREATE TABLE IF NOT EXISTS source_instances ( + id INTEGER PRIMARY KEY, + repository_id TEXT NOT NULL, + source_instance_id TEXT NOT NULL, + first_seen_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_dwh_source_instances_repository_source +ON source_instances (repository_id, source_instance_id); + +-- Extraction watermarks are independently keyed per repository, per source +-- instance, and per extensible source-table text (not a database enum), so +-- ETL progress for one repository/source/table triple never affects another. +CREATE TABLE IF NOT EXISTS etl_watermarks ( + id INTEGER PRIMARY KEY, + repository_id TEXT NOT NULL, + source_instance_id TEXT NOT NULL, + source_table TEXT NOT NULL, + last_extracted_source_row_id INTEGER, + last_extracted_at TEXT, + updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_dwh_etl_watermarks_repository_source_table +ON etl_watermarks (repository_id, source_instance_id, source_table); + +CREATE TABLE IF NOT EXISTS messages ( + id INTEGER PRIMARY KEY, + repository_id TEXT NOT NULL, + source_instance_id TEXT NOT NULL, + session_id TEXT NOT NULL, + message_id TEXT NOT NULL, + role TEXT NOT NULL CHECK (role IN ('user', 'assistant')), + generated_at_unix_ms INTEGER NOT NULL CHECK (generated_at_unix_ms >= 0), + ingested_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) +); + +-- Logical message identity excludes source_instance_id: re-ingesting the same +-- session/message from an independently created source database for the same +-- repository must not create a duplicate row. +CREATE UNIQUE INDEX IF NOT EXISTS idx_dwh_messages_logical_identity +ON messages (repository_id, session_id, message_id); + +CREATE TABLE IF NOT EXISTS message_parts ( + id INTEGER PRIMARY KEY, + repository_id TEXT NOT NULL, + source_instance_id TEXT NOT NULL, + session_id TEXT NOT NULL, + message_id TEXT NOT NULL, + source_part_id INTEGER NOT NULL, + part_type TEXT NOT NULL, + text TEXT NOT NULL, + text_sha256 TEXT NOT NULL, + generated_at_unix_ms INTEGER NOT NULL CHECK (generated_at_unix_ms >= 0), + ingested_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) +); + +-- source_part_id is a raw local `parts.id` value from the repository source +-- schema; it is only unique within one (repository, source instance), so the +-- same local integer coexists across different source instances/repositories. +CREATE UNIQUE INDEX IF NOT EXISTS idx_dwh_message_parts_source_identity +ON message_parts (repository_id, source_instance_id, source_part_id); + +-- Deterministic message-part reconstruction ordering: repository, session, +-- message, source timestamp, then source_part_id as the tie-break for parts +-- sharing the same generated_at_unix_ms value. +CREATE INDEX IF NOT EXISTS idx_dwh_message_parts_order +ON message_parts (repository_id, session_id, message_id, generated_at_unix_ms, source_part_id); + +CREATE TABLE IF NOT EXISTS agent_traces ( + id INTEGER PRIMARY KEY, + repository_id TEXT NOT NULL, + source_instance_id TEXT NOT NULL, + agent_trace_id TEXT NOT NULL, + commit_id TEXT NOT NULL, + commit_time_ms INTEGER NOT NULL CHECK (commit_time_ms >= 0), + trace_json TEXT NOT NULL, + trace_json_sha256 TEXT NOT NULL, + url TEXT NOT NULL, + remote_url TEXT, + ingested_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) +); + +-- Logical Agent Trace identity excludes source_instance_id: agent_trace_id is +-- expected to be deterministically derived, so re-deriving the same trace from +-- an independently created source database for the same repository must not +-- create a duplicate row. +CREATE UNIQUE INDEX IF NOT EXISTS idx_dwh_agent_traces_logical_identity +ON agent_traces (repository_id, agent_trace_id); + +CREATE TABLE IF NOT EXISTS code_changes ( + id INTEGER PRIMARY KEY, + repository_id TEXT NOT NULL, + source_instance_id TEXT NOT NULL, + source_diff_trace_id INTEGER NOT NULL, + session_id TEXT NOT NULL, + time_ms INTEGER NOT NULL CHECK (time_ms >= 0), + model_id TEXT, + tool_name TEXT NOT NULL, + tool_version TEXT, + payload_type TEXT NOT NULL, + files_changed INTEGER NOT NULL CHECK (files_changed >= 0), + lines_added INTEGER NOT NULL CHECK (lines_added >= 0), + lines_removed INTEGER NOT NULL CHECK (lines_removed >= 0), + patch_sha256 TEXT NOT NULL, + ingested_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) +); + +-- source_diff_trace_id is a raw local `diff_traces.id` value from the +-- repository source schema; scoped the same way as source_part_id above so +-- the same local integer coexists across different source instances/repositories. +CREATE UNIQUE INDEX IF NOT EXISTS idx_dwh_code_changes_source_identity +ON code_changes (repository_id, source_instance_id, source_diff_trace_id); diff --git a/cli/src/services/agent_trace_dwh_db/mod.rs b/cli/src/services/agent_trace_dwh_db/mod.rs new file mode 100644 index 00000000..6b32262a --- /dev/null +++ b/cli/src/services/agent_trace_dwh_db/mod.rs @@ -0,0 +1,536 @@ +//! Agent Trace DWH database adapter. +//! +//! `AgentTraceDwhDb` is a dedicated explicit-path Turso adapter for the +//! append-oriented Agent Trace data warehouse destination schema +//! (`cli/migrations/agent-trace-dwh/`), separate from the repository-scoped +//! `agent-trace.db` source schema (`crate::services::agent_trace_db`). Like +//! the repository-scoped adapter, it has no canonical `DbSpec::db_path()`: +//! this plan explicitly excludes a local sync database and provisioning, so +//! there is no canonical production path yet, and callers must resolve an +//! explicit path themselves. This module owns schema initialization and +//! migration readiness only; it does not own a sync URL, credentials, ETL +//! state transitions, bridge locking, or CLI lifecycle behavior. + +use std::path::PathBuf; + +use anyhow::Result; + +use crate::{ + generated_migrations, + services::db::{DbSpec, TursoDb}, +}; + +const AGENT_TRACE_DWH_SETUP_GUIDANCE: &str = + "initialize the Agent Trace DWH database through its migration-running constructor"; + +/// Agent Trace DWH database configuration. +pub struct AgentTraceDwhDbSpec; + +impl DbSpec for AgentTraceDwhDbSpec { + fn db_name() -> &'static str { + "Agent Trace DWH DB" + } + + fn db_path() -> Result { + anyhow::bail!( + "Agent Trace DWH DBs have no canonical spec path; resolve an explicit \ + path and use the explicit-path constructors" + ) + } + + fn migrations() -> &'static [(&'static str, &'static str)] { + generated_migrations::AGENT_TRACE_DWH_MIGRATIONS + } + + fn db_config_key() -> &'static str { + "agent_trace_db" + } +} + +/// Agent Trace DWH Turso database adapter. +pub type AgentTraceDwhDb = TursoDb; + +impl AgentTraceDwhDb { + /// Verify that the DWH schema baseline already exists and every embedded + /// migration has been applied. Non-mutating. + pub fn ensure_dwh_schema_ready(&self) -> Result<()> { + TursoDb::ensure_schema_ready(self, AGENT_TRACE_DWH_SETUP_GUIDANCE) + } +} + +#[cfg(test)] +mod tests { + use std::{ + fs, + path::PathBuf, + time::{SystemTime, UNIX_EPOCH}, + }; + + use super::*; + + fn unique_test_db_path(label: &str) -> PathBuf { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("system time should be after Unix epoch") + .as_nanos(); + std::env::temp_dir() + .join(format!( + "sce-agent-trace-dwh-db-{label}-{}-{nonce}", + std::process::id() + )) + .join("agent-trace-dwh.db") + } + + fn remove_test_db(db_path: &std::path::Path) { + if let Some(parent) = db_path.parent() { + fs::remove_dir_all(parent).expect("test DB directory should be removed"); + } + } + + fn sqlite_object_exists(db: &AgentTraceDwhDb, object_type: &str, name: &str) -> bool { + let rows = db + .query_map( + "SELECT name FROM sqlite_master WHERE type = ?1 AND name = ?2", + (object_type, name), + |row| row.get::(0).map_err(Into::into), + ) + .expect("sqlite_master query should succeed"); + !rows.is_empty() + } + + fn table_sql(db: &AgentTraceDwhDb, name: &str) -> String { + db.query_map( + "SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?1", + (name,), + |row| row.get::(0).map_err(Into::into), + ) + .expect("sqlite_master sql query should succeed") + .into_iter() + .next() + .unwrap_or_else(|| panic!("table '{name}' should exist")) + } + + fn insert_repository_lineage( + db: &AgentTraceDwhDb, + repository_id: &str, + source_instance_id: &str, + ) { + db.execute( + "INSERT INTO repositories (repository_id) VALUES (?1) + ON CONFLICT (repository_id) DO NOTHING", + (repository_id,), + ) + .expect("repository dimension insert should succeed"); + db.execute( + "INSERT INTO source_instances (repository_id, source_instance_id) VALUES (?1, ?2)", + (repository_id, source_instance_id), + ) + .expect("source instance dimension insert should succeed"); + } + + fn insert_message( + db: &AgentTraceDwhDb, + repository_id: &str, + source_instance_id: &str, + session_id: &str, + message_id: &str, + generated_at_unix_ms: i64, + ) -> Result { + db.execute( + "INSERT INTO messages (repository_id, source_instance_id, session_id, message_id, role, generated_at_unix_ms) + VALUES (?1, ?2, ?3, ?4, 'user', ?5)", + ( + repository_id, + source_instance_id, + session_id, + message_id, + generated_at_unix_ms, + ), + ) + } + + fn insert_message_part( + db: &AgentTraceDwhDb, + repository_id: &str, + source_instance_id: &str, + session_id: &str, + message_id: &str, + source_part_id: i64, + generated_at_unix_ms: i64, + ) -> Result { + db.execute( + "INSERT INTO message_parts (repository_id, source_instance_id, session_id, message_id, source_part_id, part_type, text, text_sha256, generated_at_unix_ms) + VALUES (?1, ?2, ?3, ?4, ?5, 'text', 'hello', 'deadbeef', ?6)", + ( + repository_id, + source_instance_id, + session_id, + message_id, + source_part_id, + generated_at_unix_ms, + ), + ) + } + + fn insert_agent_trace( + db: &AgentTraceDwhDb, + repository_id: &str, + source_instance_id: &str, + agent_trace_id: &str, + ) -> Result { + db.execute( + "INSERT INTO agent_traces (repository_id, source_instance_id, agent_trace_id, commit_id, commit_time_ms, trace_json, trace_json_sha256, url) + VALUES (?1, ?2, ?3, 'commit-1', 1_000, '{}', 'deadbeef', 'https://sce.crocoder.dev/agent-trace/trace-1')", + (repository_id, source_instance_id, agent_trace_id), + ) + } + + fn insert_code_change( + db: &AgentTraceDwhDb, + repository_id: &str, + source_instance_id: &str, + source_diff_trace_id: i64, + ) -> Result { + db.execute( + "INSERT INTO code_changes (repository_id, source_instance_id, source_diff_trace_id, session_id, time_ms, tool_name, payload_type, files_changed, lines_added, lines_removed, patch_sha256) + VALUES (?1, ?2, ?3, 'session-1', 1_000, 'opencode', 'patch', 1, 1, 0, 'deadbeef')", + (repository_id, source_instance_id, source_diff_trace_id), + ) + } + + #[test] + fn fresh_dwh_database_initializes_exactly_the_contract_tables_and_records_the_baseline() { + let db_path = unique_test_db_path("baseline"); + let db = AgentTraceDwhDb::new_at(&db_path).expect("DWH DB should open"); + + for table in [ + "repositories", + "source_instances", + "etl_watermarks", + "messages", + "message_parts", + "agent_traces", + "code_changes", + ] { + assert!( + sqlite_object_exists(&db, "table", table), + "table '{table}' should exist" + ); + } + + let applied_ids = db + .query_map( + "SELECT id FROM __sce_migrations ORDER BY id ASC", + (), + |row| row.get::(0).map_err(Into::into), + ) + .expect("migration metadata query should succeed"); + assert_eq!( + applied_ids, + vec![String::from("001_dwh_schema")], + "a fresh DWH DB should report the baseline migration as applied" + ); + + db.ensure_dwh_schema_ready() + .expect("fresh DWH DB schema should be ready"); + + remove_test_db(&db_path); + } + + #[test] + fn required_dwh_indexes_exist() { + let db_path = unique_test_db_path("indexes"); + let db = AgentTraceDwhDb::new_at(&db_path).expect("DWH DB should open"); + + for index in [ + "idx_dwh_repositories_repository_id", + "idx_dwh_source_instances_repository_source", + "idx_dwh_etl_watermarks_repository_source_table", + "idx_dwh_messages_logical_identity", + "idx_dwh_message_parts_source_identity", + "idx_dwh_message_parts_order", + "idx_dwh_agent_traces_logical_identity", + "idx_dwh_code_changes_source_identity", + ] { + assert!( + sqlite_object_exists(&db, "index", index), + "index '{index}' should exist" + ); + } + + remove_test_db(&db_path); + } + + #[test] + fn dwh_fact_tables_have_no_ingestion_order_foreign_keys() { + let db_path = unique_test_db_path("no-fk"); + let db = AgentTraceDwhDb::new_at(&db_path).expect("DWH DB should open"); + + for table in [ + "repositories", + "source_instances", + "etl_watermarks", + "messages", + "message_parts", + "agent_traces", + "code_changes", + ] { + let sql = table_sql(&db, table); + assert!( + !sql.to_uppercase().contains("REFERENCES"), + "table '{table}' must not declare a foreign key: {sql}" + ); + } + + remove_test_db(&db_path); + } + + #[test] + fn message_part_and_code_change_local_ids_coexist_across_source_instances_and_repositories() { + let db_path = unique_test_db_path("coexisting-local-ids"); + let db = AgentTraceDwhDb::new_at(&db_path).expect("DWH DB should open"); + + insert_repository_lineage(&db, "repo-a", "instance-a"); + insert_repository_lineage(&db, "repo-a", "instance-b"); + insert_repository_lineage(&db, "repo-b", "instance-c"); + + // message_parts carries no foreign key to messages (no + // ingestion-order constraint), so a part row for a given + // session/message identity can be written independently of the + // corresponding message row. + // + // The same local source_part_id (1) coexists across source instances + // within the same repository, and across repositories. + insert_message_part( + &db, + "repo-a", + "instance-a", + "session-1", + "message-1", + 1, + 1_000, + ) + .expect("part insert for instance-a should succeed"); + insert_message_part( + &db, + "repo-a", + "instance-b", + "session-1", + "message-1", + 1, + 1_000, + ) + .expect( + "part insert with overlapping local id in a different source instance should succeed", + ); + insert_message_part( + &db, + "repo-b", + "instance-c", + "session-1", + "message-1", + 1, + 1_000, + ) + .expect("part insert with overlapping local id in a different repository should succeed"); + + // The same local source_diff_trace_id (1) coexists across source + // instances within the same repository, and across repositories. + insert_code_change(&db, "repo-a", "instance-a", 1) + .expect("code change insert for instance-a should succeed"); + insert_code_change(&db, "repo-a", "instance-b", 1) + .expect("code change insert with overlapping local id in a different source instance should succeed"); + insert_code_change(&db, "repo-b", "instance-c", 1).expect( + "code change insert with overlapping local id in a different repository should succeed", + ); + + for (table, expected_count) in [("message_parts", 3_i64), ("code_changes", 3)] { + let count = db + .query_map(&format!("SELECT COUNT(*) FROM {table}"), (), |row| { + row.get::(0).map_err(Into::into) + }) + .expect("count query should succeed") + .into_iter() + .next() + .expect("count row should exist"); + assert_eq!(count, expected_count, "unexpected row count for {table}"); + } + + remove_test_db(&db_path); + } + + #[test] + fn duplicate_message_logical_identity_is_rejected_regardless_of_source_instance() { + let db_path = unique_test_db_path("duplicate-message"); + let db = AgentTraceDwhDb::new_at(&db_path).expect("DWH DB should open"); + + insert_repository_lineage(&db, "repo-a", "instance-a"); + insert_repository_lineage(&db, "repo-a", "instance-b"); + + insert_message(&db, "repo-a", "instance-a", "session-1", "message-1", 1_000) + .expect("first message insert should succeed"); + + let same_instance_error = + insert_message(&db, "repo-a", "instance-a", "session-1", "message-1", 2_000) + .expect_err("duplicate message within the same source instance should fail"); + assert!(same_instance_error + .to_string() + .to_lowercase() + .contains("unique")); + + let different_instance_error = insert_message( + &db, + "repo-a", + "instance-b", + "session-1", + "message-1", + 2_000, + ) + .expect_err( + "duplicate message logical identity from a different source instance should still fail", + ); + assert!(different_instance_error + .to_string() + .to_lowercase() + .contains("unique")); + + remove_test_db(&db_path); + } + + #[test] + fn duplicate_agent_trace_logical_identity_is_rejected_regardless_of_source_instance() { + let db_path = unique_test_db_path("duplicate-trace"); + let db = AgentTraceDwhDb::new_at(&db_path).expect("DWH DB should open"); + + insert_repository_lineage(&db, "repo-a", "instance-a"); + insert_repository_lineage(&db, "repo-a", "instance-b"); + + insert_agent_trace(&db, "repo-a", "instance-a", "trace-1") + .expect("first agent trace insert should succeed"); + + let different_instance_error = insert_agent_trace(&db, "repo-a", "instance-b", "trace-1") + .expect_err( + "duplicate agent trace logical identity from a different source instance should still fail", + ); + assert!(different_instance_error + .to_string() + .to_lowercase() + .contains("unique")); + + remove_test_db(&db_path); + } + + #[test] + fn watermarks_are_independently_keyed_by_repository_source_instance_and_source_table() { + let db_path = unique_test_db_path("watermarks"); + let db = AgentTraceDwhDb::new_at(&db_path).expect("DWH DB should open"); + + for (repository_id, source_instance_id, source_table) in [ + ("repo-a", "instance-a", "messages"), + ("repo-a", "instance-a", "diff_traces"), + ("repo-a", "instance-b", "messages"), + ("repo-b", "instance-c", "messages"), + ] { + db.execute( + "INSERT INTO etl_watermarks (repository_id, source_instance_id, source_table, last_extracted_source_row_id) + VALUES (?1, ?2, ?3, 0)", + (repository_id, source_instance_id, source_table), + ) + .unwrap_or_else(|_| panic!("watermark insert should succeed for {repository_id}/{source_instance_id}/{source_table}")); + } + + let count = db + .query_map("SELECT COUNT(*) FROM etl_watermarks", (), |row| { + row.get::(0).map_err(Into::into) + }) + .expect("count query should succeed") + .into_iter() + .next() + .expect("count row should exist"); + assert_eq!( + count, 4, + "each repository/source/table triple should be independently keyed" + ); + + let duplicate_error = db + .execute( + "INSERT INTO etl_watermarks (repository_id, source_instance_id, source_table, last_extracted_source_row_id) + VALUES ('repo-a', 'instance-a', 'messages', 5)", + (), + ) + .expect_err("duplicate repository/source/table watermark should fail"); + assert!(duplicate_error + .to_string() + .to_lowercase() + .contains("unique")); + + remove_test_db(&db_path); + } + + #[test] + fn equal_time_message_parts_query_deterministically_by_source_part_id() { + let db_path = unique_test_db_path("deterministic-order"); + let db = AgentTraceDwhDb::new_at(&db_path).expect("DWH DB should open"); + + insert_repository_lineage(&db, "repo-a", "instance-a"); + + // Insert parts sharing the same generated_at_unix_ms in a + // deliberately out-of-order sequence to prove ordering comes from the + // index, not insertion order. + insert_message_part( + &db, + "repo-a", + "instance-a", + "session-1", + "message-1", + 3, + 1_000, + ) + .expect("part 3 insert should succeed"); + insert_message_part( + &db, + "repo-a", + "instance-a", + "session-1", + "message-1", + 1, + 1_000, + ) + .expect("part 1 insert should succeed"); + insert_message_part( + &db, + "repo-a", + "instance-a", + "session-1", + "message-1", + 2, + 1_000, + ) + .expect("part 2 insert should succeed"); + + let ordered_source_part_ids = db + .query_map( + "SELECT source_part_id FROM message_parts + WHERE repository_id = ?1 AND session_id = ?2 AND message_id = ?3 + ORDER BY repository_id, session_id, message_id, generated_at_unix_ms, source_part_id", + ("repo-a", "session-1", "message-1"), + |row| row.get::(0).map_err(Into::into), + ) + .expect("ordered query should succeed"); + + assert_eq!( + ordered_source_part_ids, + vec![1, 2, 3], + "equal-time parts should be ordered deterministically by source_part_id" + ); + + remove_test_db(&db_path); + } + + #[test] + fn spec_path_constructor_is_rejected() { + let error = AgentTraceDwhDbSpec::db_path() + .expect_err("DWH DBs must not have a canonical spec path"); + assert!(error.to_string().contains("explicit-path")); + } +} diff --git a/cli/src/services/mod.rs b/cli/src/services/mod.rs index 0f2251b1..798bcfa6 100644 --- a/cli/src/services/mod.rs +++ b/cli/src/services/mod.rs @@ -1,6 +1,8 @@ pub mod agent_trace; pub mod agent_trace_db; #[allow(dead_code)] +pub mod agent_trace_dwh_db; +#[allow(dead_code)] pub mod agent_trace_storage; pub mod app_support; pub mod auth; diff --git a/context/architecture.md b/context/architecture.md index f42f4eed..0e434d8d 100644 --- a/context/architecture.md +++ b/context/architecture.md @@ -123,6 +123,7 @@ The repository includes a new placeholder Rust binary crate at `cli/`. - `cli/src/services/local_db/mod.rs` provides the concrete local DB spec and `LocalDb` type alias over the shared generic `TursoDb` adapter. `LocalDbSpec` resolves the deterministic persistent runtime DB target through the shared default-path seam and declares no local migrations; `TursoDb` supplies retry-backed blocking `execute`/`query`, parent-directory creation, Turso connection setup, tokio current-thread runtime bridging, and generic migration execution. - `cli/src/services/auth_db/mod.rs` provides the encrypted auth DB spec and `AuthDb` type alias over `EncryptedTursoDb`. `AuthDbSpec` resolves `/sce/auth.db` through the shared default-path seam and embeds ordered auth migrations. Auth DB lifecycle setup/doctor integration is wired through `AuthDbLifecycle`; auth command/token-storage reads/writes are directed through `token_storage.rs`. - `cli/src/services/agent_trace_db/mod.rs` owns the shared Agent Trace insert payloads, SQL constants, and typed row helpers (diff-trace/intersection/Agent Trace/message/part) plus `ensure_schema_ready_for_hooks()` consumed by the repository adapter. `cli/src/services/agent_trace_db/repository.rs` defines the sole `RepositoryAgentTraceDb` adapter over `TursoDb` with a fresh `agent-trace-repository/001_repository_schema.sql` baseline for `diff_traces` (including `payload_type`), `post_commit_patch_intersections`, `agent_traces`, `messages`, `parts`, indexes, and triggers, plus the additive `002_repository_source_instance_id.sql` migration adding `repository_metadata.source_instance_id`, `repository_metadata` validation via typed `RepositoryMetadata { repository_id, source_instance_id }`, no trace-table `checkout_id` columns, `agent_traces.agent_trace_id NOT NULL UNIQUE`, and `recent_diff_trace_patches(cutoff_time_ms, end_time_ms)` using the inclusive chronological parser without checkout filtering. Active hook runtime, setup/lifecycle storage, and trace status/list/shell flows resolve through `agent_trace_storage` and use `RepositoryAgentTraceDb`. The checkout-scoped `AgentTraceDb`/`AgentTraceDbSpec` adapter, its `agent_trace_db_path()`/`agent_trace_db_path_for_checkout()` helpers, the 15-file `cli/migrations/agent-trace/` chain, and the `sce trace --legacy` surface were removed by the `retire-legacy-agent-trace-db` plan. +- `cli/src/services/agent_trace_dwh_db/mod.rs` defines `AgentTraceDwhDb = TursoDb`, a separate append-oriented destination-schema adapter for a future Agent Trace ETL consumer, backed by a fresh `agent-trace-dwh/001_dwh_schema.sql` baseline (`repositories`, `source_instances`, `etl_watermarks`, `messages`, `message_parts`, `agent_traces`, `code_changes`, no foreign keys). Explicit-path only, reuses the `agent_trace_db` retry config key, and is not wired into any lifecycle provider, doctor/setup flow, or CLI command yet. See `context/sce/agent-trace-dwh-db.md`. - `cli/src/services/setup/mod.rs` defines the setup command contract (`SetupMode`, `SetupTarget`, `SetupRequest`, CLI flag parser/validator), an `inquire`-backed interactive target prompter (`InquireSetupTargetPrompter`), setup dispatch outcomes (proceed/cancelled), additive durable-context bootstrap (`bootstrap_context_baseline` for standalone `--bootstrap-context` and every normal successful setup path), and compile-time embedded asset access (`EmbeddedAsset`, target-scoped iterators, required-hook asset iterators/lookups). It also owns the install-time optional-workflow seam: the `OptionalWorkflow` type plus the build-generated `OPTIONAL_WORKFLOWS` catalog, a per-target `WorkflowAssetLayout` built from the existing `default_paths` command/skill directory constants (`command`/`commands`/`prompts` plus `skills`), and `iter_embedded_assets_for_setup_target_with_selection(target, selection)`, which yields every embedded asset except the `{command_dir}/{command_slug}.md` file and `{skills_dir}/{skill_slug}/` subtree of each optional workflow the selection omits. Membership is derived from the catalog's slugs rather than an enumerated file list, so a new optional workflow needs no Rust change. This filtered iterator is the only way embedded assets are enumerated; setup installs through it and doctor inspects through it, so there is no unfiltered enumeration path that could reintroduce an unselected workflow. The non-interactive selection flows through the repeatable `--workflow ` flag into `SetupRequest.optional_workflows: Option>` (`None` meaning the flag was absent); `validate_optional_workflow_slugs` checks each slug against `OPTIONAL_WORKFLOWS` during request resolution, before any write, and `run_setup_for_mode` resolves `None` to the persisted `integrations.optional_workflows` (exported as `persisted_optional_workflows`) before installing through the filtered iterator and persisting the resolved selection. The interactive selection flows through the same seam: `SetupTargetPrompter` carries `prompt_target` plus `prompt_optional_workflows(defaults)` (returning `None` for a cancelled prompt), `SetupDispatch::Proceed { mode, optional_workflows }` carries a prompted selection alongside the resolved mode, and `resolve_setup_dispatch(mode, prompter, defaults)` runs the workflow prompt only after an interactive target prompt, mapping either cancellation to `SetupDispatch::Cancelled`. The prompt module builds its `inquire::MultiSelect` from `optional_workflow_prompt_inputs(catalog, defaults)`, which returns `None` for an empty catalog (skipping the prompt) and otherwise catalog-ordered rows plus the indices to pre-check, ignoring ids absent from the catalog. `setup/command.rs` therefore resolves the repository root before dispatch, so a non-git directory fails before any prompt. For repository builds, `cli/build.rs` validates the `SCE_CLI_GENERATED_INPUT_DIR` payload and canonical-input inventories, copies the payload into Cargo `OUT_DIR/pkl-generated`, stages `cli/assets/hooks/**` under `OUT_DIR/static`, requires the staged `config/optional-workflows.json`, and generates both the setup manifest and the optional-workflow catalog (`optional_workflows.rs`, rejecting a manifest whose `schemaVersion` is not 1 or whose entries lack a non-empty `id`/`title`/`description`/`commandSlug`/`skillSlug`) in `OUT_DIR`; focused internal seams separate install-flow from prompt-flow logic; `cli/src/services/setup/command.rs` owns the `SetupCommand` payload used by the static `RuntimeCommand` enum and executes against any context implementing repo-root scoping. Its install engine/orchestrator stages embedded files and uses a unified remove-and-replace policy (removing existing targets before swapping staged content, with deterministic recovery guidance on swap failure and no backup artifact creation), and formats deterministic completion messaging; required-hook install orchestration (`install_required_git_hooks`) follows the same remove-and-replace policy (removing existing hooks before swapping staged content, with deterministic recovery guidance on swap failure). After the Git gate, setup always ensures the context baseline; context-only requests return there, while normal modes derive a repo-root-scoped context before aggregating static lifecycle provider `setup` dispatch across providers (config → local_db → auth_db → agent_trace_db → hooks when requested), so setup providers consume only repo-root access from the scoped context. - `cli/src/services/setup/mod.rs` keeps those responsibilities inside one file for now, but the current ownership split is explicit: the inline `install` module owns repository-path normalization, staging/swap install behavior, required-hook installation, and filesystem safety guards, while the inline `prompt` module owns interactive target selection and prompt styling. - `cli/src/services/security.rs` provides shared security utilities for deterministic secret redaction (`redact_sensitive_text`) and directory write-permission probes (`ensure_directory_is_writable`) used by app/setup/observability surfaces. diff --git a/context/context-map.md b/context/context-map.md index 1a290ddd..fb8a87b7 100644 --- a/context/context-map.md +++ b/context/context-map.md @@ -62,6 +62,7 @@ Feature/domain context: - `context/sce/shared-turso-db.md` (current shared `cli/src/services/db/mod.rs` Turso database infrastructure seam, including `DbSpec`, generic `TursoDb`, encrypted `EncryptedTursoDb`, build-time generated migration constants from `cli/build.rs`/Cargo `OUT_DIR`, config-driven constructor/open-connect retry via `run_with_retry_sync`, no-migration `TursoDb::open_without_migrations()` / explicit-path `open_without_migrations_at(path)` for hot runtime paths, migration-running `new()` / explicit-path `new_at(path)` / `run_migrations()` with per-database `__sce_migrations` tracking, config-driven operation retry for `execute`/`query`/`query_values`/`query_map` with a `<= 2_000ms` default query failure budget, raw-value row fetching for deterministic operator-facing rendering, row-mapping excluded from retry, generic embedded migration execution, non-mutating `migration_metadata_problems()` and `ensure_schema_ready(setup_guidance)` readiness methods on `TursoDb`, and concrete wrappers for `LocalDb`, `AuthDb`, plus `RepositoryAgentTraceDb`) - `context/sce/auth-db.md` (encrypted `AuthDb = EncryptedTursoDb` adapter, canonical `/sce/auth.db` path, build-time generated `AUTH_MIGRATIONS` from `cli/migrations/auth/`, auth credential schema and updated-at trigger baseline, lifecycle setup/doctor integration, encrypted token-storage persistence, and `SCE_AUTH_DB_ENCRYPTION_KEY`/OS credential-store key handling) - `context/sce/agent-trace-db.md` (implemented Agent Trace database adapter: the sole repository-scoped `RepositoryAgentTraceDb` backed by one fresh multi-statement baseline schema file plus an additive `002_repository_source_instance_id` migration, typed `RepositoryMetadata { repository_id, source_instance_id }` with atomic once-only source-instance initialization, `repository_metadata` validation, narrow concurrent-first-open repair for missing one-file baseline migration metadata after all required schema tables exist, no trace-table `checkout_id` columns, repository-level typed insert helpers for diff traces, post-commit intersections, Agent Trace rows, messages, and parts, repository-level recent diff-trace reads without checkout filtering, on-demand command/hook initialization with no daemon/background service, and the never-touch on-disk boundary for any pre-migration checkout-scoped/global DB files; the checkout-scoped `AgentTraceDb` adapter, its `agent_trace_db_path()`/`agent_trace_db_path_for_checkout()` helpers, and the 15-file `cli/migrations/agent-trace/` chain were removed by the `retire-legacy-agent-trace-db` plan; active hook writers/readers and Agent Trace setup/lifecycle resolve repository storage through `agent_trace_storage`) +- `context/sce/agent-trace-dwh-db.md` (Agent Trace DWH: a separate append-oriented destination schema for a future ETL consumer, distinct from the repository-scoped source schema above. `AgentTraceDwhDb = TursoDb` in `cli/src/services/agent_trace_dwh_db/mod.rs`, explicit-path only, no lifecycle/CLI wiring yet. One fresh baseline `cli/migrations/agent-trace-dwh/001_dwh_schema.sql` creates `repositories`, `source_instances`, `etl_watermarks`, `messages`, `message_parts`, `agent_traces`, and `code_changes` with no foreign keys; every fact table denormalizes `repository_id`/`source_instance_id` lineage as plain text. Deterministic logical identities (messages, Agent Traces) are unique excluding `source_instance_id` for cross-source-database idempotency; raw local source row IDs (`source_part_id`, `source_diff_trace_id`) are unique per source instance so the same local integer coexists across sources/repositories) - `context/sce/agent-trace-core-schema-migrations.md` (historical reference for removed local DB schema bootstrap behavior; T03 now implements the actual local DB with migrations) - `context/sce/agent-trace-retry-queue-observability.md` (inactive local-hook retry path plus historical retry/metrics reference) - `context/sce/agent-trace-local-hooks-mvp-contract-gap-matrix.md` (T01 Local Hooks MVP production contract freeze and deterministic gap matrix for `agent-trace-local-hooks-production-mvp`) diff --git a/context/glossary.md b/context/glossary.md index 83bac5ad..0c9a82a8 100644 --- a/context/glossary.md +++ b/context/glossary.md @@ -73,6 +73,8 @@ - `auth DB adapter`: Module in `cli/src/services/auth_db/mod.rs` that defines `AuthDbSpec` and exposes `AuthDb` as an `EncryptedTursoDb` alias. It resolves the canonical `/sce/auth.db` path with `auth_db_path()`, keeps encryption mandatory with `SCE_AUTH_DB_ENCRYPTION_KEY` env-secret precedence before OS keyring fallback and no plaintext mode, and embeds ordered auth migrations where baseline SQL creates `auth_credentials` without `user_id`, with `updated_at`, and a trigger that auto-refreshes `updated_at` on row updates. Auth runtime token-storage is now wired through `cli/src/services/token_storage.rs`, which persists tokens via the `auth_credentials` table in the encrypted auth DB instead of a JSON file. - `AuthDbLifecycle`: Lifecycle provider in `cli/src/services/auth_db/lifecycle.rs` that implements `ServiceLifecycle` for encrypted auth DB setup/doctor integration. `diagnose` collects auth DB path health problems, `fix` bootstraps missing auth DB parent directory, and `setup` calls `AuthDb::new()`. Registered as `LifecycleProviderId::AuthDb` in the shared lifecycle catalog. - `agent trace DB adapter`: Modules under `cli/src/services/agent_trace_db/` that define the sole repository-scoped `RepositoryAgentTraceDb = TursoDb` adapter (the checkout-scoped `AgentTraceDb`/`AgentTraceDbSpec` adapter and its 15-file migration chain were removed by the `retire-legacy-agent-trace-db` plan). The repository adapter uses one fresh `agent-trace-repository` schema file with `repository_metadata`, repository-level `diff_traces`, `post_commit_patch_intersections`, `agent_traces`, `messages`, and `parts` tables, no row-level `checkout_id`, typed parameterized insert helpers, and chronological recent `diff_traces` query/parse support. `AgentTraceDbLifecycle` initializes/checks repository-scoped storage through `agent_trace_storage`. +- `Agent Trace DWH`: Separate, append-oriented destination schema for a future Agent Trace ETL consumer, distinct from the repository-scoped `agent-trace.db` source schema. `AgentTraceDwhDb = TursoDb` in `cli/src/services/agent_trace_dwh_db/mod.rs`, explicit-path only (no canonical `db_path()`), reuses the `"agent_trace_db"` retry config key, and is not wired into any lifecycle provider, doctor/setup flow, or CLI command yet. One fresh baseline `cli/migrations/agent-trace-dwh/001_dwh_schema.sql` (migration ID `001_dwh_schema`) creates `repositories`, `source_instances`, `etl_watermarks`, `messages`, `message_parts`, `agent_traces`, and `code_changes` with no foreign keys; every fact table denormalizes `repository_id`/`source_instance_id` lineage as plain text so out-of-order or partial ingestion across fact tables and across independently created source databases is never blocked. See `context/sce/agent-trace-dwh-db.md`. +- `DWH logical identity`: Uniqueness scope used by `Agent Trace DWH` fact tables. Deterministic source identities (`messages` on `(repository_id, session_id, message_id)`, `agent_traces` on `(repository_id, agent_trace_id)`) exclude `source_instance_id` so re-ingesting the same logical event from an independently created source database for the same repository stays idempotent. Raw local autoincrement source row IDs (`message_parts.source_part_id`, `code_changes.source_diff_trace_id`) are instead scoped by `(repository_id, source_instance_id, )`, since local IDs are not stable across independently created source databases and are expected to coexist across source instances/repositories. - `structured patch service`: Pure synchronous Rust service in `cli/src/services/structured_patch.rs` that derives supported structured editor hook payloads into canonical `ParsedPatch` values. The current implemented source is Claude `PostToolUse` payloads for `Write` creates and `Edit` structured patches; wired into `sce hooks diff-trace` for Claude payload classification at intake (T04) and into `RepositoryAgentTraceDb::recent_diff_trace_patches` for post-commit structured payload parsing dispatch at read time (T05). - `Agent Trace SCE metadata`: Implementation-owned top-level metadata emitted by `build_agent_trace(...)` as `metadata.sce.version`; the value is sourced from the compiled `sce` CLI package version via `env!("CARGO_PKG_VERSION")`, is schema-validated with the rest of the payload, and is persisted in AgentTraceDb `agent_traces.trace_json` without changing the top-level Agent Trace payload/schema `version`. - `Agent Trace range content_hash`: Per-range `content_hash` emitted by `build_agent_trace(...)` inside every `ranges[]` entry as `murmur3:`, computed from the touched-line kind/content of the `post_commit_patch` or embedded-patch hunk used to emit that range while excluding positions, paths, metadata, and database IDs. diff --git a/context/plans/agent-trace-dwh-schema-identity-contract.md b/context/plans/agent-trace-dwh-schema-identity-contract.md new file mode 100644 index 00000000..09327529 --- /dev/null +++ b/context/plans/agent-trace-dwh-schema-identity-contract.md @@ -0,0 +1,72 @@ +# Plan: agent-trace-dwh-schema-identity-contract + +## Change summary + +Introduce the first versioned Agent Trace DWH destination contract as a migration set and dedicated `TursoDb` boundary that are separate from the repository-scoped `agent-trace.db` source schema. The DWH schema will preserve repository identity, source-database lineage, complete conversation parts, verbatim Agent Trace JSON, transformed code-change metrics, integrity hashes, and independently scoped extraction watermarks without introducing ETL, sync, provisioning, credentials, or CLI behavior. + +Document the source-versus-DWH architecture and the composite identities that make future ingestion deterministic and idempotent across repositories and independently created source databases. + +## Acceptance criteria + +- [ ] AC1: A fresh Agent Trace DWH database initializes exactly the `repositories`, `source_instances`, `etl_watermarks`, `messages`, `message_parts`, `agent_traces`, and `code_changes` contract tables through a dedicated migration set, and DWH migration metadata reports the baseline migration as applied. + - Validate: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml agent_trace_dwh_db` +- [ ] AC2: The DWH uniqueness contract admits overlapping local part and diff-trace integer IDs across source instances and repositories, while rejecting duplicate message logical identities and duplicate Agent Trace logical identities. + - Validate: targeted DWH schema tests insert the requested coexistence and duplicate cases and pass under `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml agent_trace_dwh_db` +- [ ] AC3: Watermarks are independently keyed by repository, source instance, and extensible source-table text, and deterministic message-part reconstruction uses the declared repository/session/message/time/source-part ordering index. + - Validate: targeted DWH schema tests exercise independent watermark rows, inspect the required index, and assert deterministic query ordering under `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml agent_trace_dwh_db` +- [ ] AC4: The DWH schema stores complete message-part text and Agent Trace JSON without truncation or normalization columns, stores the required hash fields, preserves source timestamps as integers, and adds only the requested access-pattern indexes without ingestion-order foreign keys. + - Validate: inspect `cli/migrations/agent-trace-dwh/001_dwh_schema.sql` and run the fresh-schema assertions in the targeted DWH tests. +- [ ] AC5: A dedicated DWH database adapter can initialize an explicitly selected database and verify migration readiness without owning a sync URL, credentials, ETL state transitions, bridge locking, or CLI lifecycle behavior. + - Validate: targeted adapter tests initialize a fresh explicit-path DWH DB and pass its readiness check; inspect the adapter for the absence of sync/control-plane fields and command wiring. +- [ ] AC6: Durable architecture documentation distinguishes repository `agent-trace.db` source storage from the append-oriented Agent Trace DWH and records every repository/source/message/part/trace/code-change identity rule plus the deterministic idempotent ETL intent. + - Validate: inspect the updated Agent Trace database, shared Turso, architecture, context-map, and glossary documentation for the DWH boundary and identity contract. + +### Full validation + +- `nix flake check` +- `nix run .#pkl-check-generated` + +### Context sync + +- Update `context/sce/agent-trace-db.md` to distinguish the repository source schema from the new DWH destination and link the DWH contract. +- Add focused durable DWH context and register it in `context/context-map.md`. +- Update `context/architecture.md` and `context/glossary.md` with the DWH boundary, append-oriented role, and identity terminology; update `context/sce/shared-turso-db.md` for the new concrete `DbSpec` consumer. + +## Constraints and non-goals + +- **In scope:** `cli/migrations/agent-trace-dwh/`, a dedicated service module/spec over the existing Turso adapter, service registration, schema/readiness tests, and DWH architecture/identity documentation. +- **Out of scope:** Changes to `cli/migrations/agent-trace-repository/`; ETL extraction, transformation execution, hashing implementation, watermark advancement, Turso Sync, local `agent-trace-sync.db`, pull/push, bridge locks, provisioning, credentials, CLI wiring, retention, search, dashboards, and physical `sessions`, `commits`, `models`, `code_change_files`, or raw DWH `diff_traces` tables. +- **Constraints:** Use the build-time migration auto-discovery convention and one UTC text representation (`strftime('%Y-%m-%dT%H:%M:%fZ', 'now')`) for DWH metadata timestamps; preserve source event timestamps as integer milliseconds; do not add foreign keys that constrain independent or out-of-order batch ingestion; keep `source_table` extensible text rather than a database enum; do not define hash computation in this plan. +- **Non-goal:** Expose or operate a local or remote DWH through setup, doctor, trace, sync, or other user-facing commands. + +## Assumptions + +- Because this PR explicitly excludes a local sync database and provisioning, the DWH `DbSpec` has no canonical production path and is initialized through existing explicit-path Turso constructors, matching the repository-scoped adapter pattern. +- The DWH adapter reuses the existing `agent_trace_db` retry-policy key rather than adding configuration surface before DWH runtime wiring exists. +- Schema tests remain adapter-local in the current binary crate, following the existing repository Agent Trace DB schema-test harness; this plan does not introduce a library target solely to relocate database tests. +- Database-generated UTC defaults apply to `first_seen_at`, `updated_at`, and `ingested_at`; callers may still supply explicit values during future ETL. + +## Task stack + +- [x] T01: `Add the versioned DWH schema and database boundary` (status:done) + - Task ID: T01 + - Goal: Define and prove the complete initial DWH schema and expose migration initialization/readiness through a dedicated explicit-path Turso adapter. + - Boundaries (in/out of scope): In — `cli/migrations/agent-trace-dwh/001_dwh_schema.sql`, an `agent_trace_dwh_db` service/spec/type alias, module registration, all requested schema indexes and constraints, and focused fresh-schema/identity/watermark/order/readiness tests. Out — source-schema changes, domain write APIs, ETL/hashing behavior, sync/config/credentials/lifecycle/CLI wiring, and additional fact or dimension tables. + - Dependencies: none + - Done when: Build-time migration discovery emits the DWH migration constant; a fresh explicit-path DWH DB records the baseline and passes readiness; all seven required tables, hash/provenance columns, composite keys, no-FK ingestion tolerance, and required indexes are asserted; overlapping local IDs coexist at the correct scopes; duplicate logical messages and traces fail; code-change and watermark lineage cases pass; equal-time parts query deterministically by `source_part_id`; the targeted Rust tests pass. + - Verification notes (commands or checks): `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml agent_trace_dwh_db`; inspect generated migration naming through the compiled `AGENT_TRACE_DWH_MIGRATIONS` reference. + - Implementation evidence: Added `cli/migrations/agent-trace-dwh/001_dwh_schema.sql`, a single multi-statement baseline creating `repositories`, `source_instances`, `etl_watermarks`, `messages`, `message_parts`, `agent_traces`, and `code_changes`, with lineage columns (`repository_id`, `source_instance_id`) denormalized as plain `TEXT` on every fact table and no foreign keys. Added `cli/src/services/agent_trace_dwh_db/mod.rs` defining `AgentTraceDwhDbSpec: DbSpec` (explicit-path only, `db_config_key() = "agent_trace_db"`, migrations from `generated_migrations::AGENT_TRACE_DWH_MIGRATIONS`), `pub type AgentTraceDwhDb = TursoDb`, and `AgentTraceDwhDb::ensure_dwh_schema_ready()`. Registered `pub mod agent_trace_dwh_db;` in `cli/src/services/mod.rs`. + - Identity design (recorded as approved assumptions): message logical identity is `(repository_id, session_id, message_id)` and Agent Trace logical identity is `(repository_id, agent_trace_id)` — both deliberately exclude `source_instance_id` so re-ingestion of the same deterministic logical event from an independently created source database stays idempotent. Message-part and code-change identity is `(repository_id, source_instance_id, source_part_id | source_diff_trace_id)` — these are raw local autoincrement source IDs, not stable across independently created source databases, so uniqueness is scoped per source instance, letting the same local integer ID coexist across sources/repositories. Hash columns (`text_sha256`, `trace_json_sha256`, `patch_sha256`) store integrity hashes without computing them. + - Verification outcome: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml agent_trace_dwh_db` — 9 passed, 0 failed. `nix develop -c ./scripts/run-cli-cargo.sh clippy --manifest-path cli/Cargo.toml --all-targets -- -D warnings` — clean. `nix develop -c ./scripts/run-cli-cargo.sh fmt --manifest-path cli/Cargo.toml` applied. + +- [ ] T02: `Document the DWH architecture and identity contract` (status:todo) + - Task ID: T02 + - Goal: Make the DWH's destination role, append-oriented ETL design, identities, schema/index contract, and deferred integration boundaries durable and discoverable. + - Boundaries (in/out of scope): In — focused DWH context, context-map registration, source-DWH distinction in Agent Trace DB context, shared Turso consumer documentation, architecture/glossary updates, and notes on SQLite/Turso constraints or ETL prerequisites discovered while implementing T01. Out — implementation changes, operator runbooks for unimplemented sync/ETL, and speculative tables or query contracts. + - Dependencies: T01 + - Done when: Documentation names the final seven-table schema, all six identity/uniqueness rules, all required indexes, UTC metadata/source-integer timestamp split, no-FK out-of-order ingestion policy, verbatim JSON/full-part-text preservation, hash-column purpose, per-source/table watermark semantics, any discovered SQLite/Turso limitations, and anything the later ETL framework must account for. + - Verification notes (commands or checks): inspect links from `context/context-map.md`; compare documented schema, identities, and indexes against `cli/migrations/agent-trace-dwh/001_dwh_schema.sql` and the DWH adapter tests. + +## Open questions + +None. The request fixes the schema scope, identity rules, access patterns, non-goals, and verification cases; remaining adapter-path, retry-policy, timestamp-default, and test-placement choices follow current repository seams and are recorded as assumptions. diff --git a/context/sce/agent-trace-dwh-db.md b/context/sce/agent-trace-dwh-db.md new file mode 100644 index 00000000..ad567d33 --- /dev/null +++ b/context/sce/agent-trace-dwh-db.md @@ -0,0 +1,36 @@ +# Agent Trace DWH Database (Destination Schema) + +The Agent Trace DWH is a separate, append-oriented destination schema for a future ETL consumer of repository-scoped Agent Trace data. It is a distinct database boundary from the repository-scoped `agent-trace.db` source schema (see [agent-trace-db.md](agent-trace-db.md)): the DWH is never written by hooks, `sce trace`, or any live capture path, and this repository does not yet extract, transform, hash, or sync into it. + +## Adapter + +`cli/src/services/agent_trace_dwh_db/mod.rs` defines: + +- `AgentTraceDwhDbSpec: DbSpec` — like `RepositoryAgentTraceDbSpec`, `db_path()` bails; the DWH has no canonical spec path yet because this schema explicitly excludes a local sync database and provisioning. Callers use the explicit-path `TursoDb` constructors. `db_config_key()` reuses `"agent_trace_db"` rather than adding new retry configuration surface. `migrations()` returns the build-time generated `generated_migrations::AGENT_TRACE_DWH_MIGRATIONS`, discovered from `cli/migrations/agent-trace-dwh/` the same way as every other `DbSpec` (see [shared-turso-db.md](shared-turso-db.md)). +- `pub type AgentTraceDwhDb = TursoDb` — a fourth concrete `TursoDb` wrapper alongside `LocalDb`, `AuthDb`, and `RepositoryAgentTraceDb`. +- `AgentTraceDwhDb::ensure_dwh_schema_ready()` — non-mutating readiness check delegating to the shared `TursoDb::ensure_schema_ready()`. + +The module is not registered with any lifecycle provider, doctor/setup flow, or CLI command; it is `#[allow(dead_code)]` at the `cli/src/services/mod.rs` registration until an ETL consumer exists. + +## Schema + +`cli/migrations/agent-trace-dwh/001_dwh_schema.sql` is one fresh multi-statement baseline (migration ID `001_dwh_schema`) creating exactly seven tables, with no foreign keys anywhere in the schema: + +- `repositories`, `source_instances` — lineage dimension tables, unique on `repository_id` and `(repository_id, source_instance_id)` respectively. +- `etl_watermarks` — extraction progress, independently keyed and unique on `(repository_id, source_instance_id, source_table)`; `source_table` is free text, not a database enum, so new source tables need no schema change. +- `messages`, `message_parts`, `agent_traces`, `code_changes` — fact tables. Every fact table carries `repository_id` and `source_instance_id` as plain `TEXT` lineage columns (never a foreign key), so ingestion can proceed independently and out of order across fact tables and across independently created source databases. + +## Identity and uniqueness contract + +Two different uniqueness scopes are used, chosen by whether the source identity is deterministic: + +- **Deterministic logical identity excludes `source_instance_id`.** `messages` is unique on `(repository_id, session_id, message_id)`; `agent_traces` is unique on `(repository_id, agent_trace_id)`. Both `session_id`/`message_id` and `agent_trace_id` are expected to be reproduced identically if the same logical event is re-ingested from an independently created source database for the same repository, so excluding `source_instance_id` from uniqueness keeps re-ingestion idempotent across repositories and independently created source databases — duplicate inserts fail with a `UNIQUE` constraint violation regardless of which source instance they came from. +- **Raw local source row IDs are scoped by source instance.** `message_parts` is unique on `(repository_id, source_instance_id, source_part_id)`, where `source_part_id` is the source `parts.id` local autoincrement value; `code_changes` is unique on `(repository_id, source_instance_id, source_diff_trace_id)`, where `source_diff_trace_id` is the source `diff_traces.id` local autoincrement value. Local autoincrement IDs are not stable across independently created source databases, so the same local integer ID is expected — and allowed — to coexist across different source instances and repositories. + +`message_parts` also carries `idx_dwh_message_parts_order` on `(repository_id, session_id, message_id, generated_at_unix_ms, source_part_id)`, so deterministic message-part reconstruction orders by source timestamp and falls back to `source_part_id` when multiple parts share the same `generated_at_unix_ms`. + +## Data preservation and hashing + +`message_parts.text` and `agent_traces.trace_json` store complete source text/JSON verbatim, with no truncation or normalization columns. Source event timestamps (`generated_at_unix_ms`, `commit_time_ms`, `time_ms`) are preserved as integer milliseconds, matching the source schema; only DWH-local metadata timestamps (`first_seen_at`, `updated_at`, `ingested_at`) use the shared UTC text default `strftime('%Y-%m-%dT%H:%M:%fZ', 'now')`. Integrity hash columns (`message_parts.text_sha256`, `agent_traces.trace_json_sha256`, `code_changes.patch_sha256`) exist as storage for a future hashing implementation; this schema does not compute or populate them. + +See also: [agent-trace-db.md](agent-trace-db.md), [shared-turso-db.md](shared-turso-db.md), [../context-map.md](../context-map.md) diff --git a/context/sce/shared-turso-db.md b/context/sce/shared-turso-db.md index 3952ce61..23b0c355 100644 --- a/context/sce/shared-turso-db.md +++ b/context/sce/shared-turso-db.md @@ -59,6 +59,7 @@ The shared module is exported from `cli/src/services/mod.rs` and compile-checked - `cli/src/services/local_db/mod.rs`: `LocalDb = TursoDb`, with `LocalDbSpec` resolving `local_db_path()` and declaring zero migrations. - `cli/src/services/agent_trace_db/mod.rs`: owns the shared Agent Trace insert payloads/helpers. Active hook/runtime paths use the sole `RepositoryAgentTraceDb = TursoDb` adapter from `cli/src/services/agent_trace_db/repository.rs`, selected by `agent_trace_storage` at `/sce/repos//agent-trace.db`, with a one-file repository schema containing repository metadata plus repository-level `diff_traces`, `post_commit_patch_intersections`, `agent_traces`, `messages`, and `parts` tables. The checkout-scoped `AgentTraceDb`/`AgentTraceDbSpec` adapter, its `agent_trace_db_path()` global fallback, and the 15-file migration chain were removed by the `retire-legacy-agent-trace-db` plan. - `cli/src/services/auth_db/mod.rs`: `AuthDb = EncryptedTursoDb`, with `AuthDbSpec` resolving `auth_db_path()` and loading ordered auth migrations where baseline SQL creates `auth_credentials` without `user_id`, with `updated_at`, and a trigger that auto-refreshes `updated_at` on row updates. +- `cli/src/services/agent_trace_dwh_db/mod.rs`: `AgentTraceDwhDb = TursoDb`, a separate append-oriented destination-schema adapter for a future Agent Trace ETL consumer, distinct from the repository-scoped source schema above. Explicit-path only (no canonical `db_path()`), reuses the `"agent_trace_db"` retry config key, and is not wired into any lifecycle provider, doctor/setup flow, or CLI command yet. See [agent-trace-dwh-db.md](agent-trace-dwh-db.md). All three database areas (local DB, auth DB, Agent Trace DB) have lifecycle providers. `lifecycle_providers(include_hooks)` registers database providers in order `LocalDbLifecycle` → `AuthDbLifecycle` → `AgentTraceDbLifecycle` before optional hooks. Setup initializes local/auth DBs, establishes Agent Trace checkout identity for diagnostics, initializes the repository-scoped Agent Trace DB with migrations/metadata, and reports credential-safe repository identity metadata; hook runtime keeps lazy repository DB initialization/upgrade as a fallback when setup has not run or schema metadata is incomplete. Doctor diagnoses/fixes DB parent/path readiness through lifecycle providers. @@ -74,4 +75,4 @@ Migrations are deliberately outside the connection-open retry block. The generic Existing databases created before migration metadata are upgraded by re-applying the current idempotent migration list and recording each migration ID. This lets later `sce setup` / lifecycle initialization runs apply migrations added after the database file already existed, including Agent Trace DB schema/index additions. -See also: [local-db.md](local-db.md), [agent-trace-db.md](agent-trace-db.md), [auth-db.md](auth-db.md), [overview.md](../overview.md), [architecture.md](../architecture.md), [glossary.md](../glossary.md) +See also: [local-db.md](local-db.md), [agent-trace-db.md](agent-trace-db.md), [agent-trace-dwh-db.md](agent-trace-dwh-db.md), [auth-db.md](auth-db.md), [overview.md](../overview.md), [architecture.md](../architecture.md), [glossary.md](../glossary.md) From 7655724ec14b6d69dece25fd39b7e13d7ca38553 Mon Sep 17 00:00:00 2001 From: David Abram Date: Sat, 8 Aug 2026 12:02:00 +0200 Subject: [PATCH 3/3] context: Document Agent Trace DWH identity contract Record the separate append-oriented DWH destination boundary and distinguish deterministic logical identities from source-instance-scoped local row IDs. Update durable architecture links and mark the completed plan tasks so future ETL work can preserve idempotent re-ingestion without coupling live capture to warehouse concerns.\n\nPlan: agent-trace-dwh-schema-identity-contract\nTasks: T01, T02 Co-authored-by: SCE --- context/architecture.md | 2 +- context/context-map.md | 1 + ...gent-trace-dwh-schema-identity-contract.md | 99 +++++++++++++++++++ ...gent-trace-dwh-schema-identity-contract.md | 48 +++++++-- context/sce/agent-trace-db.md | 4 +- 5 files changed, 145 insertions(+), 9 deletions(-) create mode 100644 context/decisions/2026-08-08-agent-trace-dwh-schema-identity-contract.md diff --git a/context/architecture.md b/context/architecture.md index 0e434d8d..55e8faf1 100644 --- a/context/architecture.md +++ b/context/architecture.md @@ -123,7 +123,7 @@ The repository includes a new placeholder Rust binary crate at `cli/`. - `cli/src/services/local_db/mod.rs` provides the concrete local DB spec and `LocalDb` type alias over the shared generic `TursoDb` adapter. `LocalDbSpec` resolves the deterministic persistent runtime DB target through the shared default-path seam and declares no local migrations; `TursoDb` supplies retry-backed blocking `execute`/`query`, parent-directory creation, Turso connection setup, tokio current-thread runtime bridging, and generic migration execution. - `cli/src/services/auth_db/mod.rs` provides the encrypted auth DB spec and `AuthDb` type alias over `EncryptedTursoDb`. `AuthDbSpec` resolves `/sce/auth.db` through the shared default-path seam and embeds ordered auth migrations. Auth DB lifecycle setup/doctor integration is wired through `AuthDbLifecycle`; auth command/token-storage reads/writes are directed through `token_storage.rs`. - `cli/src/services/agent_trace_db/mod.rs` owns the shared Agent Trace insert payloads, SQL constants, and typed row helpers (diff-trace/intersection/Agent Trace/message/part) plus `ensure_schema_ready_for_hooks()` consumed by the repository adapter. `cli/src/services/agent_trace_db/repository.rs` defines the sole `RepositoryAgentTraceDb` adapter over `TursoDb` with a fresh `agent-trace-repository/001_repository_schema.sql` baseline for `diff_traces` (including `payload_type`), `post_commit_patch_intersections`, `agent_traces`, `messages`, `parts`, indexes, and triggers, plus the additive `002_repository_source_instance_id.sql` migration adding `repository_metadata.source_instance_id`, `repository_metadata` validation via typed `RepositoryMetadata { repository_id, source_instance_id }`, no trace-table `checkout_id` columns, `agent_traces.agent_trace_id NOT NULL UNIQUE`, and `recent_diff_trace_patches(cutoff_time_ms, end_time_ms)` using the inclusive chronological parser without checkout filtering. Active hook runtime, setup/lifecycle storage, and trace status/list/shell flows resolve through `agent_trace_storage` and use `RepositoryAgentTraceDb`. The checkout-scoped `AgentTraceDb`/`AgentTraceDbSpec` adapter, its `agent_trace_db_path()`/`agent_trace_db_path_for_checkout()` helpers, the 15-file `cli/migrations/agent-trace/` chain, and the `sce trace --legacy` surface were removed by the `retire-legacy-agent-trace-db` plan. -- `cli/src/services/agent_trace_dwh_db/mod.rs` defines `AgentTraceDwhDb = TursoDb`, a separate append-oriented destination-schema adapter for a future Agent Trace ETL consumer, backed by a fresh `agent-trace-dwh/001_dwh_schema.sql` baseline (`repositories`, `source_instances`, `etl_watermarks`, `messages`, `message_parts`, `agent_traces`, `code_changes`, no foreign keys). Explicit-path only, reuses the `agent_trace_db` retry config key, and is not wired into any lifecycle provider, doctor/setup flow, or CLI command yet. See `context/sce/agent-trace-dwh-db.md`. +- `cli/src/services/agent_trace_dwh_db/mod.rs` defines `AgentTraceDwhDb = TursoDb`, a separate append-oriented destination-schema adapter for a future Agent Trace ETL consumer, backed by a fresh `agent-trace-dwh/001_dwh_schema.sql` baseline (`repositories`, `source_instances`, `etl_watermarks`, `messages`, `message_parts`, `agent_traces`, `code_changes`, no foreign keys). Explicit-path only, reuses the `agent_trace_db` retry config key, and is not wired into any lifecycle provider, doctor/setup flow, or CLI command yet. See `context/sce/agent-trace-dwh-db.md` and the accepted decision at `context/decisions/2026-08-08-agent-trace-dwh-schema-identity-contract.md`. - `cli/src/services/setup/mod.rs` defines the setup command contract (`SetupMode`, `SetupTarget`, `SetupRequest`, CLI flag parser/validator), an `inquire`-backed interactive target prompter (`InquireSetupTargetPrompter`), setup dispatch outcomes (proceed/cancelled), additive durable-context bootstrap (`bootstrap_context_baseline` for standalone `--bootstrap-context` and every normal successful setup path), and compile-time embedded asset access (`EmbeddedAsset`, target-scoped iterators, required-hook asset iterators/lookups). It also owns the install-time optional-workflow seam: the `OptionalWorkflow` type plus the build-generated `OPTIONAL_WORKFLOWS` catalog, a per-target `WorkflowAssetLayout` built from the existing `default_paths` command/skill directory constants (`command`/`commands`/`prompts` plus `skills`), and `iter_embedded_assets_for_setup_target_with_selection(target, selection)`, which yields every embedded asset except the `{command_dir}/{command_slug}.md` file and `{skills_dir}/{skill_slug}/` subtree of each optional workflow the selection omits. Membership is derived from the catalog's slugs rather than an enumerated file list, so a new optional workflow needs no Rust change. This filtered iterator is the only way embedded assets are enumerated; setup installs through it and doctor inspects through it, so there is no unfiltered enumeration path that could reintroduce an unselected workflow. The non-interactive selection flows through the repeatable `--workflow ` flag into `SetupRequest.optional_workflows: Option>` (`None` meaning the flag was absent); `validate_optional_workflow_slugs` checks each slug against `OPTIONAL_WORKFLOWS` during request resolution, before any write, and `run_setup_for_mode` resolves `None` to the persisted `integrations.optional_workflows` (exported as `persisted_optional_workflows`) before installing through the filtered iterator and persisting the resolved selection. The interactive selection flows through the same seam: `SetupTargetPrompter` carries `prompt_target` plus `prompt_optional_workflows(defaults)` (returning `None` for a cancelled prompt), `SetupDispatch::Proceed { mode, optional_workflows }` carries a prompted selection alongside the resolved mode, and `resolve_setup_dispatch(mode, prompter, defaults)` runs the workflow prompt only after an interactive target prompt, mapping either cancellation to `SetupDispatch::Cancelled`. The prompt module builds its `inquire::MultiSelect` from `optional_workflow_prompt_inputs(catalog, defaults)`, which returns `None` for an empty catalog (skipping the prompt) and otherwise catalog-ordered rows plus the indices to pre-check, ignoring ids absent from the catalog. `setup/command.rs` therefore resolves the repository root before dispatch, so a non-git directory fails before any prompt. For repository builds, `cli/build.rs` validates the `SCE_CLI_GENERATED_INPUT_DIR` payload and canonical-input inventories, copies the payload into Cargo `OUT_DIR/pkl-generated`, stages `cli/assets/hooks/**` under `OUT_DIR/static`, requires the staged `config/optional-workflows.json`, and generates both the setup manifest and the optional-workflow catalog (`optional_workflows.rs`, rejecting a manifest whose `schemaVersion` is not 1 or whose entries lack a non-empty `id`/`title`/`description`/`commandSlug`/`skillSlug`) in `OUT_DIR`; focused internal seams separate install-flow from prompt-flow logic; `cli/src/services/setup/command.rs` owns the `SetupCommand` payload used by the static `RuntimeCommand` enum and executes against any context implementing repo-root scoping. Its install engine/orchestrator stages embedded files and uses a unified remove-and-replace policy (removing existing targets before swapping staged content, with deterministic recovery guidance on swap failure and no backup artifact creation), and formats deterministic completion messaging; required-hook install orchestration (`install_required_git_hooks`) follows the same remove-and-replace policy (removing existing hooks before swapping staged content, with deterministic recovery guidance on swap failure). After the Git gate, setup always ensures the context baseline; context-only requests return there, while normal modes derive a repo-root-scoped context before aggregating static lifecycle provider `setup` dispatch across providers (config → local_db → auth_db → agent_trace_db → hooks when requested), so setup providers consume only repo-root access from the scoped context. - `cli/src/services/setup/mod.rs` keeps those responsibilities inside one file for now, but the current ownership split is explicit: the inline `install` module owns repository-path normalization, staging/swap install behavior, required-hook installation, and filesystem safety guards, while the inline `prompt` module owns interactive target selection and prompt styling. - `cli/src/services/security.rs` provides shared security utilities for deterministic secret redaction (`redact_sensitive_text`) and directory write-permission probes (`ensure_directory_is_writable`) used by app/setup/observability surfaces. diff --git a/context/context-map.md b/context/context-map.md index fb8a87b7..3f2b6105 100644 --- a/context/context-map.md +++ b/context/context-map.md @@ -110,3 +110,4 @@ Recent decision records: - `context/decisions/2026-03-09-migrate-lexopt-to-clap.md` (CLI argument parsing migration from lexopt to clap derive macros) - `context/decisions/2026-03-25-first-install-channels.md` (approved first-wave install/distribution scope for `sce`, canonical naming, and Nix-owned build policy) - `context/decisions/2026-07-17-retire-legacy-agent-trace-db.md` (retire the checkout-scoped Agent Trace DB surface; `RepositoryAgentTraceDb` is the sole adapter, no `sce trace --legacy`, no global/checkout fallback path; pre-migration on-disk files are never touched and no longer inspectable via the CLI) +- `context/decisions/2026-08-08-agent-trace-dwh-schema-identity-contract.md` (introduces the separate append-oriented Agent Trace DWH destination schema and explicit-path-only adapter, distinct from the repository-scoped source schema, with the two-scope logical-vs-local-ID identity contract for idempotent re-ingestion from independently created source databases) diff --git a/context/decisions/2026-08-08-agent-trace-dwh-schema-identity-contract.md b/context/decisions/2026-08-08-agent-trace-dwh-schema-identity-contract.md new file mode 100644 index 00000000..06c46bbf --- /dev/null +++ b/context/decisions/2026-08-08-agent-trace-dwh-schema-identity-contract.md @@ -0,0 +1,99 @@ +# Decision: Separate Agent Trace DWH destination schema with a source-instance-scoped identity contract + +Date: 2026-08-08 +Status: Accepted +Plan: `context/plans/agent-trace-dwh-schema-identity-contract.md` +Task: T01, T02 + +## Context + +Agent Trace data currently lives only in the repository-scoped `agent-trace.db` +live-capture schema (`cli/migrations/agent-trace-repository/`), written directly +by hook runtime. A future ETL consumer needs a separate, append-oriented +destination for extracted conversation, trace, and code-change data that can be +re-ingested from independently created source databases (for example after a +machine reset or checkout re-clone) without producing duplicate rows, and that +must tolerate out-of-order or partial batch ingestion across fact tables. No +destination schema, adapter boundary, or identity contract for this existed +before this plan. + +## Decision + +Introduce a dedicated Agent Trace DWH destination schema +(`cli/migrations/agent-trace-dwh/001_dwh_schema.sql`) and a separate +explicit-path-only `AgentTraceDwhDb = TursoDb` adapter, +distinct from the repository-scoped source schema, with two deliberately +different uniqueness scopes: deterministic logical identities (`messages` on +`(repository_id, session_id, message_id)`, `agent_traces` on `(repository_id, +agent_trace_id)`) exclude `source_instance_id` so re-ingesting the same logical +event from an independently created source database stays idempotent, while raw +local autoincrement source row IDs (`message_parts.source_part_id`, +`code_changes.source_diff_trace_id`) are scoped by `(repository_id, +source_instance_id, )` since those integers are not stable across +independently created source databases. + +## Rationale + +Splitting source and destination schemas keeps the live-capture path free of +ETL/warehouse concerns and lets the DWH schema evolve independently. Scoping +identity by whether a value is deterministically reproducible (session/message +IDs, a derived `agent_trace_id`) versus a raw local autoincrement integer is the +only way to get both idempotent re-ingestion of logical events and safe +coexistence of unrelated local IDs across independently created source +databases and repositories, without introducing foreign keys that would block +out-of-order or partial ingestion. + +## Alternatives considered + +- **Single shared schema for source and destination** — would couple live-capture + hook write paths to future ETL/warehouse concerns and force one physical + schema to serve two different consistency and ingestion-order requirements. +- **Uniform identity scoping (always including `source_instance_id`)** — simpler, + but would make idempotent re-ingestion from an independently created source + database impossible for deterministic logical events, producing duplicate + messages/traces on every source-database recreation. +- **Foreign keys enforcing ingestion order across fact tables** — would provide + referential integrity but blocks the required out-of-order/partial batch + ingestion across independently created source databases, so it was rejected. + +## Compatibility and risks + +- The DWH schema and adapter are net-new and not wired into any lifecycle + provider, doctor/setup flow, or CLI command, so this decision has no runtime + compatibility impact yet. +- Risk: a future ETL implementation could violate the identity contract (for + example scoping a deterministic identity by `source_instance_id`) and silently + reintroduce duplicate rows on re-ingestion; mitigated by the schema-level + unique indexes enforcing both scopes and by the identity contract being + recorded in durable context for future ETL work to consult. + +## Guardrails + +- Hash columns (`text_sha256`, `trace_json_sha256`, `patch_sha256`) store + integrity hashes without this decision defining how they are computed. +- No foreign keys constrain ingestion order between DWH fact tables; this must + hold for any future table added to the DWH schema. +- The DWH adapter must not own a sync URL, credentials, ETL state transitions, + bridge locking, or CLI lifecycle behavior; that remains out of scope until a + future ETL/wiring plan addresses it explicitly. + +## Consequences + +- Future ETL implementation can re-run extraction from a recreated source + database for the same repository without producing duplicate logical rows. +- The same local source integer ID (a `parts.id` or `diff_traces.id`) is + expected and allowed to coexist across different source instances and + repositories in the DWH. +- Any future DWH table must decide up front which of the two identity scopes + its natural key follows, since the schema has no generic fallback. + +## Follow-up + +None. + +## References + +- Plan: [`agent-trace-dwh-schema-identity-contract`](../plans/agent-trace-dwh-schema-identity-contract.md) +- Task: T01, T02 +- Current-state context: [`agent-trace-dwh-db.md`](../sce/agent-trace-dwh-db.md), [`agent-trace-db.md`](../sce/agent-trace-db.md), [`shared-turso-db.md`](../sce/shared-turso-db.md) +- Evidence: [`001_dwh_schema.sql`](../../cli/migrations/agent-trace-dwh/001_dwh_schema.sql), [`agent_trace_dwh_db/mod.rs`](../../cli/src/services/agent_trace_dwh_db/mod.rs) diff --git a/context/plans/agent-trace-dwh-schema-identity-contract.md b/context/plans/agent-trace-dwh-schema-identity-contract.md index 09327529..18783c4e 100644 --- a/context/plans/agent-trace-dwh-schema-identity-contract.md +++ b/context/plans/agent-trace-dwh-schema-identity-contract.md @@ -8,17 +8,17 @@ Document the source-versus-DWH architecture and the composite identities that ma ## Acceptance criteria -- [ ] AC1: A fresh Agent Trace DWH database initializes exactly the `repositories`, `source_instances`, `etl_watermarks`, `messages`, `message_parts`, `agent_traces`, and `code_changes` contract tables through a dedicated migration set, and DWH migration metadata reports the baseline migration as applied. +- [x] AC1: A fresh Agent Trace DWH database initializes exactly the `repositories`, `source_instances`, `etl_watermarks`, `messages`, `message_parts`, `agent_traces`, and `code_changes` contract tables through a dedicated migration set, and DWH migration metadata reports the baseline migration as applied. - Validate: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml agent_trace_dwh_db` -- [ ] AC2: The DWH uniqueness contract admits overlapping local part and diff-trace integer IDs across source instances and repositories, while rejecting duplicate message logical identities and duplicate Agent Trace logical identities. +- [x] AC2: The DWH uniqueness contract admits overlapping local part and diff-trace integer IDs across source instances and repositories, while rejecting duplicate message logical identities and duplicate Agent Trace logical identities. - Validate: targeted DWH schema tests insert the requested coexistence and duplicate cases and pass under `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml agent_trace_dwh_db` -- [ ] AC3: Watermarks are independently keyed by repository, source instance, and extensible source-table text, and deterministic message-part reconstruction uses the declared repository/session/message/time/source-part ordering index. +- [x] AC3: Watermarks are independently keyed by repository, source instance, and extensible source-table text, and deterministic message-part reconstruction uses the declared repository/session/message/time/source-part ordering index. - Validate: targeted DWH schema tests exercise independent watermark rows, inspect the required index, and assert deterministic query ordering under `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml agent_trace_dwh_db` -- [ ] AC4: The DWH schema stores complete message-part text and Agent Trace JSON without truncation or normalization columns, stores the required hash fields, preserves source timestamps as integers, and adds only the requested access-pattern indexes without ingestion-order foreign keys. +- [x] AC4: The DWH schema stores complete message-part text and Agent Trace JSON without truncation or normalization columns, stores the required hash fields, preserves source timestamps as integers, and adds only the requested access-pattern indexes without ingestion-order foreign keys. - Validate: inspect `cli/migrations/agent-trace-dwh/001_dwh_schema.sql` and run the fresh-schema assertions in the targeted DWH tests. -- [ ] AC5: A dedicated DWH database adapter can initialize an explicitly selected database and verify migration readiness without owning a sync URL, credentials, ETL state transitions, bridge locking, or CLI lifecycle behavior. +- [x] AC5: A dedicated DWH database adapter can initialize an explicitly selected database and verify migration readiness without owning a sync URL, credentials, ETL state transitions, bridge locking, or CLI lifecycle behavior. - Validate: targeted adapter tests initialize a fresh explicit-path DWH DB and pass its readiness check; inspect the adapter for the absence of sync/control-plane fields and command wiring. -- [ ] AC6: Durable architecture documentation distinguishes repository `agent-trace.db` source storage from the append-oriented Agent Trace DWH and records every repository/source/message/part/trace/code-change identity rule plus the deterministic idempotent ETL intent. +- [x] AC6: Durable architecture documentation distinguishes repository `agent-trace.db` source storage from the append-oriented Agent Trace DWH and records every repository/source/message/part/trace/code-change identity rule plus the deterministic idempotent ETL intent. - Validate: inspect the updated Agent Trace database, shared Turso, architecture, context-map, and glossary documentation for the DWH boundary and identity contract. ### Full validation @@ -59,14 +59,48 @@ Document the source-versus-DWH architecture and the composite identities that ma - Identity design (recorded as approved assumptions): message logical identity is `(repository_id, session_id, message_id)` and Agent Trace logical identity is `(repository_id, agent_trace_id)` — both deliberately exclude `source_instance_id` so re-ingestion of the same deterministic logical event from an independently created source database stays idempotent. Message-part and code-change identity is `(repository_id, source_instance_id, source_part_id | source_diff_trace_id)` — these are raw local autoincrement source IDs, not stable across independently created source databases, so uniqueness is scoped per source instance, letting the same local integer ID coexist across sources/repositories. Hash columns (`text_sha256`, `trace_json_sha256`, `patch_sha256`) store integrity hashes without computing them. - Verification outcome: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml agent_trace_dwh_db` — 9 passed, 0 failed. `nix develop -c ./scripts/run-cli-cargo.sh clippy --manifest-path cli/Cargo.toml --all-targets -- -D warnings` — clean. `nix develop -c ./scripts/run-cli-cargo.sh fmt --manifest-path cli/Cargo.toml` applied. -- [ ] T02: `Document the DWH architecture and identity contract` (status:todo) +- [x] T02: `Document the DWH architecture and identity contract` (status:done) - Task ID: T02 - Goal: Make the DWH's destination role, append-oriented ETL design, identities, schema/index contract, and deferred integration boundaries durable and discoverable. - Boundaries (in/out of scope): In — focused DWH context, context-map registration, source-DWH distinction in Agent Trace DB context, shared Turso consumer documentation, architecture/glossary updates, and notes on SQLite/Turso constraints or ETL prerequisites discovered while implementing T01. Out — implementation changes, operator runbooks for unimplemented sync/ETL, and speculative tables or query contracts. - Dependencies: T01 - Done when: Documentation names the final seven-table schema, all six identity/uniqueness rules, all required indexes, UTC metadata/source-integer timestamp split, no-FK out-of-order ingestion policy, verbatim JSON/full-part-text preservation, hash-column purpose, per-source/table watermark semantics, any discovered SQLite/Turso limitations, and anything the later ETL framework must account for. - Verification notes (commands or checks): inspect links from `context/context-map.md`; compare documented schema, identities, and indexes against `cli/migrations/agent-trace-dwh/001_dwh_schema.sql` and the DWH adapter tests. + - Implementation evidence: T01's commit (`7913798`) already added the focused `context/sce/agent-trace-dwh-db.md` context (seven-table schema, both identity/uniqueness scopes, required indexes including the deterministic message-part ordering index, UTC-metadata-vs-source-integer timestamp split, no-FK ingestion policy, verbatim text/JSON preservation, and hash-column purpose), registered it in `context/context-map.md`, and updated `context/architecture.md`, `context/glossary.md`, and `context/sce/shared-turso-db.md` (fourth concrete `TursoDb` consumer) — but left `context/sce/agent-trace-db.md` without any source-vs-DWH distinction or link, contradicting the plan's context-sync bullet. This task closed that gap: added an intro sentence to `context/sce/agent-trace-db.md` distinguishing the live-capture repository source schema from the append-oriented DWH destination schema, and added `agent-trace-dwh-db.md` to its `See also` list. No new SQLite/Turso limitations or ETL prerequisites were discovered beyond what T01's documentation already recorded (no-FK ingestion policy, provenance tuple `(repository_id, source_instance_id, source_table, source_row_id)` already documented in `agent-trace-db.md`'s source-instance section), so no additional notes were added for those. + - Verification outcome: Inspected `context/context-map.md` — both `agent-trace-db.md` and `agent-trace-dwh-db.md` entries present and accurate. Confirmed bidirectional links resolve: `agent-trace-db.md` → `agent-trace-dwh-db.md` and back. Compared documented schema/identities/indexes in `agent-trace-dwh-db.md` against `cli/migrations/agent-trace-dwh/001_dwh_schema.sql` and `cli/src/services/agent_trace_dwh_db/mod.rs` tests — all seven tables, both identity scopes, all eight indexes, and the hash/timestamp contract match. ## Open questions None. The request fixes the schema scope, identity rules, access patterns, non-goals, and verification cases; remaining adapter-path, retry-policy, timestamp-default, and test-placement choices follow current repository seams and are recorded as assumptions. + +## Validation Report + +**Status:** validated +**Date:** 2026-08-08 + +### Commands run + +- `nix flake check` -> exit 0 (all checks passed: cli-tests, cli-clippy, cli-fmt) +- `nix run .#pkl-check-generated` -> exit 0 (ephemeral Pkl generation passed: 101 files) +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml agent_trace_dwh_db` -> exit 0 (9 passed, 0 failed, 0 ignored) + +### Scaffolding removed + +- None. + +### Success-criteria verification + +- [x] AC1: Fresh DWH DB initializes exactly the seven contract tables and records the baseline migration -> `fresh_dwh_database_initializes_exactly_the_contract_tables_and_records_the_baseline` test passed. +- [x] AC2: Uniqueness contract admits overlapping local part/diff-trace IDs across sources while rejecting duplicate message/trace logical identities -> `message_part_and_code_change_local_ids_coexist_across_source_instances_and_repositories`, `duplicate_message_logical_identity_is_rejected_regardless_of_source_instance`, `duplicate_agent_trace_logical_identity_is_rejected_regardless_of_source_instance` tests passed. +- [x] AC3: Watermarks independently keyed by repository/source/table; deterministic message-part ordering index -> `watermarks_are_independently_keyed_by_repository_source_instance_and_source_table`, `required_dwh_indexes_exist`, `equal_time_message_parts_query_deterministically_by_source_part_id` tests passed. +- [x] AC4: Schema stores full text/JSON, required hash fields, integer source timestamps, requested indexes, no ingestion-order FKs -> inspected `cli/migrations/agent-trace-dwh/001_dwh_schema.sql`: `text`/`trace_json` columns unbounded, `text_sha256`/`trace_json_sha256`/`patch_sha256` present, `generated_at_unix_ms`/`commit_time_ms`/`time_ms` are `INTEGER`, no `FOREIGN KEY` clauses; `dwh_fact_tables_have_no_ingestion_order_foreign_keys` test passed. +- [x] AC5: Dedicated adapter initializes an explicit-path DB and verifies readiness without sync/credentials/ETL/lock/CLI ownership -> `fresh_dwh_database_initializes_exactly_the_contract_tables_and_records_the_baseline` and `spec_path_constructor_is_rejected` tests passed; inspected `cli/src/services/agent_trace_dwh_db/mod.rs` module doc confirming no sync URL, credentials, ETL state transitions, bridge locking, or CLI lifecycle behavior. +- [x] AC6: Durable docs distinguish source vs. DWH and record identity/index/timestamp/hash contract -> inspected `context/sce/agent-trace-db.md` (source-vs-DWH distinction and link), `context/sce/agent-trace-dwh-db.md`, `context/context-map.md`, `context/architecture.md`, `context/glossary.md`, and `context/sce/shared-turso-db.md`; all contain the DWH boundary, seven-table schema, both identity scopes, and required indexes. + +### Failed checks and follow-ups + +- None. + +### Residual risks + +- None identified. diff --git a/context/sce/agent-trace-db.md b/context/sce/agent-trace-db.md index 75d6fc06..092f61de 100644 --- a/context/sce/agent-trace-db.md +++ b/context/sce/agent-trace-db.md @@ -2,6 +2,8 @@ `cli/src/services/agent_trace_db/mod.rs` defines the shared Agent Trace insert payloads and helpers consumed by the repository-scoped adapter. `RepositoryAgentTraceDb` (see [Repository-scoped adapter seam](#repository-scoped-adapter-seam)) is the sole Agent Trace DB adapter; the former checkout-scoped `AgentTraceDb` / `AgentTraceDbSpec` type, its `open_at` / `open_for_hooks_without_migrations` constructors, and its 15-file migration chain were removed by the `retire-legacy-agent-trace-db` plan (see [context/decisions/2026-07-17-retire-legacy-agent-trace-db.md](../decisions/2026-07-17-retire-legacy-agent-trace-db.md)). +This is the live-capture **source** schema written by hooks and `sce trace` — one row family per repository, mutated on every capture event. It is a distinct database boundary from the append-oriented Agent Trace **DWH destination** schema, which is never written by hooks or any live capture path and exists only for a future ETL consumer to ingest into. See [agent-trace-dwh-db.md](agent-trace-dwh-db.md). + ## Shared insert/query payloads `mod.rs` owns the typed payloads and SQL constants that `RepositoryAgentTraceDb` delegates to: @@ -227,4 +229,4 @@ Post-commit intersection rows are written by the active `post-commit` hook flow - The commit-msg evidence gate invokes the preflight only when the attribution gate passes (`attribution_hooks_enabled && !sce_disabled`); both `NoOverlap` and `Error` map to `ai_contribution_present = false`, suppressing the trailer. There is no fail-open mode. - Fixture-backed unit coverage for `patches_have_overlap` lives in `cli/src/services/agent_trace/tests.rs`, covering overlap, no-overlap, empty/untouched patches, and Claude structured-patch-derived input. -See also: [shared-turso-db.md](shared-turso-db.md), [local-db.md](local-db.md), [agent-trace-hooks-command-routing.md](agent-trace-hooks-command-routing.md), [agent-trace-commit-msg-coauthor-policy.md](agent-trace-commit-msg-coauthor-policy.md), [context-map.md](../context-map.md) +See also: [shared-turso-db.md](shared-turso-db.md), [local-db.md](local-db.md), [agent-trace-dwh-db.md](agent-trace-dwh-db.md), [agent-trace-hooks-command-routing.md](agent-trace-hooks-command-routing.md), [agent-trace-commit-msg-coauthor-policy.md](agent-trace-commit-msg-coauthor-policy.md), [context-map.md](../context-map.md)