From c1582ba2f5d3605302a0f4c29b03e423fd05971e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 10:07:13 +0100 Subject: [PATCH 01/19] feat: store Recall history in ClickHouse A [recall] or targets..recall connection on the clickhouse adapter now sends history to MergeTree tables in an existing ClickHouse database through the adapter's connection. Recall creates and additively upgrades its tables and views there, prints the DDL when the user may not, and batches RowBinary inserts with deterministic deduplication tokens. Lookups before each insert keep the first copy of a key, record conflicting reuse and refresh artifacts older than a week. Table TTLs follow the configured retention; a store with another retention is reported once a day with the statements that would change it. sqb recall query, schema and output read the ClickHouse store. Also fix the ClickHouse view definitions so aggregate aliases no longer shadow the columns they filter on, and take queued records when a heartbeat falls due before a batch fills. --- .../bindings/_helpers/native_host/adapter.rs | 30 + .../_helpers/recall/clickhouse_calls.rs | 225 ++++++ .../_helpers/recall/clickhouse_store.rs | 663 ++++++++++++++++++ .../src/bindings/_helpers/recall/command.rs | 123 +++- .../src/bindings/_helpers/recall/constants.rs | 2 + .../src/bindings/_helpers/recall/mod.rs | 2 + .../src/bindings/_helpers/recall/rendering.rs | 38 +- .../src/bindings/_helpers/recall/session.rs | 57 +- .../src/clickhouse/_helpers/digits.rs | 19 + .../src/clickhouse/_helpers/literals.rs | 85 +++ .../src/clickhouse/_helpers/mod.rs | 5 + .../src/clickhouse/_helpers/planning.rs | 166 +++++ .../src/clickhouse/_helpers/record_key.rs | 14 + .../src/clickhouse/_helpers/row_binary.rs | 110 +++ .../src/clickhouse/constants.rs | 51 ++ .../clickhouse/main/deduplication_token.rs | 22 + .../src/clickhouse/main/encode_rows.rs | 49 ++ .../src/clickhouse/main/error_kind.rs | 28 + .../src/clickhouse/main/mod.rs | 8 + .../clickhouse/main/parse_column_encoding.rs | 34 + .../src/clickhouse/main/plan_insert.rs | 19 + .../src/clickhouse/main/store_state_sql.rs | 47 ++ .../src/clickhouse/main/stored_rows_sql.rs | 85 +++ .../src/clickhouse/main/ttl_mismatches.rs | 39 ++ crates/sqlbuild-recall/src/clickhouse/mod.rs | 9 + .../sqlbuild-recall/src/clickhouse/models.rs | 66 ++ .../sqlbuild-recall/src/clickhouse/tests.rs | 6 + .../src/clickhouse/tests/helpers.rs | 106 +++ .../src/clickhouse/tests/test_clickhouse.rs | 285 ++++++++ .../src/clickhouse/tests/test_types.rs | 39 ++ crates/sqlbuild-recall/src/lib.rs | 1 + .../src/schema/_helpers/clickhouse_views.rs | 106 +-- .../src/schema/main/clickhouse_relation.rs | 10 + crates/sqlbuild-recall/src/schema/main/mod.rs | 2 + .../schema/main/retention_fix_statements.rs | 29 + .../src/settings/_helpers/layers.rs | 59 ++ .../sqlbuild-recall/src/settings/constants.rs | 5 + crates/sqlbuild-recall/src/settings/models.rs | 17 +- .../src/settings/tests/test_resolve.rs | 88 ++- crates/sqlbuild-recall/src/store/mod.rs | 1 + crates/sqlbuild-recall/src/store/types.rs | 40 ++ .../src/writer/_helpers/delivery.rs | 135 +++- crates/sqlbuild-recall/src/writer/models.rs | 14 +- .../src/writer/tests/helpers.rs | 24 +- .../src/writer/tests/test_session.rs | 26 +- .../src/writer/tests/test_types.rs | 7 + .../sqlbuild/references/docs/cli/recall.md | 5 +- .../docs/concepts/observability/recall.md | 55 +- src/sqlbuild/_native_host.py | 17 + src/sqlbuild/_native_host.pyi | 13 + .../clickhouse/_helpers/recall_store.py | 136 ++++ .../clickhouse/classes/clickhouse_adapter.py | 2 + .../clickhouse/classes/recall_connections.py | 37 + src/sqlbuild/adapters/clickhouse/types.py | 5 + .../commands/main/clickhouse/_test_types.py | 59 ++ .../cli/commands/main/clickhouse/conftest.py | 18 +- .../cli/commands/main/clickhouse/helpers.py | 164 +++++ .../main/clickhouse/test_clickhouse_recall.py | 425 +++++++++++ website/src/content/docs/docs/cli/recall.mdx | 5 +- .../docs/concepts/observability/recall.mdx | 54 +- 60 files changed, 3833 insertions(+), 158 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_calls.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs create mode 100644 crates/sqlbuild-recall/src/clickhouse/_helpers/digits.rs create mode 100644 crates/sqlbuild-recall/src/clickhouse/_helpers/literals.rs create mode 100644 crates/sqlbuild-recall/src/clickhouse/_helpers/mod.rs create mode 100644 crates/sqlbuild-recall/src/clickhouse/_helpers/planning.rs create mode 100644 crates/sqlbuild-recall/src/clickhouse/_helpers/record_key.rs create mode 100644 crates/sqlbuild-recall/src/clickhouse/_helpers/row_binary.rs create mode 100644 crates/sqlbuild-recall/src/clickhouse/constants.rs create mode 100644 crates/sqlbuild-recall/src/clickhouse/main/deduplication_token.rs create mode 100644 crates/sqlbuild-recall/src/clickhouse/main/encode_rows.rs create mode 100644 crates/sqlbuild-recall/src/clickhouse/main/error_kind.rs create mode 100644 crates/sqlbuild-recall/src/clickhouse/main/mod.rs create mode 100644 crates/sqlbuild-recall/src/clickhouse/main/parse_column_encoding.rs create mode 100644 crates/sqlbuild-recall/src/clickhouse/main/plan_insert.rs create mode 100644 crates/sqlbuild-recall/src/clickhouse/main/store_state_sql.rs create mode 100644 crates/sqlbuild-recall/src/clickhouse/main/stored_rows_sql.rs create mode 100644 crates/sqlbuild-recall/src/clickhouse/main/ttl_mismatches.rs create mode 100644 crates/sqlbuild-recall/src/clickhouse/mod.rs create mode 100644 crates/sqlbuild-recall/src/clickhouse/models.rs create mode 100644 crates/sqlbuild-recall/src/clickhouse/tests.rs create mode 100644 crates/sqlbuild-recall/src/clickhouse/tests/helpers.rs create mode 100644 crates/sqlbuild-recall/src/clickhouse/tests/test_clickhouse.rs create mode 100644 crates/sqlbuild-recall/src/clickhouse/tests/test_types.rs create mode 100644 crates/sqlbuild-recall/src/schema/main/clickhouse_relation.rs create mode 100644 crates/sqlbuild-recall/src/schema/main/retention_fix_statements.rs create mode 100644 crates/sqlbuild-recall/src/store/types.rs create mode 100644 src/sqlbuild/adapters/clickhouse/_helpers/recall_store.py create mode 100644 src/sqlbuild/adapters/clickhouse/classes/recall_connections.py create mode 100644 src/sqlbuild/adapters/clickhouse/types.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/adapter.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/adapter.rs index bd79305828..79b2ec1b1a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/adapter.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/adapter.rs @@ -44,6 +44,11 @@ pub(crate) enum AdapterHost { NormalizeRelationType, NormalizeType, SafeCastNullability, + ConnectRecallStore, + RunRecallQuery, + RunRecallCommand, + InsertRecallRows, + CloseRecallStore, } /// Every adapter entry, in `AdapterHost` order. @@ -243,6 +248,31 @@ pub(crate) const ENTRIES: &[HostEntry] = &[ kind: HostKind::Function, edge: HostEdge::Adapter, }, + HostEntry { + name: "connect_recall_store", + kind: HostKind::Function, + edge: HostEdge::Adapter, + }, + HostEntry { + name: "run_recall_query", + kind: HostKind::Function, + edge: HostEdge::Adapter, + }, + HostEntry { + name: "run_recall_command", + kind: HostKind::Function, + edge: HostEdge::Adapter, + }, + HostEntry { + name: "insert_recall_rows", + kind: HostKind::Function, + edge: HostEdge::Adapter, + }, + HostEntry { + name: "close_recall_store", + kind: HostKind::Function, + edge: HostEdge::Adapter, + }, ]; impl AdapterHost { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_calls.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_calls.rs new file mode 100644 index 0000000000..d0f7651cd1 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_calls.rs @@ -0,0 +1,225 @@ +//! Calls into the ClickHouse adapter's Recall store operations, through the native host. + +use pyo3::prelude::{Bound, FromPyObject, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyBytes, PyDict, PyDictMethods, PyInt, PyList, PyString}; +use sqlbuild_config::models::ConfigValue; +use sqlbuild_recall::clickhouse::main::error_kind::error_kind; +use sqlbuild_recall::records::models::RecallValue; +use sqlbuild_recall::store::errors::{StoreError, StoreErrorKind}; +use sqlbuild_recall::store::models::QueryResult; + +use crate::bindings::_helpers::boundary::config_values::config_value_to_python; +use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::native_host::adapter::AdapterHost; +use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::native_threads::constants::OUTPUT_SUPPRESSED_ATTRIBUTE; + +/// What `sqb recall query` reports about a ClickHouse query besides its rows. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct QueryStatistics { + pub(crate) rows_read: Option, + pub(crate) bytes_read: Option, + pub(crate) query_id: Option, +} + +/// One result cell as the adapter returns it: plain values only. +#[derive(FromPyObject)] +enum Cell { + Integer(i64), + Real(f64), + Text(String), + Bytes(Vec), +} + +/// A query reply: column names, rows of cells, and rows read, bytes read and query ID. +type QueryReply = ( + Vec, + Vec>>, + (Option, Option, Option), +); + +/// The adapter's handle for one open store connection; the connection itself stays in Python. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct StoreHandle(u64); + +/// A failure the adapter reported: whether it came from the ClickHouse client, and its message. +type Failure = Option<(bool, String)>; + +/// A store failure from the adapter's report: client errors by server code, anything else as is. +fn reported(action: &str, (from_client, message): (bool, String)) -> StoreError { + let kind: StoreErrorKind = if from_client { + error_kind(&message) + } else { + StoreErrorKind::Other + }; + StoreError { + kind, + message: format!( + "cannot {action}: {}", + message.lines().next().unwrap_or_default() + ), + } +} + +fn raised(action: &str, error: &PyErr) -> StoreError { + StoreError { + kind: StoreErrorKind::Other, + message: format!("cannot {action}: {error}"), + } +} + +/// Call the host entry `entry` with `arguments` and unpack its `(failure, value)` reply. +fn reply<'py>( + py: Python<'py>, + entry: AdapterHost, + arguments: &[(&str, Bound<'py, PyAny>)], + action: &str, +) -> Result, StoreError> { + let called: PyResult<(Failure, Bound<'py, PyAny>)> = host::entry(py, entry) + .and_then(|callable| call_with(&callable, arguments)) + .and_then(|replied| replied.extract()); + match called { + Ok((None, value)) => Ok(value), + Ok((Some(failure), _)) => Err(reported(action, failure)), + Err(error) => Err(raised(action, &error)), + } +} + +/// Connect with the store's connection options; the calling thread's output is never captured. +pub(crate) fn connect( + py: Python<'_>, + config: &[(String, ConfigValue)], + action: &str, +) -> Result { + let options = PyDict::new(py); + for (key, value) in config { + config_value_to_python(py, value.clone()) + .and_then(|value| options.set_item(key, value)) + .map_err(|error| raised(action, &error))?; + } + reply( + py, + AdapterHost::ConnectRecallStore, + &[ + ("config", options.into_any()), + ( + "quiet_thread_attribute", + PyString::new(py, OUTPUT_SUPPRESSED_ATTRIBUTE).into_any(), + ), + ], + action, + )? + .extract() + .map(StoreHandle) + .map_err(|error| raised(action, &error)) +} + +fn handle_value(py: Python<'_>, handle: StoreHandle) -> Bound<'_, PyAny> { + PyInt::new(py, handle.0).into_any() +} + +fn cell_value(cell: Option) -> RecallValue { + match cell { + None => RecallValue::Null, + Some(Cell::Integer(number)) => RecallValue::Integer(number), + Some(Cell::Real(number)) => RecallValue::Real(number), + Some(Cell::Text(text)) => RecallValue::Text(text), + Some(Cell::Bytes(data)) => RecallValue::Blob(data), + } +} + +/// The columns, rows and statistics of `sql` run with `settings`. +pub(crate) fn query( + py: Python<'_>, + handle: StoreHandle, + (sql, settings): (&str, &[(&str, i64)]), + action: &str, +) -> Result<(QueryResult, QueryStatistics), StoreError> { + let query_settings = PyDict::new(py); + for (name, value) in settings { + query_settings + .set_item(*name, *value) + .map_err(|error| raised(action, &error))?; + } + let value = reply( + py, + AdapterHost::RunRecallQuery, + &[ + ("handle", handle_value(py, handle)), + ("sql", PyString::new(py, sql).into_any()), + ("settings", query_settings.into_any()), + ], + action, + )?; + let (columns, rows, (rows_read, bytes_read, query_id)): QueryReply = + value.extract().map_err(|error| raised(action, &error))?; + Ok(( + QueryResult { + columns, + rows: rows + .into_iter() + .map(|row| row.into_iter().map(cell_value).collect()) + .collect(), + truncated: false, + }, + QueryStatistics { + rows_read, + bytes_read, + query_id, + }, + )) +} + +/// Run one DDL statement. +pub(crate) fn command( + py: Python<'_>, + handle: StoreHandle, + sql: &str, + action: &str, +) -> Result<(), StoreError> { + reply( + py, + AdapterHost::RunRecallCommand, + &[ + ("handle", handle_value(py, handle)), + ("sql", PyString::new(py, sql).into_any()), + ], + action, + ) + .map(drop) +} + +/// Insert one `RowBinary` block into `table` under `deduplication_token`. +pub(crate) fn insert( + py: Python<'_>, + handle: StoreHandle, + (table, column_names, payload): (&str, &[String], &[u8]), + deduplication_token: &str, +) -> Result<(), StoreError> { + let action = "write Recall rows"; + let names = PyList::new(py, column_names).map_err(|error| raised(action, &error))?; + reply( + py, + AdapterHost::InsertRecallRows, + &[ + ("handle", handle_value(py, handle)), + ("table", PyString::new(py, table).into_any()), + ("column_names", names.into_any()), + ("payload", PyBytes::new(py, payload).into_any()), + ( + "deduplication_token", + PyString::new(py, deduplication_token).into_any(), + ), + ], + action, + ) + .map(drop) +} + +/// Close the connection; a failure to close is ignored. +pub(crate) fn close(py: Python<'_>, handle: StoreHandle) { + drop( + host::entry(py, AdapterHost::CloseRecallStore) + .and_then(|callable| call_with(&callable, &[("handle", handle_value(py, handle))])), + ); +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs new file mode 100644 index 0000000000..8cdf5bb2ee --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs @@ -0,0 +1,663 @@ +//! The ClickHouse Recall store: written and read through the ClickHouse adapter's connection. + +use std::collections::{HashMap, HashSet}; +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant, SystemTime}; + +use pyo3::prelude::Python; +use sqlbuild_config::models::ConfigValue; +use sqlbuild_recall::clickhouse::constants::{BATCH_ROWS, BATCH_WINDOW_MS}; +use sqlbuild_recall::clickhouse::main::deduplication_token::deduplication_token; +use sqlbuild_recall::clickhouse::main::encode_rows::encode_rows; +use sqlbuild_recall::clickhouse::main::parse_column_encoding::parse_column_encoding; +use sqlbuild_recall::clickhouse::main::plan_insert::plan_insert; +use sqlbuild_recall::clickhouse::main::store_state_sql::store_state_sql; +use sqlbuild_recall::clickhouse::main::stored_rows_sql::stored_rows_sql; +use sqlbuild_recall::clickhouse::main::ttl_mismatches::ttl_mismatches; +use sqlbuild_recall::clickhouse::models::{StoredColumn, StoredRow, TtlMismatch}; +use sqlbuild_recall::records::main::format_timestamp::format_timestamp; +use sqlbuild_recall::records::main::meta_row::meta_row; +use sqlbuild_recall::records::models::{OutputStream, RecallRow, RecallValue, RowEnvelope}; +use sqlbuild_recall::schema::constants::{ + ALL_TABLES, ARTIFACT_REFRESH_DAYS, META_SCHEMA_COMPONENT, META_SCHEMA_VERSION_KEY, + RECALL_SCHEMA_VERSION, VIEW_NAMES, +}; +use sqlbuild_recall::schema::main::add_column_statement::add_column_statement; +use sqlbuild_recall::schema::main::clickhouse_relation::clickhouse_relation; +use sqlbuild_recall::schema::main::retention_fix_statements::retention_fix_statements; +use sqlbuild_recall::schema::main::table_columns::table_columns; +use sqlbuild_recall::schema::main::table_spec::table_spec; +use sqlbuild_recall::schema::main::table_statements::table_statements; +use sqlbuild_recall::schema::main::view_statements::view_statements; +use sqlbuild_recall::schema::models::{DdlOptions, Dialect, Table}; +use sqlbuild_recall::store::constants::LAST_INVOCATION_SELECTOR; +use sqlbuild_recall::store::errors::{StoreError, StoreErrorKind}; +use sqlbuild_recall::store::models::{ + DescribedColumn, DescribedRelation, InsertOutcome, OutputTranscript, QueryResult, +}; +use sqlbuild_recall::store::types::{RecallStore, RecallStoreOpener}; + +use crate::bindings::_helpers::recall::clickhouse_calls::{ + self as calls, QueryStatistics, StoreHandle, +}; +use crate::bindings::_helpers::recall::constants::SECONDS_PER_DAY; + +/// Where a ClickHouse store lives and how long it keeps history. +#[derive(Clone, Debug)] +pub(crate) struct ClickHouseLocation { + pub(crate) config: Vec<(String, ConfigValue)>, + pub(crate) options: DdlOptions, +} + +impl ClickHouseLocation { + pub(crate) fn new(config: &[(String, ConfigValue)], database: &str, retention: u32) -> Self { + Self { + config: config.to_vec(), + options: DdlOptions { + database: Some(database.to_owned()), + retention_days: retention, + cluster: None, + }, + } + } + + fn connect_action(&self) -> String { + format!("connect to the Recall store in `{}`", self.database()) + } + + fn database(&self) -> &str { + self.options.database.as_deref().unwrap_or_default() + } + + fn table(&self, name: &str) -> String { + clickhouse_relation(name, &self.options) + } +} + +/// Opens the store on the delivery thread; `retention_notice` decides whether a retention conflict is reported this time. +#[derive(Debug)] +pub(crate) struct ClickHouseStoreOpener { + pub(crate) location: ClickHouseLocation, + pub(crate) retention_notice: fn(&Path, &str) -> bool, + pub(crate) project_dir: PathBuf, +} + +/// The Recall relations present: tables with their `CREATE` queries, views, and the stored schema version. +struct StoreState { + tables: Vec<(String, String)>, + views: HashSet, + version: i64, +} + +/// The store the delivery thread writes. +struct ClickHouseStore { + location: ClickHouseLocation, + connection: Option, + columns: HashMap>, + notices: Vec, +} + +fn text(value: Option<&RecallValue>) -> String { + match value { + Some(RecallValue::Text(text)) => text.clone(), + Some(RecallValue::Blob(data)) => String::from_utf8_lossy(data).into_owned(), + Some(RecallValue::Integer(number)) => number.to_string(), + _ => String::new(), + } +} + +fn integer(value: Option<&RecallValue>) -> i64 { + match value { + Some(RecallValue::Integer(number)) => *number, + _ => 0, + } +} + +/// `text` as a ClickHouse string literal. +fn literal(text: &str) -> String { + format!("'{}'", text.replace('\\', "\\\\").replace('\'', "\\'")) +} + +impl ClickHouseStore { + fn connection(&self) -> Result { + self.connection.ok_or_else(|| StoreError { + kind: StoreErrorKind::Other, + message: "the Recall store connection is closed".to_owned(), + }) + } + + fn query(&self, py: Python<'_>, sql: &str, action: &str) -> Result { + calls::query(py, self.connection()?, (sql, &[]), action).map(|(result, _)| result) + } + + fn command(&self, py: Python<'_>, sql: &str, action: &str) -> Result<(), StoreError> { + calls::command(py, self.connection()?, sql, action) + } + + /// The Recall relations present, with their `CREATE` queries, and the stored schema version. + fn state(&self, py: Python<'_>) -> Result { + let sql = store_state_sql(self.location.database()); + let relations: QueryResult = self.query(py, &sql.relations, "inspect the Recall store")?; + let mut tables: Vec<(String, String)> = Vec::new(); + let mut views: HashSet = HashSet::new(); + for row in &relations.rows { + let name = text(row.first()); + if integer(row.get(1)) == 1 { + views.insert(name); + } else { + tables.push((name, text(row.get(2)))); + } + } + let version: i64 = if tables + .iter() + .any(|(name, _)| name == table_spec(Table::Meta).name) + { + let found = self.query(py, &sql.schema_version, "read the Recall schema version")?; + integer(found.rows.first().and_then(|row| row.first())) + } else { + 0 + }; + Ok(StoreState { + tables, + views, + version, + }) + } + + fn require_database(&self, py: Python<'_>) -> Result<(), StoreError> { + let found = self.query( + py, + &format!( + "SELECT count() FROM system.databases WHERE name = {}", + literal(self.location.database()) + ), + "inspect the Recall store", + )?; + if integer(found.rows.first().and_then(|row| row.first())) == 0 { + return Err(StoreError { + kind: StoreErrorKind::NotFound, + message: format!( + "ClickHouse database `{}` does not exist; Recall creates its tables only in an \ + existing database", + self.location.database() + ), + }); + } + Ok(()) + } + + fn read_columns(&mut self, py: Python<'_>) -> Result<(), StoreError> { + let sql = store_state_sql(self.location.database()); + let found = self.query(py, &sql.columns, "inspect the Recall tables")?; + let mut columns: HashMap> = HashMap::new(); + for row in &found.rows { + let table_name = text(row.first()); + let Some(table) = ALL_TABLES + .iter() + .find(|table| table_spec(**table).name == table_name) + else { + continue; + }; + let name = text(row.get(1)); + let type_text = text(row.get(2)); + let (encoding, nullable) = parse_column_encoding(&type_text).ok_or_else(|| StoreError { + kind: StoreErrorKind::Other, + message: format!( + "Recall column {table_name}.{name} has type {type_text}, which Recall does not \ + write" + ), + })?; + columns.entry(*table).or_default().push(StoredColumn { + name, + encoding, + nullable, + }); + } + self.columns = columns; + Ok(()) + } + + /// Create missing tables, add missing columns and replace the views when this build's schema is newer; a newer store keeps its views. + fn ensure_schema( + &mut self, + py: Python<'_>, + envelope: &RowEnvelope, + ) -> Result, StoreError> { + let StoreState { + tables, + views, + version, + } = self.state(py)?; + let complete: bool = tables.len() == ALL_TABLES.len() && views.len() == VIEW_NAMES.len(); + if complete && version >= RECALL_SCHEMA_VERSION { + self.read_columns(py)?; + return Ok(tables); + } + self.require_database(py)?; + for statement in table_statements(Dialect::ClickHouse, &self.location.options) { + self.command(py, &statement, "create the Recall tables")?; + } + self.read_columns(py)?; + for table in ALL_TABLES { + let spec = table_spec(table); + let present: &[StoredColumn] = self.columns.get(&table).map_or(&[], Vec::as_slice); + for column in table_columns(table) { + if !present.iter().any(|stored| stored.name == column.name) { + let statement = add_column_statement( + Dialect::ClickHouse, + &self.location.options, + (spec, &column), + ); + self.command(py, &statement, "add a Recall column")?; + } + } + } + self.read_columns(py)?; + let upgrade: bool = version < RECALL_SCHEMA_VERSION; + for (name, statement) in view_statements(Dialect::ClickHouse, &self.location.options) { + if upgrade { + self.command(py, &statement, "create the Recall views")?; + } else if !views.contains(name) { + let statement = + statement.replacen("CREATE OR REPLACE VIEW", "CREATE VIEW IF NOT EXISTS", 1); + self.command(py, &statement, "create the Recall views")?; + } + } + if upgrade { + let now = format_timestamp(SystemTime::now()); + let row = meta_row( + envelope, + ( + META_SCHEMA_COMPONENT, + META_SCHEMA_VERSION_KEY, + &RECALL_SCHEMA_VERSION.to_string(), + ), + &now, + ); + self.write(py, Table::Meta, &[row], SystemTime::now())?; + } + Ok(self.state(py)?.tables) + } + + fn write( + &self, + py: Python<'_>, + table: Table, + rows: &[RecallRow], + now: SystemTime, + ) -> Result<(), StoreError> { + let columns: &[StoredColumn] = self.columns.get(&table).map_or(&[], Vec::as_slice); + let micros: i64 = now + .duration_since(SystemTime::UNIX_EPOCH) + .map_or(0, |elapsed| { + i64::try_from(elapsed.as_micros()).unwrap_or(i64::MAX) + }); + let (names, block) = encode_rows(rows, columns, micros).map_err(|message| StoreError { + kind: StoreErrorKind::Other, + message, + })?; + calls::insert( + py, + self.connection()?, + (&self.location.table(table_spec(table).name), &names, &block), + &deduplication_token(rows), + ) + } + + fn stored_rows( + &self, + py: Python<'_>, + rows: &[RecallRow], + ) -> Result, StoreError> { + let Some(sql) = stored_rows_sql(rows, &self.location.options) else { + return Ok(Vec::new()); + }; + let found = self.query(py, &sql, "read stored Recall rows")?; + Ok(found + .rows + .iter() + .map(|row| { + let hashes: Vec = text(row.get(3)).split(',').map(str::to_owned).collect(); + let newest: String = text(row.get(4)); + StoredRow { + dataset: text(row.first()), + record_key: text(row.get(1)), + first_hash: text(row.get(2)), + hashes, + newest_seen_at: (!newest.is_empty()).then_some(newest), + } + }) + .collect()) + } + + fn retention_conflict(&self, tables: &[(String, String)]) -> Option { + let mismatches: Vec = + ttl_mismatches(tables, self.location.options.retention_days); + let first: &TtlMismatch = mismatches.first()?; + let stored: String = first + .stored_days + .map_or_else(|| "no day TTL".to_owned(), |days| format!("{days} days")); + Some(format!( + "the ClickHouse store in `{}` keeps history for {stored} by its table TTLs, but this \ + project is configured for {} days; retention is a store-wide policy and Recall never \ + changes TTLs itself. An administrator can apply the configured retention with:\n\n{}", + self.location.database(), + self.location.options.retention_days, + retention_fix_statements( + &mismatches + .iter() + .map(|mismatch| mismatch.table.clone()) + .collect::>(), + &self.location.options + ) + )) + } +} + +impl RecallStore for ClickHouseStore { + fn insert(&self, rows: &[RecallRow], _give_up: Instant) -> Result { + Python::attach(|py| { + let now: SystemTime = SystemTime::now(); + let refresh_before: SystemTime = now + .checked_sub(Duration::from_secs( + u64::from(ARTIFACT_REFRESH_DAYS) * SECONDS_PER_DAY, + )) + .unwrap_or(SystemTime::UNIX_EPOCH); + let stored: Vec = self.stored_rows(py, rows)?; + let plan = plan_insert( + rows.to_vec(), + &stored, + (&format_timestamp(now), &format_timestamp(refresh_before)), + ); + for (table, table_rows) in &plan.inserts { + self.write(py, *table, table_rows, now)?; + } + Ok(plan.outcome) + }) + } + + fn take_notices(&mut self) -> Vec { + std::mem::take(&mut self.notices) + } + + fn close(&mut self) { + if let Some(connection) = self.connection.take() { + Python::attach(|py| calls::close(py, connection)); + } + } +} + +impl RecallStoreOpener for ClickHouseStoreOpener { + fn open( + &self, + envelope: &RowEnvelope, + _give_up: Instant, + ) -> Result, StoreError> { + Python::attach(|py| { + let mut store = ClickHouseStore { + location: self.location.clone(), + connection: Some(calls::connect( + py, + &self.location.config, + &self.location.connect_action(), + )?), + columns: HashMap::new(), + notices: Vec::new(), + }; + match store.ensure_schema(py, envelope) { + Ok(tables) => { + if let Some(notice) = store.retention_conflict(&tables) + && (self.retention_notice)( + &self.project_dir, + &format!("{}:retention", self.location.database()), + ) + { + store.notices.push(notice); + } + Ok(Box::new(store) as Box) + } + Err(error) => { + store.close(); + Err(error) + } + } + }) + } + + fn batch_rows(&self) -> usize { + BATCH_ROWS + } + + fn batch_window(&self) -> Duration { + Duration::from_millis(BATCH_WINDOW_MS) + } +} + +/// A ClickHouse store opened for `sqb recall`. +pub(crate) struct ClickHouseReader { + store: ClickHouseStore, +} + +impl ClickHouseReader { + pub(crate) fn open(py: Python<'_>, location: &ClickHouseLocation) -> Result { + let mut store = ClickHouseStore { + location: location.clone(), + connection: Some(calls::connect( + py, + &location.config, + &location.connect_action(), + )?), + columns: HashMap::new(), + notices: Vec::new(), + }; + if store.state(py)?.tables.is_empty() { + store.close(); + return Err(StoreError { + kind: StoreErrorKind::NotFound, + message: format!( + "no Recall history is recorded yet in ClickHouse database `{}`; run a command \ + first", + location.database() + ), + }); + } + Ok(Self { store }) + } + + /// The database the store lives in. + pub(crate) fn database(&self) -> &str { + self.store.location.database() + } + + /// At most `limit` rows of a read-only query, with the server's statistics. + pub(crate) fn query( + &self, + py: Python<'_>, + sql: &str, + limit: Option, + ) -> Result<(QueryResult, QueryStatistics), StoreError> { + let mut settings: Vec<(&str, i64)> = vec![("readonly", 2)]; + if let Some(limit) = limit { + settings.push(("limit", i64::try_from(limit).unwrap_or(i64::MAX - 1) + 1)); + } + let (mut result, statistics) = calls::query( + py, + self.store.connection()?, + (sql, &settings), + "run the Recall query", + ) + .map_err(|failure| StoreError { + kind: if failure.kind == StoreErrorKind::Busy { + StoreErrorKind::Other + } else { + StoreErrorKind::Query + }, + message: failure.message, + })?; + if let Some(limit) = limit + && result.rows.len() > limit + { + result.rows.truncate(limit); + result.truncated = true; + } + Ok((result, statistics)) + } + + /// The store's views, then its tables, with their columns. + pub(crate) fn describe(&self, py: Python<'_>) -> Result, StoreError> { + let names: Vec = VIEW_NAMES + .iter() + .map(|name| literal(name)) + .chain( + ALL_TABLES + .iter() + .map(|table| literal(table_spec(*table).name)), + ) + .collect(); + let found = self.store.query( + py, + &format!( + "SELECT c.table, if(t.engine = 'View', 'view', 'table') AS kind, c.name, c.type \ + FROM system.columns AS c INNER JOIN system.tables AS t \ + ON t.database = c.database AND t.name = c.table \ + WHERE c.database = {} AND c.table IN ({}) \ + ORDER BY kind = 'table', c.table, c.position", + literal(self.database()), + names.join(", ") + ), + "describe the Recall store", + )?; + let mut relations: Vec = Vec::new(); + for row in &found.rows { + let name = text(row.first()); + if relations.last().is_none_or(|last| last.name != name) { + relations.push(DescribedRelation { + name, + kind: text(row.get(1)), + columns: Vec::new(), + }); + } + let declared_type = text(row.get(3)); + if let Some(relation) = relations.last_mut() { + relation.columns.push(DescribedColumn { + name: text(row.get(2)), + nullable: declared_type.starts_with("Nullable("), + declared_type, + }); + } + } + Ok(relations) + } + + /// The full invocation ID an ID, unique prefix or `last` names. + pub(crate) fn resolve_invocation( + &self, + py: Python<'_>, + selector: &str, + ) -> Result { + let table = self + .store + .location + .table(table_spec(Table::Invocations).name); + let sql = if selector == LAST_INVOCATION_SELECTOR { + format!( + "SELECT invocation_id FROM {table} \ + ORDER BY invocation_started_at DESC, invocation_id DESC LIMIT 1" + ) + } else { + format!( + "SELECT DISTINCT invocation_id FROM {table} \ + WHERE startsWith(invocation_id, {}) ORDER BY invocation_id LIMIT 3", + literal(selector) + ) + }; + let found = self.store.query(py, &sql, "find the invocation")?; + let candidates: Vec = found.rows.iter().map(|row| text(row.first())).collect(); + if let Some(exact) = candidates.iter().find(|candidate| *candidate == selector) { + return Ok(exact.clone()); + } + match candidates.as_slice() { + [] => Err(StoreError { + kind: StoreErrorKind::NotFound, + message: if selector == LAST_INVOCATION_SELECTOR { + "no invocations are recorded yet".to_owned() + } else { + format!("no recorded invocation matches '{selector}'") + }, + }), + [only] => Ok(only.clone()), + several => Err(StoreError { + kind: StoreErrorKind::Ambiguous, + message: format!( + "'{selector}' matches several invocations: {}{}", + several[..2].join(", "), + if several.len() > 2 { ", …" } else { "" } + ), + }), + } + } + + /// One invocation's exact output bytes, deduplicated by sequence, and its completeness. + pub(crate) fn read_output_transcript( + &self, + py: Python<'_>, + invocation_id: &str, + stream: Option, + ) -> Result { + let id = literal(invocation_id); + let stream_filter: String = stream.map_or_else(String::new, |stream| { + format!(" AND stream = {}", literal(stream.as_str())) + }); + let output = self.store.location.table(table_spec(Table::Output).name); + let chunks = self.store.query( + py, + &format!( + "SELECT data FROM (SELECT sequence, stream, data FROM {output} \ + WHERE invocation_id = {id} ORDER BY sequence, ingested_at LIMIT 1 BY sequence) \ + WHERE 1{stream_filter} ORDER BY sequence" + ), + "read Recall output", + )?; + let mut transcript = OutputTranscript { + invocation_id: invocation_id.to_owned(), + ..OutputTranscript::default() + }; + for row in &chunks.rows { + match row.first() { + Some(RecallValue::Blob(data)) => transcript.data.extend_from_slice(data), + Some(RecallValue::Text(text)) => transcript.data.extend_from_slice(text.as_bytes()), + _ => {} + } + } + let database = self.store.location.table("invocations"); + let status = self.store.query( + py, + &format!("SELECT status FROM {database} WHERE invocation_id = {id}"), + "read the Recall invocation", + )?; + transcript.status = status.rows.first().map(|row| text(row.first())); + let completeness_view = self.store.location.table("output_invocations"); + let completeness = self.store.query( + py, + &format!( + "SELECT missing_sequences, reported_drops, filtered, flush_complete, complete, \ + gap_reason FROM {completeness_view} WHERE invocation_id = {id}" + ), + "read Recall output completeness", + )?; + if let Some(row) = completeness.rows.first() { + transcript.missing_sequences = integer(row.first()); + transcript.reported_drops = integer(row.get(1)); + transcript.filtered = integer(row.get(2)); + transcript.flush_complete = integer(row.get(3)) != 0; + transcript.complete = integer(row.get(4)) != 0; + let reason = text(row.get(5)); + transcript.gap_reason = (!reason.is_empty()).then_some(reason); + } + Ok(transcript) + } + + pub(crate) fn close(mut self) { + self.store.close(); + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs index c7f01ac6fd..e21ed8aaac 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs @@ -6,6 +6,7 @@ use std::time::Instant; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyStringMethods}; use sqlbuild_recall::records::models::OutputStream; +use sqlbuild_recall::schema::models::Dialect; use sqlbuild_recall::settings::main::resolve_recall_settings::resolve_recall_settings; use sqlbuild_recall::settings::models::{RecallSettings, SettingsRequest, StoreLocation}; use sqlbuild_recall::store::errors::{StoreError, StoreErrorKind}; @@ -18,6 +19,7 @@ use sqlbuild_recall::store::models::{OpenMode, SqliteStore}; use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; use crate::bindings::_helpers::query_command::query::run_query_command; +use crate::bindings::_helpers::recall::clickhouse_store::{ClickHouseLocation, ClickHouseReader}; use crate::bindings::_helpers::recall::console::{write_stderr, write_stdout_bytes}; use crate::bindings::_helpers::recall::constants::{ DEFAULT_MAX_CELL_BYTES, DEFAULT_QUERY_LIMIT, JSON_FORMAT, LAST_INVOCATION, LONG_FORMAT, @@ -115,7 +117,49 @@ fn settings(py: Python<'_>, recall: &RecallRequest) -> PyResult<(PathBuf, Recall Ok((project_dir, settings)) } -fn open_store(py: Python<'_>, settings: &RecallSettings) -> PyResult<(SqliteStore, String)> { +/// An open store for reading: the native SQLite file or a ClickHouse database. +enum ReadStore { + Sqlite { store: SqliteStore, path: String }, + ClickHouse(ClickHouseReader), +} + +impl ReadStore { + fn dialect(&self) -> Dialect { + match self { + Self::Sqlite { .. } => Dialect::Sqlite, + Self::ClickHouse(_) => Dialect::ClickHouse, + } + } + + fn adapter(&self) -> &'static str { + match self.dialect() { + Dialect::Sqlite => "sqlite", + Dialect::ClickHouse => "clickhouse", + } + } + + fn label(&self) -> String { + match self { + Self::Sqlite { path, .. } => path.clone(), + Self::ClickHouse(reader) => reader.database().to_owned(), + } + } + + fn resolve_invocation(&self, py: Python<'_>, selector: &str) -> Result { + match self { + Self::Sqlite { store, .. } => resolve_invocation(store, selector), + Self::ClickHouse(reader) => reader.resolve_invocation(py, selector), + } + } + + fn close(self) { + if let Self::ClickHouse(reader) = self { + reader.close(); + } + } +} + +fn open_store(py: Python<'_>, settings: &RecallSettings) -> PyResult { match &settings.store { StoreLocation::Sqlite { path } => { let store = open_sqlite_store(path, OpenMode::ReadOnly).map_err(|error| { @@ -132,8 +176,29 @@ fn open_store(py: Python<'_>, settings: &RecallSettings) -> PyResult<(SqliteStor store_failure(py, &error) } })?; - Ok((store, path.display().to_string())) + Ok(ReadStore::Sqlite { + store, + path: path.display().to_string(), + }) } + StoreLocation::ClickHouse { + config, database, .. + } => ClickHouseReader::open( + py, + &ClickHouseLocation::new(config, database, settings.retention_days), + ) + .map(ReadStore::ClickHouse) + .map_err(|error| { + let error = if error.kind == StoreErrorKind::NotFound { + error + } else { + StoreError { + kind: StoreErrorKind::Other, + ..error + } + }; + user_error(py, &error.message, RECALL_STORE_CODE) + }), StoreLocation::Unsupported { connection, adapter, @@ -192,10 +257,20 @@ fn run_query(py: Python<'_>, recall: &RecallRequest) -> PyResult { if recall.backend.as_deref() == Some(WAREHOUSE_BACKEND) { return warehouse_query(py, recall, &project_dir, (&sql, limit)); } - let (store, path) = open_store(py, &settings)?; + let store = open_store(py, &settings)?; let started = Instant::now(); let limit = limit.map(|limit| usize::try_from(limit).unwrap_or(usize::MAX)); - let result = query_store(&store, &sql, limit).map_err(|error| store_failure(py, &error))?; + let queried = match &store { + ReadStore::Sqlite { store, .. } => { + query_store(store, &sql, limit).map(|result| (result, None)) + } + ReadStore::ClickHouse(reader) => reader + .query(py, &sql, limit) + .map(|(result, statistics)| (result, Some(statistics))), + }; + let (adapter, label) = (store.adapter(), store.label()); + store.close(); + let (result, statistics) = queried.map_err(|error| store_failure(py, &error))?; let format: &str = &recall.format; let max_bytes: Option = match recall.max_cell_bytes { Some(bytes) if bytes <= 0 => None, @@ -209,9 +284,11 @@ fn run_query(py: Python<'_>, recall: &RecallRequest) -> PyResult { max_bytes, &QueryContext { target: settings.target.as_deref(), - store_path: &path, + store_path: &label, + adapter, limit, elapsed_ms: started.elapsed().as_secs_f64() * 1000.0, + statistics: statistics.as_ref(), }, ); write_stdout_bytes(py, rendered.as_bytes())?; @@ -235,12 +312,23 @@ fn run_schema(py: Python<'_>, recall: &RecallRequest) -> PyResult { }; return warehouse_query(py, recall, &project_dir, (&sql, None)); } - let (store, path) = open_store(py, &settings)?; - let relations = describe_store(&store).map_err(|error| store_failure(py, &error))?; + let store = open_store(py, &settings)?; + let described = match &store { + ReadStore::Sqlite { store, .. } => describe_store(store), + ReadStore::ClickHouse(reader) => reader.describe(py), + }; + let (dialect, label) = (store.dialect(), store.label()); + store.close(); + let relations = described.map_err(|error| store_failure(py, &error))?; let json_output: bool = recall.format == JSON_FORMAT; write_stdout_bytes( py, - render_schema(&relations, json_output, (&path, settings.retention_days)).as_bytes(), + render_schema( + &relations, + json_output, + (&label, dialect, settings.retention_days), + ) + .as_bytes(), )?; Ok(0) } @@ -263,13 +351,22 @@ fn tail_lines(data: &[u8], lines: usize) -> &[u8] { fn run_output(py: Python<'_>, recall: &RecallRequest) -> PyResult { let (_, settings) = settings(py, recall)?; - let (store, _) = open_store(py, &settings)?; + let store = open_store(py, &settings)?; let selector: &str = recall.invocation.as_deref().unwrap_or(LAST_INVOCATION); - let invocation_id = - resolve_invocation(&store, selector).map_err(|error| store_failure(py, &error))?; let stream = recall.stream.as_deref().and_then(OutputStream::from_name); - let transcript = read_output_transcript(&store, &invocation_id, stream) - .map_err(|error| store_failure(py, &error))?; + let read = store + .resolve_invocation(py, selector) + .and_then(|invocation_id| match &store { + ReadStore::Sqlite { store, .. } => { + read_output_transcript(store, &invocation_id, stream) + } + ReadStore::ClickHouse(reader) => { + reader.read_output_transcript(py, &invocation_id, stream) + } + }); + store.close(); + let transcript = read.map_err(|error| store_failure(py, &error))?; + let invocation_id: &str = &transcript.invocation_id; let data: &[u8] = match recall.tail { Some(lines) => tail_lines(&transcript.data, usize::try_from(lines).unwrap_or(0)), None => &transcript.data, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs index ff0f8711f2..e1093942ed 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs @@ -89,3 +89,5 @@ pub(crate) const UTC_ZONE_NAME: &str = "UTC"; pub(crate) const CLOSE_POLL_MS: u64 = 50; /// Longest wait at startup for another process moving a corrupt store aside; the writer thread retries within the shutdown budget. pub(crate) const QUARANTINE_START_WAIT_MS: u64 = 200; +/// Seconds in a day, for the artifact refresh cutoff. +pub(crate) const SECONDS_PER_DAY: u64 = 24 * 60 * 60; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/mod.rs index e05fcbcf73..1baa25d9fd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/mod.rs @@ -1,3 +1,5 @@ +pub(crate) mod clickhouse_calls; +pub(crate) mod clickhouse_store; pub(crate) mod command; pub(crate) mod console; pub(crate) mod constants; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/rendering.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/rendering.rs index 40b51110f2..a0a0fe946f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/rendering.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/rendering.rs @@ -6,8 +6,10 @@ use sqlbuild_core::query_command::main::render_query_long::render_query_long; use sqlbuild_core::query_command::main::render_query_table::render_query_table; use sqlbuild_core::query_command::models::QueryDisplay; use sqlbuild_recall::records::models::RecallValue; +use sqlbuild_recall::schema::models::Dialect; use sqlbuild_recall::store::models::{DescribedColumn, DescribedRelation, QueryResult}; +use crate::bindings::_helpers::recall::clickhouse_calls::QueryStatistics; use crate::bindings::_helpers::recall::constants::{ BASE64_ALPHABET, CSV_FORMAT, JOIN_KEYS, JSON_FORMAT, TABLE_FORMAT, }; @@ -91,9 +93,13 @@ fn json_cell(value: &RecallValue) -> Value { /// What the JSON document reports besides rows. pub(crate) struct QueryContext<'a> { pub(crate) target: Option<&'a str>, + /// The SQLite file or the ClickHouse database. pub(crate) store_path: &'a str, + pub(crate) adapter: &'a str, pub(crate) limit: Option, pub(crate) elapsed_ms: f64, + /// Server statistics; ClickHouse only. + pub(crate) statistics: Option<&'a QueryStatistics>, } /// The rendered result in `format`: `long`, `table`, `csv` or `json`. @@ -112,7 +118,7 @@ pub(crate) fn render_result( let document = json!({ "backend": "logs", "target": context.target, - "store": {"adapter": "sqlite", "database": context.store_path}, + "store": {"adapter": context.adapter, "database": context.store_path}, "columns": result.columns, "rows": rows, "row_count": result.rows.len(), @@ -120,9 +126,9 @@ pub(crate) fn render_result( "limit": context.limit, "statistics": { "elapsed_ms": context.elapsed_ms, - "rows_read": Value::Null, - "bytes_read": Value::Null, - "query_id": Value::Null, + "rows_read": context.statistics.and_then(|statistics| statistics.rows_read), + "bytes_read": context.statistics.and_then(|statistics| statistics.bytes_read), + "query_id": context.statistics.and_then(|statistics| statistics.query_id.clone()), }, }); return format!("{document}\n"); @@ -176,8 +182,20 @@ fn relation_note(name: &str) -> &'static str { pub(crate) fn render_schema( relations: &[DescribedRelation], json_output: bool, - (store_path, retention_days): (&str, u32), + (store_path, store_dialect, retention_days): (&str, Dialect, u32), ) -> String { + let (adapter, dialect, time_semantics): (&str, &str, &str) = match store_dialect { + Dialect::ClickHouse => ( + "clickhouse", + "ClickHouse", + "timestamps are DateTime64(6, 'UTC')", + ), + Dialect::Sqlite => ( + "sqlite", + "SQLite", + "timestamps are ISO-8601 UTC text with microseconds", + ), + }; let filter_hint = "filter invocation_started_at to bound scans; all rows of an invocation expire together"; if json_output { @@ -211,18 +229,18 @@ pub(crate) fn render_schema( .collect(); let document = json!({ "backend": "logs", - "dialect": "sqlite", - "store": {"adapter": "sqlite", "database": store_path}, + "dialect": adapter, + "store": {"adapter": adapter, "database": store_path}, "retention_days": retention_days, - "time_semantics": "timestamps are ISO-8601 UTC text with microseconds", + "time_semantics": time_semantics, "bounds": filter_hint, "relations": items, }); return format!("{document}\n"); } let mut lines: Vec = vec![ - format!("Recall store {store_path} (SQLite dialect, retention {retention_days} days)"), - "Timestamps are ISO-8601 UTC text with microseconds; ".to_owned() + filter_hint + ".", + format!("Recall store {store_path} ({dialect} dialect, retention {retention_days} days)"), + format!("T{}; {filter_hint}.", &time_semantics[1..]), ]; for relation in relations { lines.push(String::new()); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs index 179ad65510..6c0c3bad72 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs @@ -21,12 +21,15 @@ use sqlbuild_recall::store::models::QuarantineOutcome; use sqlbuild_recall::writer::constants::DEFAULT_HEARTBEAT_INTERVAL_MS; use sqlbuild_recall::writer::main::start_recall_session::start_recall_session; use sqlbuild_recall::writer::models::{ - DeliverySummary, RecallSession, SessionConfig, SetupOutcome, + DeliverySummary, RecallSession, SessionConfig, SessionStore, SetupOutcome, }; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::output_capture::records::invocation_context; +use crate::bindings::_helpers::recall::clickhouse_store::{ + ClickHouseLocation, ClickHouseStoreOpener, +}; use crate::bindings::_helpers::recall::console::write_stderr; use crate::bindings::_helpers::recall::constants::{ CLOSE_POLL_MS, DDL_NOTICE_PREFIX, HEARTBEAT_ENVIRONMENT, HOST_TOKEN_LENGTH, @@ -216,25 +219,30 @@ fn permission_notice( (settings, project_dir): (&RecallSettings, &Path), reason: &str, ) -> Option { - let StoreLocation::Sqlite { path } = &settings.store else { - return None; + let (dialect, database, place): (Dialect, Option, String) = match &settings.store { + StoreLocation::Sqlite { path } => (Dialect::Sqlite, None, format!("at {}", path.display())), + StoreLocation::ClickHouse { database, .. } => ( + Dialect::ClickHouse, + Some(database.clone()), + format!("in ClickHouse database `{database}`"), + ), + StoreLocation::Unsupported { .. } => return None, }; - if !notice_due(project_dir, &path.to_string_lossy()) { + if !notice_due(project_dir, &place) { return None; } let script = ddl_script( - Dialect::Sqlite, + dialect, &DdlOptions { - database: None, + database, retention_days: settings.retention_days, cluster: None, }, ); Some(format!( "Recall history is not being written{}: {reason}. An administrator can create the \ - store at {} with:\n\n{script}\n", + store {place} with:\n\n{script}\n", target_phrase(settings), - path.display() )) } @@ -284,8 +292,22 @@ fn started_session( (args, identity, facts): (&Bound<'_, PyAny>, &Bound<'_, PyAny>, &CommandFacts), (settings, project_dir): (&RecallSettings, &Path), ) -> PyResult>> { - let store_path: PathBuf = match &settings.store { - StoreLocation::Sqlite { path } => path.clone(), + let store: SessionStore = match &settings.store { + StoreLocation::Sqlite { path } => { + notify(py, quarantined_notice(path)); + if let Some(reason) = store_location_problem(path) { + notify(py, permission_notice((settings, project_dir), &reason)); + return Ok(None); + } + SessionStore::Sqlite(path.clone()) + } + StoreLocation::ClickHouse { + config, database, .. + } => SessionStore::Hosted(Arc::new(ClickHouseStoreOpener { + location: ClickHouseLocation::new(config, database, settings.retention_days), + retention_notice: notice_due, + project_dir: project_dir.to_path_buf(), + })), StoreLocation::Unsupported { connection, adapter, @@ -297,17 +319,12 @@ fn started_session( return Ok(None); } }; - notify(py, quarantined_notice(&store_path)); - if let Some(reason) = store_location_problem(&store_path) { - notify(py, permission_notice((settings, project_dir), &reason)); - return Ok(None); - } let Some(invocation_id) = text_value(&native_field(identity, "invocation_id")?)? else { return Ok(None); }; let context = invocation_facts(py, (args, facts), (settings, project_dir))?; let session = start_recall_session(SessionConfig { - store_path, + store, maintenance_owner: format!( "{}:{}", context.host_token.as_deref().unwrap_or("host"), @@ -376,7 +393,13 @@ fn closing_notices(recall: &InvocationRecall, session: &RecallSession) -> Vec = session .notices() .into_iter() - .map(|notice| format!("Recall: {notice}.\n")) + .map(|notice| { + if notice.ends_with('\n') { + format!("Recall: {notice}") + } else { + format!("Recall: {notice}.\n") + } + }) .collect(); if let (Some(SetupOutcome::PermissionDenied { reason }), Some((settings, project_dir))) = (session.wait_for_setup(Duration::ZERO), &recall.settings) diff --git a/crates/sqlbuild-recall/src/clickhouse/_helpers/digits.rs b/crates/sqlbuild-recall/src/clickhouse/_helpers/digits.rs new file mode 100644 index 0000000000..baaf7e2504 --- /dev/null +++ b/crates/sqlbuild-recall/src/clickhouse/_helpers/digits.rs @@ -0,0 +1,19 @@ +//! Leading decimal digits of text, without a fallible parse. + +use crate::clickhouse::constants::DECIMAL_RADIX; + +/// The number the leading ASCII digits of `text` spell; `None` when there are none or it overflows. +pub(crate) fn leading_number(text: &str) -> Option { + let mut number: Option = None; + for character in text.chars() { + let Some(digit) = character.to_digit(DECIMAL_RADIX) else { + break; + }; + number = number + .unwrap_or_default() + .checked_mul(u64::from(DECIMAL_RADIX))? + .checked_add(u64::from(digit)) + .map(Some)?; + } + number +} diff --git a/crates/sqlbuild-recall/src/clickhouse/_helpers/literals.rs b/crates/sqlbuild-recall/src/clickhouse/_helpers/literals.rs new file mode 100644 index 0000000000..5b8457bf76 --- /dev/null +++ b/crates/sqlbuild-recall/src/clickhouse/_helpers/literals.rs @@ -0,0 +1,85 @@ +//! ClickHouse literals and key expressions for the store's own queries. + +use std::time::{SystemTime, UNIX_EPOCH}; + +use crate::clickhouse::constants::TIMESTAMP_PATTERN; +use crate::records::main::format_timestamp::format_timestamp; +use crate::records::main::parse_timestamp::parse_timestamp; +use crate::records::models::RecallValue; +use crate::schema::main::table_columns::table_columns; +use crate::schema::models::{ColumnType, Table}; + +/// Microseconds since the epoch; times before it count back. +pub(crate) fn epoch_micros(time: SystemTime) -> i64 { + match time.duration_since(UNIX_EPOCH) { + Ok(after) => i64::try_from(after.as_micros()).unwrap_or(i64::MAX), + Err(before) => -i64::try_from(before.duration().as_micros()).unwrap_or(i64::MAX), + } +} + +/// The microseconds of Recall timestamp text, `None` when it is not one. +pub(crate) fn timestamp_micros(text: &str) -> Option { + parse_timestamp(text).map(epoch_micros) +} + +/// A single-quoted ClickHouse string literal. +pub(crate) fn string_literal(text: &str) -> String { + let mut quoted = String::with_capacity(text.len() + 2); + quoted.push('\''); + for character in text.chars() { + match character { + '\\' => quoted.push_str("\\\\"), + '\'' => quoted.push_str("\\'"), + other => quoted.push(other), + } + } + quoted.push('\''); + quoted +} + +/// The literal of a key value of `column_type`; timestamps as `DateTime64` microseconds. +pub(crate) fn value_literal(value: &RecallValue, column_type: ColumnType) -> String { + match (value, column_type) { + (RecallValue::Text(text), ColumnType::Timestamp) => timestamp_micros(text).map_or_else( + || string_literal(text), + |micros| format!("fromUnixTimestamp64Micro({micros}, 'UTC')"), + ), + (RecallValue::Text(text), _) => string_literal(text), + (RecallValue::Integer(number), _) => number.to_string(), + (RecallValue::Real(number), _) => number.to_string(), + (RecallValue::Blob(bytes), _) => string_literal(&String::from_utf8_lossy(bytes)), + (RecallValue::Null, _) => "NULL".to_owned(), + } +} + +/// The type of `column` in `table`. +pub(crate) fn column_type(table: Table, column: &str) -> ColumnType { + table_columns(table) + .iter() + .find(|spec| spec.name == column) + .map_or(ColumnType::Text, |spec| spec.column_type) +} + +/// SQL rendering `column` as the text a conflict's record key holds. +pub(crate) fn key_text_expression(table: Table, column: &str) -> String { + match column_type(table, column) { + ColumnType::Timestamp => { + format!("formatDateTime(`{column}`, '{TIMESTAMP_PATTERN}', 'UTC')") + } + _ => format!("toString(`{column}`)"), + } +} + +/// A value's text as a record key holds it: timestamps normalised to Recall's format. +pub(crate) fn key_text(value: Option<&RecallValue>, column_type: ColumnType) -> String { + match value { + Some(RecallValue::Text(text)) if column_type == ColumnType::Timestamp => { + parse_timestamp(text).map_or_else(|| text.clone(), format_timestamp) + } + Some(RecallValue::Text(text)) => text.clone(), + Some(RecallValue::Integer(number)) => number.to_string(), + Some(RecallValue::Real(number)) => number.to_string(), + Some(RecallValue::Blob(_)) => "".to_owned(), + Some(RecallValue::Null) | None => String::new(), + } +} diff --git a/crates/sqlbuild-recall/src/clickhouse/_helpers/mod.rs b/crates/sqlbuild-recall/src/clickhouse/_helpers/mod.rs new file mode 100644 index 0000000000..982c7b526b --- /dev/null +++ b/crates/sqlbuild-recall/src/clickhouse/_helpers/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod digits; +pub(crate) mod literals; +pub(crate) mod planning; +pub(crate) mod record_key; +pub(crate) mod row_binary; diff --git a/crates/sqlbuild-recall/src/clickhouse/_helpers/planning.rs b/crates/sqlbuild-recall/src/clickhouse/_helpers/planning.rs new file mode 100644 index 0000000000..b026cac795 --- /dev/null +++ b/crates/sqlbuild-recall/src/clickhouse/_helpers/planning.rs @@ -0,0 +1,166 @@ +//! What an insert plan knows about each key, and the conflict evidence it records. + +use std::collections::{BTreeMap, HashMap}; + +use crate::clickhouse::_helpers::literals::timestamp_micros; +use crate::clickhouse::_helpers::record_key::record_key; +use crate::clickhouse::constants::UNCHECKED_TABLES; +use crate::clickhouse::models::{InsertPlan, StoredRow}; +use crate::records::main::conflict_row::conflict_row; +use crate::records::models::{RecallRow, RecallValue, RowEnvelope}; +use crate::schema::main::table_spec::table_spec; +use crate::schema::models::Table; +use crate::store::models::ConflictEvidence; + +/// What one key holds: its first digest, every digest, and an artifact's newest sighting. +#[derive(Clone, Debug)] +pub(crate) struct Held { + pub(crate) first_hash: String, + pub(crate) hashes: Vec, + pub(crate) newest_micros: Option, +} + +impl Held { + /// A key first written by this batch with `hash`. + pub(crate) fn fresh(hash: &str) -> Self { + Self { + first_hash: hash.to_owned(), + hashes: vec![hash.to_owned()], + newest_micros: None, + } + } + + /// Whether the newest stored copy was seen before `cutoff`. + pub(crate) fn seen_before(&self, cutoff: Option) -> bool { + self.newest_micros + .zip(cutoff) + .is_some_and(|(newest, cutoff)| newest < cutoff) + } +} + +/// A row's physical table name and record key. +pub(crate) fn planned_key(row: &RecallRow) -> (String, String) { + (table_spec(row.table).name.to_owned(), record_key(row)) +} + +/// The evidence and `recall_conflicts` row for `row` rejected against what `key` holds. +pub(crate) fn conflict( + row: &RecallRow, + (record_key, existing): (String, &Held), + detected_at: &str, +) -> (ConflictEvidence, RecallRow) { + let evidence = ConflictEvidence { + table: row.table, + record_key, + existing_sha256: existing.first_hash.clone(), + rejected_sha256: row.content_hash().unwrap_or_default().to_owned(), + }; + let recorded = conflict_record(row, &evidence, detected_at); + (evidence, recorded) +} + +/// The keys already stored, by physical table name and record key. +pub(crate) fn held_keys(stored: &[StoredRow]) -> HashMap<(String, String), Held> { + let mut held: HashMap<(String, String), Held> = HashMap::with_capacity(stored.len()); + for row in stored { + held.insert( + (row.dataset.clone(), row.record_key.clone()), + Held { + first_hash: row.first_hash.clone(), + hashes: row.hashes.clone(), + newest_micros: row.newest_seen_at.as_deref().and_then(timestamp_micros), + }, + ); + } + held +} + +fn text_of(row: &RecallRow, column: &str) -> String { + match row.value(column) { + Some(RecallValue::Text(text)) => text.clone(), + _ => String::new(), + } +} + +/// The `recall_conflicts` row recording `evidence` against `row`'s invocation. +fn conflict_record(row: &RecallRow, evidence: &ConflictEvidence, detected_at: &str) -> RecallRow { + let envelope = RowEnvelope { + invocation_id: text_of(row, "invocation_id"), + invocation_started_at: text_of(row, "invocation_started_at"), + producer_version: text_of(row, "producer_version"), + }; + conflict_row( + &envelope, + (table_spec(row.table).name, &evidence.record_key), + (&evidence.existing_sha256, &evidence.rejected_sha256), + detected_at, + ) +} + +/// Sorts a batch's rows into writes, already-stored copies and conflicts. +pub(crate) struct Planner { + held: HashMap<(String, String), Held>, + refresh_micros: Option, + detected_at: String, + delivered: BTreeMap, + inserts: BTreeMap>, + plan: InsertPlan, +} + +impl Planner { + pub(crate) fn new(stored: &[StoredRow], (detected_at, refresh_before): (&str, &str)) -> Self { + Self { + held: held_keys(stored), + refresh_micros: timestamp_micros(refresh_before), + detected_at: detected_at.to_owned(), + delivered: BTreeMap::new(), + inserts: BTreeMap::new(), + plan: InsertPlan::default(), + } + } + + /// Whether `row` must be written: unchecked, new, or a stale artifact; `None` when it conflicts and was recorded as such. + fn admit(&mut self, row: &RecallRow) -> Option { + let key: (String, String) = planned_key(row); + let hash: &str = row.content_hash().unwrap_or_default(); + if UNCHECKED_TABLES.contains(&row.table) { + return Some(true); + } + let Some(known) = self.held.get(&key) else { + self.held.insert(key, Held::fresh(hash)); + return Some(true); + }; + if !known.hashes.iter().any(|stored| stored == hash) { + let (evidence, recorded) = conflict(row, (key.1.clone(), known), &self.detected_at); + self.inserts + .entry(Table::Conflicts) + .or_default() + .push(recorded); + self.plan.outcome.conflicts.push(evidence); + return None; + } + let stale: bool = row.table == Table::Artifacts && known.seen_before(self.refresh_micros); + if stale { + self.held.insert(key, Held::fresh(hash)); + } + Some(stale) + } + + /// Count `row` as delivered and queue it when it must be written. + pub(crate) fn add(&mut self, row: RecallRow) { + let Some(write) = self.admit(&row) else { + return; + }; + *self.delivered.entry(row.table).or_default() += 1; + if write { + self.inserts.entry(row.table).or_default().push(row); + } + } + + pub(crate) fn finish(self) -> InsertPlan { + let mut plan: InsertPlan = self.plan; + plan.inserts = self.inserts.into_iter().collect(); + plan.outcome.delivered = self.delivered.into_iter().collect(); + plan + } +} diff --git a/crates/sqlbuild-recall/src/clickhouse/_helpers/record_key.rs b/crates/sqlbuild-recall/src/clickhouse/_helpers/record_key.rs new file mode 100644 index 0000000000..84e6c7c5e0 --- /dev/null +++ b/crates/sqlbuild-recall/src/clickhouse/_helpers/record_key.rs @@ -0,0 +1,14 @@ +//! A row's key values joined as conflicts record them. + +use crate::clickhouse::_helpers::literals::{column_type, key_text}; +use crate::records::models::RecallRow; +use crate::schema::main::table_spec::table_spec; + +/// The key of `row`, its values joined by `:`. +pub(crate) fn record_key(row: &RecallRow) -> String { + let mut parts: Vec = Vec::new(); + for name in table_spec(row.table).key { + parts.push(key_text(row.value(name), column_type(row.table, name))); + } + parts.join(":") +} diff --git a/crates/sqlbuild-recall/src/clickhouse/_helpers/row_binary.rs b/crates/sqlbuild-recall/src/clickhouse/_helpers/row_binary.rs new file mode 100644 index 0000000000..1f15a0d968 --- /dev/null +++ b/crates/sqlbuild-recall/src/clickhouse/_helpers/row_binary.rs @@ -0,0 +1,110 @@ +//! `RowBinary` encoding of one Recall value; each function takes the block and returns it extended. + +use crate::clickhouse::_helpers::literals::timestamp_micros; +use crate::clickhouse::constants::{ + DECIMAL_SCALE, MICROS_DIGITS, VARINT_CONTINUE, VARINT_MASK, VARINT_PAYLOAD_BITS, +}; +use crate::clickhouse::models::{ColumnEncoding, StoredColumn}; +use crate::records::models::RecallValue; + +fn varint(mut block: Vec, mut value: u64) -> Vec { + loop { + let byte = u8::try_from(value & VARINT_MASK).unwrap_or_default(); + value >>= VARINT_PAYLOAD_BITS; + if value == 0 { + block.push(byte); + return block; + } + block.push(byte | VARINT_CONTINUE); + } +} + +fn bytes(block: Vec, data: &[u8]) -> Vec { + let mut block = varint(block, u64::try_from(data.len()).unwrap_or(u64::MAX)); + block.extend_from_slice(data); + block +} + +/// `micros` scaled to `precision` fractional digits. +pub(crate) fn scaled(micros: i64, precision: u64) -> i64 { + let mut value: i64 = micros; + for _ in MICROS_DIGITS..precision { + value = value.saturating_mul(DECIMAL_SCALE); + } + for _ in precision..MICROS_DIGITS { + value /= DECIMAL_SCALE; + } + value +} + +fn mismatch(column: &StoredColumn, expected: &str) -> String { + format!( + "Recall column {} holds a value that is not {expected}", + column.name + ) +} + +fn payload(block: Vec, column: &StoredColumn, value: &RecallValue) -> Result, String> { + let mut block = block; + match (column.encoding, value) { + (ColumnEncoding::String, RecallValue::Text(text)) => { + return Ok(bytes(block, text.as_bytes())); + } + (ColumnEncoding::String, RecallValue::Blob(data)) => return Ok(bytes(block, data)), + (ColumnEncoding::String, RecallValue::Null) => return Ok(bytes(block, b"")), + (ColumnEncoding::Int64, RecallValue::Integer(number)) => { + block.extend_from_slice(&number.to_le_bytes()); + } + ( + ColumnEncoding::Int64 | ColumnEncoding::UInt8 | ColumnEncoding::Float64, + RecallValue::Null, + ) => { + let width: usize = if column.encoding == ColumnEncoding::UInt8 { + 1 + } else { + 8 + }; + block.extend(std::iter::repeat_n(0, width)); + } + (ColumnEncoding::UInt8, RecallValue::Integer(number)) => block.push(u8::from(*number != 0)), + (ColumnEncoding::Float64, RecallValue::Real(number)) => { + block.extend_from_slice(&number.to_le_bytes()); + } + (ColumnEncoding::Float64, RecallValue::Integer(number)) => { + let real: f64 = *number as f64; + block.extend_from_slice(&real.to_le_bytes()); + } + (ColumnEncoding::DateTime64(precision), RecallValue::Text(text)) => { + let micros = timestamp_micros(text).ok_or_else(|| mismatch(column, "a timestamp"))?; + block.extend_from_slice(&scaled(micros, precision).to_le_bytes()); + } + (ColumnEncoding::DateTime64(_), RecallValue::Null) => { + block.extend_from_slice(&0_i64.to_le_bytes()) + } + (ColumnEncoding::String, _) => return Err(mismatch(column, "text")), + (ColumnEncoding::Int64 | ColumnEncoding::UInt8, _) => { + return Err(mismatch(column, "an integer")); + } + (ColumnEncoding::Float64, _) => return Err(mismatch(column, "a number")), + (ColumnEncoding::DateTime64(_), _) => return Err(mismatch(column, "a timestamp")), + } + Ok(block) +} + +/// `block` with `value` appended as `column` stores it; `Err` names a value the column cannot hold. +pub(crate) fn encoded( + block: Vec, + column: &StoredColumn, + value: &RecallValue, +) -> Result, String> { + if !column.nullable { + return payload(block, column, value); + } + let mut block = block; + let missing: bool = matches!(value, RecallValue::Null); + block.push(u8::from(missing)); + if missing { + return Ok(block); + } + payload(block, column, value) +} diff --git a/crates/sqlbuild-recall/src/clickhouse/constants.rs b/crates/sqlbuild-recall/src/clickhouse/constants.rs new file mode 100644 index 0000000000..46bf41da3d --- /dev/null +++ b/crates/sqlbuild-recall/src/clickhouse/constants.rs @@ -0,0 +1,51 @@ +//! ClickHouse store batching, error codes, encoding and SQL markers. + +use crate::schema::models::Table; + +/// Rows one insert batch takes; ClickHouse prefers few large inserts to many small ones. +pub const BATCH_ROWS: usize = 5_000; +/// Longest wait for a batch to fill, so each table gets at most about one part per second per writer. +pub const BATCH_WINDOW_MS: u64 = 1_000; +/// Server error codes meaning the user may not create or write the store. +pub const PERMISSION_CODES: [u64; 4] = [ + 164, // READONLY + 291, // DATABASE_ACCESS_DENIED + 497, // ACCESS_DENIED + 516, // AUTHENTICATION_FAILED +]; +/// Server error codes worth retrying with the same rows and deduplication tokens. +pub const TRANSIENT_CODES: [u64; 9] = [ + 159, // TIMEOUT_EXCEEDED + 202, // TOO_MANY_SIMULTANEOUS_QUERIES + 209, // SOCKET_TIMEOUT + 210, // NETWORK_ERROR + 241, // MEMORY_LIMIT_EXCEEDED + 242, // TABLE_IS_READ_ONLY + 252, // TOO_MANY_PARTS + 319, // UNKNOWN_STATUS_OF_INSERT + 425, // SYSTEM_ERROR +]; +/// UNKNOWN_DATABASE: Recall never creates databases. +pub const UNKNOWN_DATABASE_CODE: u64 = 81; +/// Where a client error message states the server code. +pub(crate) const SERVER_CODE_MARKERS: [&str; 2] = ["Code: ", "code: "]; +/// The most fractional digits a `DateTime64` holds. +pub(crate) const MAX_DATETIME64_PRECISION: u64 = 9; +/// The fractional digits of Recall timestamps. +pub(crate) const MICROS_DIGITS: u64 = 6; +/// One decimal digit of timestamp scale. +pub(crate) const DECIMAL_SCALE: i64 = 10; +/// The column the store sets when a row is written. +pub(crate) const INGESTED_AT_COLUMN: &str = "ingested_at"; +pub(crate) const VARINT_PAYLOAD_BITS: u32 = 7; +pub(crate) const VARINT_CONTINUE: u8 = 0x80; +pub(crate) const VARINT_MASK: u64 = 0x7f; +pub(crate) const DECIMAL_RADIX: u32 = 10; +/// Tables written without a lookup: unique by construction, or evidence themselves. +pub(crate) const UNCHECKED_TABLES: [Table; 2] = [Table::Meta, Table::Conflicts]; +/// `formatDateTime` pattern of Recall timestamp text. +pub(crate) const TIMESTAMP_PATTERN: &str = "%Y-%m-%dT%H:%i:%S.%fZ"; +/// Where a `CREATE TABLE` query states its TTL, the clause after it, and the day interval in it. +pub(crate) const TTL_MARKER: &str = " TTL "; +pub(crate) const SETTINGS_MARKER: &str = " SETTINGS "; +pub(crate) const DAY_INTERVAL: &str = "toIntervalDay("; diff --git a/crates/sqlbuild-recall/src/clickhouse/main/deduplication_token.rs b/crates/sqlbuild-recall/src/clickhouse/main/deduplication_token.rs new file mode 100644 index 0000000000..84751b1197 --- /dev/null +++ b/crates/sqlbuild-recall/src/clickhouse/main/deduplication_token.rs @@ -0,0 +1,22 @@ +//! `deduplication_token`: the `insert_deduplication_token` of one table's insert block. + +use crate::clickhouse::_helpers::record_key::record_key; +use crate::records::main::sha256_hex::sha256_hex; +use crate::records::models::RecallRow; +use crate::schema::main::table_spec::table_spec; + +/// A digest of the table and each row's key and content, so a retried block is recognised. +#[must_use] +pub fn deduplication_token(rows: &[RecallRow]) -> String { + let mut material: String = String::new(); + if let Some(first) = rows.first() { + material.push_str(table_spec(first.table).name); + } + for row in rows { + material.push('\n'); + material.push_str(&record_key(row)); + material.push('\u{0}'); + material.push_str(row.content_hash().unwrap_or_default()); + } + sha256_hex(material.as_bytes()) +} diff --git a/crates/sqlbuild-recall/src/clickhouse/main/encode_rows.rs b/crates/sqlbuild-recall/src/clickhouse/main/encode_rows.rs new file mode 100644 index 0000000000..a91f7a0e35 --- /dev/null +++ b/crates/sqlbuild-recall/src/clickhouse/main/encode_rows.rs @@ -0,0 +1,49 @@ +//! `encode_rows`: one table's rows as a `RowBinary` insert block. + +use crate::clickhouse::_helpers::row_binary::{encoded, scaled}; +use crate::clickhouse::constants::{INGESTED_AT_COLUMN, MICROS_DIGITS}; +use crate::clickhouse::models::{ColumnEncoding, StoredColumn}; +use crate::records::models::{RecallRow, RecallValue}; + +/// The column names and `RowBinary` block of `rows`, all of one table, written with `ingested_at`. Values for columns the table lacks are left out; columns the rows lack take their defaults. +pub fn encode_rows( + rows: &[RecallRow], + columns: &[StoredColumn], + ingested_at_micros: i64, +) -> Result<(Vec, Vec), String> { + let Some(first) = rows.first() else { + return Ok((Vec::new(), Vec::new())); + }; + let written: Vec<&StoredColumn> = columns + .iter() + .filter(|column| column.name == INGESTED_AT_COLUMN || first.value(&column.name).is_some()) + .collect(); + let mut block: Vec = Vec::new(); + for row in rows { + for column in &written { + let value: RecallValue = if column.name == INGESTED_AT_COLUMN { + RecallValue::Null + } else { + row.value(&column.name) + .cloned() + .unwrap_or(RecallValue::Null) + }; + if column.name == INGESTED_AT_COLUMN { + let precision: u64 = match column.encoding { + ColumnEncoding::DateTime64(precision) => precision, + _ => MICROS_DIGITS, + }; + if column.nullable { + block.push(0); + } + block.extend_from_slice(&scaled(ingested_at_micros, precision).to_le_bytes()); + continue; + } + block = encoded(block, column, &value)?; + } + } + Ok(( + written.iter().map(|column| column.name.clone()).collect(), + block, + )) +} diff --git a/crates/sqlbuild-recall/src/clickhouse/main/error_kind.rs b/crates/sqlbuild-recall/src/clickhouse/main/error_kind.rs new file mode 100644 index 0000000000..883e2d7b44 --- /dev/null +++ b/crates/sqlbuild-recall/src/clickhouse/main/error_kind.rs @@ -0,0 +1,28 @@ +//! `error_kind`: what a ClickHouse failure means for Recall delivery. + +use crate::clickhouse::_helpers::digits::leading_number; +use crate::clickhouse::constants::{ + PERMISSION_CODES, SERVER_CODE_MARKERS, TRANSIENT_CODES, UNKNOWN_DATABASE_CODE, +}; +use crate::store::errors::StoreErrorKind; + +/// The kind of a client failure from its message: permission codes deny the store, transient codes and failures without a server code (network, timeouts) are retried, an unknown database is reported as such. +#[must_use] +pub fn error_kind(message: &str) -> StoreErrorKind { + let code: Option = SERVER_CODE_MARKERS.iter().find_map(|marker| { + let start: usize = message.find(marker)? + marker.len(); + leading_number(&message[start..]) + }); + let Some(code) = code else { + return StoreErrorKind::Busy; + }; + if PERMISSION_CODES.contains(&code) { + StoreErrorKind::Permission + } else if TRANSIENT_CODES.contains(&code) { + StoreErrorKind::Busy + } else if code == UNKNOWN_DATABASE_CODE { + StoreErrorKind::NotFound + } else { + StoreErrorKind::Other + } +} diff --git a/crates/sqlbuild-recall/src/clickhouse/main/mod.rs b/crates/sqlbuild-recall/src/clickhouse/main/mod.rs new file mode 100644 index 0000000000..25dd9762df --- /dev/null +++ b/crates/sqlbuild-recall/src/clickhouse/main/mod.rs @@ -0,0 +1,8 @@ +pub mod deduplication_token; +pub mod encode_rows; +pub mod error_kind; +pub mod parse_column_encoding; +pub mod plan_insert; +pub mod store_state_sql; +pub mod stored_rows_sql; +pub mod ttl_mismatches; diff --git a/crates/sqlbuild-recall/src/clickhouse/main/parse_column_encoding.rs b/crates/sqlbuild-recall/src/clickhouse/main/parse_column_encoding.rs new file mode 100644 index 0000000000..56dcd739aa --- /dev/null +++ b/crates/sqlbuild-recall/src/clickhouse/main/parse_column_encoding.rs @@ -0,0 +1,34 @@ +//! `parse_column_encoding`: how a stored column's ClickHouse type is written. + +use crate::clickhouse::_helpers::digits::leading_number; +use crate::clickhouse::constants::MAX_DATETIME64_PRECISION; +use crate::clickhouse::models::ColumnEncoding; + +/// The encoding and nullability of `type_text`, `None` for a type Recall never creates. +#[must_use] +pub fn parse_column_encoding(type_text: &str) -> Option<(ColumnEncoding, bool)> { + let text: &str = type_text.trim(); + if let Some(inner) = text + .strip_prefix("Nullable(") + .and_then(|rest| rest.strip_suffix(')')) + { + return parse_column_encoding(inner) + .filter(|(_, nullable)| !nullable) + .map(|(encoding, _)| (encoding, true)); + } + let encoding = match text { + "String" => ColumnEncoding::String, + "Int64" => ColumnEncoding::Int64, + "Float64" => ColumnEncoding::Float64, + "UInt8" | "Bool" => ColumnEncoding::UInt8, + _ => { + let arguments: &str = text.strip_prefix("DateTime64(")?; + let precision: u64 = leading_number(arguments)?; + if precision > MAX_DATETIME64_PRECISION { + return None; + } + ColumnEncoding::DateTime64(precision) + } + }; + Some((encoding, false)) +} diff --git a/crates/sqlbuild-recall/src/clickhouse/main/plan_insert.rs b/crates/sqlbuild-recall/src/clickhouse/main/plan_insert.rs new file mode 100644 index 0000000000..557ea51ef9 --- /dev/null +++ b/crates/sqlbuild-recall/src/clickhouse/main/plan_insert.rs @@ -0,0 +1,19 @@ +//! `plan_insert`: which rows of a batch still need writing, and which reuse a key with different content. + +use crate::clickhouse::_helpers::planning::Planner; +use crate::clickhouse::models::{InsertPlan, StoredRow}; +use crate::records::models::RecallRow; + +/// Plan `rows` against what the store holds. An equal copy is delivered without writing; an artifact is written again only when its newest copy was seen before `refresh_before` (Recall timestamp text), so it outlives every reference. A key holding different content keeps the stored row, and a `recall_conflicts` row records the rejected digest. +#[must_use] +pub fn plan_insert( + rows: Vec, + stored: &[StoredRow], + (detected_at, refresh_before): (&str, &str), +) -> InsertPlan { + let mut planner = Planner::new(stored, (detected_at, refresh_before)); + for row in rows { + planner.add(row); + } + planner.finish() +} diff --git a/crates/sqlbuild-recall/src/clickhouse/main/store_state_sql.rs b/crates/sqlbuild-recall/src/clickhouse/main/store_state_sql.rs new file mode 100644 index 0000000000..d8c021948b --- /dev/null +++ b/crates/sqlbuild-recall/src/clickhouse/main/store_state_sql.rs @@ -0,0 +1,47 @@ +//! `store_state_sql`: what setup reads about a ClickHouse store. + +use crate::clickhouse::_helpers::literals::string_literal; +use crate::clickhouse::models::StoreStateSql; +use crate::schema::constants::{ + ALL_TABLES, META_SCHEMA_COMPONENT, META_SCHEMA_VERSION_KEY, VIEW_NAMES, +}; +use crate::schema::main::clickhouse_relation::clickhouse_relation; +use crate::schema::main::table_spec::table_spec; +use crate::schema::models::{DdlOptions, Table}; + +/// The relation, column and schema-version queries of the store in `database`. +#[must_use] +pub fn store_state_sql(database: &str) -> StoreStateSql { + let mut names: Vec = Vec::new(); + for table in ALL_TABLES { + names.push(string_literal(table_spec(table).name)); + } + let tables: String = names.join(", "); + for view in VIEW_NAMES { + names.push(string_literal(view)); + } + let database_literal: String = string_literal(database); + let options = DdlOptions { + database: Some(database.to_owned()), + retention_days: 0, + cluster: None, + }; + StoreStateSql { + relations: format!( + "SELECT name, toUInt8(engine = 'View') AS is_view, create_table_query \ + FROM system.tables WHERE database = {database_literal} AND name IN ({})", + names.join(", ") + ), + columns: format!( + "SELECT table, name, type FROM system.columns \ + WHERE database = {database_literal} AND table IN ({tables}) ORDER BY table, position" + ), + schema_version: format!( + "SELECT toInt64(max(toInt64OrZero(value))) AS schema_version FROM {} \ + WHERE component = {} AND key = {}", + clickhouse_relation(table_spec(Table::Meta).name, &options), + string_literal(META_SCHEMA_COMPONENT), + string_literal(META_SCHEMA_VERSION_KEY) + ), + } +} diff --git a/crates/sqlbuild-recall/src/clickhouse/main/stored_rows_sql.rs b/crates/sqlbuild-recall/src/clickhouse/main/stored_rows_sql.rs new file mode 100644 index 0000000000..f889fb3c83 --- /dev/null +++ b/crates/sqlbuild-recall/src/clickhouse/main/stored_rows_sql.rs @@ -0,0 +1,85 @@ +//! `stored_rows_sql`: what the keys of one batch already hold in a ClickHouse store. + +use std::collections::BTreeMap; + +use crate::clickhouse::_helpers::literals::{ + column_type, key_text_expression, string_literal, value_literal, +}; +use crate::clickhouse::constants::{TIMESTAMP_PATTERN, UNCHECKED_TABLES}; +use crate::records::models::RecallRow; +use crate::schema::main::clickhouse_relation::clickhouse_relation; +use crate::schema::main::table_spec::table_spec; +use crate::schema::models::{DdlOptions, Table}; + +fn distinct_literals(table: Table, rows: &[&RecallRow], column: &str) -> String { + let mut literals: Vec = rows + .iter() + .filter_map(|row| row.value(column)) + .map(|value| value_literal(value, column_type(table, column))) + .collect(); + literals.sort(); + literals.dedup(); + literals.join(", ") +} + +fn table_lookup(table: Table, rows: &[&RecallRow], options: &DdlOptions) -> String { + let spec = table_spec(table); + let key: Vec = spec + .key + .iter() + .map(|column| key_text_expression(table, column)) + .collect(); + let mut filters: Vec = Vec::new(); + if !spec.key.contains(&"invocation_id") && table != Table::Artifacts { + filters.push(format!( + "invocation_id IN ({})", + distinct_literals(table, rows, "invocation_id") + )); + } + for column in spec.key { + filters.push(format!( + "`{column}` IN ({})", + distinct_literals(table, rows, column) + )); + } + let newest: String = if table == Table::Artifacts { + format!("formatDateTime(max(seen_at), '{TIMESTAMP_PATTERN}', 'UTC')") + } else { + "''".to_owned() + }; + format!( + "SELECT {dataset} AS dataset, {key} AS record_key, \ + argMin(content_hash, ingested_at) AS first_hash, \ + arrayStringConcat(groupUniqArray(content_hash), ',') AS hashes, {newest} AS newest_seen_at \ + FROM {name} WHERE {filters} GROUP BY record_key", + dataset = string_literal(spec.name), + key = if key.len() == 1 { + key[0].clone() + } else { + format!("concat({})", key.join(", ':', ")) + }, + name = clickhouse_relation(spec.name, options), + filters = filters.join(" AND "), + ) +} + +/// One `UNION ALL` query over every checked table of `rows`, `None` when no row needs a lookup. Columns: `dataset`, `record_key`, `first_hash`, `hashes`, `newest_seen_at` (artifacts only, else empty). +#[must_use] +pub fn stored_rows_sql(rows: &[RecallRow], options: &DdlOptions) -> Option { + let mut by_table: BTreeMap> = BTreeMap::new(); + for row in rows { + if !UNCHECKED_TABLES.contains(&row.table) { + by_table.entry(row.table).or_default().push(row); + } + } + if by_table.is_empty() { + return None; + } + Some( + by_table + .iter() + .map(|(table, rows)| table_lookup(*table, rows, options)) + .collect::>() + .join("\nUNION ALL\n"), + ) +} diff --git a/crates/sqlbuild-recall/src/clickhouse/main/ttl_mismatches.rs b/crates/sqlbuild-recall/src/clickhouse/main/ttl_mismatches.rs new file mode 100644 index 0000000000..8994585257 --- /dev/null +++ b/crates/sqlbuild-recall/src/clickhouse/main/ttl_mismatches.rs @@ -0,0 +1,39 @@ +//! `ttl_mismatches`: tables whose stored TTL differs from the configured retention. + +use crate::clickhouse::_helpers::digits::leading_number; +use crate::clickhouse::constants::{DAY_INTERVAL, SETTINGS_MARKER, TTL_MARKER}; +use crate::clickhouse::models::TtlMismatch; +use crate::schema::constants::ALL_TABLES; +use crate::schema::main::table_spec::table_spec; + +/// Each Recall table in `relations` (name, `CREATE` query) whose TTL keeps rows for other than `retention_days` plus the table's extra days. +#[must_use] +pub fn ttl_mismatches(relations: &[(String, String)], retention_days: u32) -> Vec { + let mut mismatches: Vec = Vec::new(); + for table in ALL_TABLES { + let spec = table_spec(table); + let Some((_, query)) = relations.iter().find(|(name, _)| name == spec.name) else { + continue; + }; + let clause: Option<&str> = query + .find(TTL_MARKER) + .map(|start| &query[start + TTL_MARKER.len()..]) + .and_then(|clause| clause.split(SETTINGS_MARKER).next()); + let stored_days: Option = clause + .and_then(|clause| { + clause + .find(DAY_INTERVAL) + .map(|start| &clause[start + DAY_INTERVAL.len()..]) + }) + .and_then(leading_number); + let configured_days: u64 = u64::from(retention_days.saturating_add(spec.ttl_extra_days)); + if stored_days != Some(configured_days) { + mismatches.push(TtlMismatch { + table: spec.name.to_owned(), + stored_days, + configured_days, + }); + } + } + mismatches +} diff --git a/crates/sqlbuild-recall/src/clickhouse/mod.rs b/crates/sqlbuild-recall/src/clickhouse/mod.rs new file mode 100644 index 0000000000..c4d17d2aa6 --- /dev/null +++ b/crates/sqlbuild-recall/src/clickhouse/mod.rs @@ -0,0 +1,9 @@ +//! The ClickHouse Recall store's native parts: row encoding, stored-row lookups, insert planning, error kinds and retention conflicts. The connection is the ClickHouse adapter's, driven by the host. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-recall/src/clickhouse/models.rs b/crates/sqlbuild-recall/src/clickhouse/models.rs new file mode 100644 index 0000000000..c4e322c556 --- /dev/null +++ b/crates/sqlbuild-recall/src/clickhouse/models.rs @@ -0,0 +1,66 @@ +//! A ClickHouse table's columns as written, stored rows found for a batch, and an insert plan. + +use crate::records::models::RecallRow; +use crate::schema::models::Table; +use crate::store::models::InsertOutcome; + +/// How one column is encoded in `RowBinary`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ColumnEncoding { + String, + Int64, + Float64, + UInt8, + /// `DateTime64` with this many fractional digits. + DateTime64(u64), +} + +/// One column of a stored table: its name, encoding and whether it is `Nullable`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct StoredColumn { + pub name: String, + pub encoding: ColumnEncoding, + pub nullable: bool, +} + +/// What a batch's keys already hold in the store. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct StoredRow { + /// The physical table name, such as `recall_events`. + pub dataset: String, + /// The key values joined by `:`, as conflicts record them. + pub record_key: String, + /// The content digest of the first copy ingested. + pub first_hash: String, + /// Every distinct content digest stored under the key. + pub hashes: Vec, + /// Artifacts only: the newest `seen_at`, as Recall timestamp text. + pub newest_seen_at: Option, +} + +/// The rows a batch still needs written, by table, and what the batch did. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct InsertPlan { + pub inserts: Vec<(Table, Vec)>, + pub outcome: InsertOutcome, +} + +/// One table whose TTL differs from the configured retention. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TtlMismatch { + pub table: String, + /// Days the stored TTL keeps rows, `None` when it has no day TTL. + pub stored_days: Option, + pub configured_days: u64, +} + +/// The queries that read a store's state at setup. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct StoreStateSql { + /// `name`, `is_view`, `create_table_query` of every Recall table and view present. + pub relations: String, + /// `table`, `name`, `type` of every column of the Recall tables, in table order. + pub columns: String, + /// The highest schema version recorded, 0 when none. + pub schema_version: String, +} diff --git a/crates/sqlbuild-recall/src/clickhouse/tests.rs b/crates/sqlbuild-recall/src/clickhouse/tests.rs new file mode 100644 index 0000000000..c6e5f9aa07 --- /dev/null +++ b/crates/sqlbuild-recall/src/clickhouse/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_clickhouse.rs"] +mod clickhouse; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-recall/src/clickhouse/tests/helpers.rs b/crates/sqlbuild-recall/src/clickhouse/tests/helpers.rs new file mode 100644 index 0000000000..698944697b --- /dev/null +++ b/crates/sqlbuild-recall/src/clickhouse/tests/helpers.rs @@ -0,0 +1,106 @@ +use crate::clickhouse::main::encode_rows::encode_rows; +use crate::clickhouse::main::plan_insert::plan_insert; +use crate::clickhouse::main::ttl_mismatches::ttl_mismatches; +use crate::clickhouse::models::{ColumnEncoding, StoredColumn, StoredRow}; +use crate::records::main::artifact_rows::artifact_rows; +use crate::records::main::heartbeat_row::heartbeat_row; +use crate::records::models::{ArtifactInput, RecallRow, RowEnvelope}; +use crate::schema::main::table_spec::table_spec; +use crate::schema::models::Table; + +pub(crate) const BEAT_KEY: &str = "inv-1:2026-10-19T00:00:01.000000Z"; +pub(crate) const DETECTED_AT: &str = "2026-10-19T00:00:03.000000Z"; +pub(crate) const REFRESH_BEFORE: &str = "2026-10-12T00:00:03.000000Z"; + +fn envelope() -> RowEnvelope { + RowEnvelope { + invocation_id: "inv-1".to_owned(), + invocation_started_at: "2026-10-19T00:00:00.000000Z".to_owned(), + producer_version: "test".to_owned(), + } +} + +/// A heartbeat row and the artifact and reference rows of one SQL text. +pub(crate) fn batch() -> Vec { + let mut rows: Vec = vec![heartbeat_row( + &envelope(), + ("2026-10-19T00:00:01.000000Z", "2026-10-19T00:01:31.000000Z"), + (Some("host"), Some(7)), + )]; + rows.extend(artifact_rows( + &envelope(), + &ArtifactInput { + digest: "sha256:abc".to_owned(), + kind: "sql".to_owned(), + media_type: "text/x-sql".to_owned(), + content: b"select 1".to_vec(), + seen_at: "2026-10-19T00:00:02.000000Z".to_owned(), + role: "executed_sql".to_owned(), + ref_id: "statement-1".to_owned(), + statement_id: Some("statement-1".to_owned()), + resource_id: None, + }, + )); + rows +} + +/// The content digest the batch's row in `table` carries. +pub(crate) fn batch_hash(table: Table) -> String { + batch() + .into_iter() + .find(|row| row.table == table) + .and_then(|row| row.content_hash().map(str::to_owned)) + .unwrap_or_default() +} + +/// The rows written per table and the conflicts found when the batch meets `stored`. +pub(crate) fn planned(stored: &[StoredRow]) -> (Vec<(&'static str, usize)>, usize) { + let plan = plan_insert(batch(), stored, (DETECTED_AT, REFRESH_BEFORE)); + let mut written: Vec<(&'static str, usize)> = plan + .inserts + .iter() + .map(|(table, rows)| (table_spec(*table).name, rows.len())) + .collect(); + written.sort_unstable(); + (written, plan.outcome.conflicts.len()) +} + +/// A stored key holding one digest. +pub(crate) fn stored(dataset: &str, (key, hash): (&str, &str), newest: Option<&str>) -> StoredRow { + StoredRow { + dataset: dataset.to_owned(), + record_key: key.to_owned(), + first_hash: hash.to_owned(), + hashes: vec![hash.to_owned()], + newest_seen_at: newest.map(str::to_owned), + } +} + +/// The columns and block of the batch's heartbeat written into `columns`. +pub(crate) fn encoded_heartbeat( + columns: &[(&str, ColumnEncoding, bool)], + ingested_at_micros: i64, +) -> Result<(Vec, Vec), String> { + let columns: Vec = columns + .iter() + .map(|(name, encoding, nullable)| StoredColumn { + name: (*name).to_owned(), + encoding: *encoding, + nullable: *nullable, + }) + .collect(); + encode_rows(&batch()[..1], &columns, ingested_at_micros) +} + +/// The little-endian bytes of the pieces, in order. +pub(crate) fn block(pieces: &[&[u8]]) -> Vec { + pieces.concat() +} + +/// The stored TTL days of the first table off `retention_days`, for an output table created by `create_query`. +pub(crate) fn first_mismatch(create_query: &str, retention_days: u32) -> Option> { + let relations = [("recall_output".to_owned(), create_query.to_owned())]; + ttl_mismatches(&relations, retention_days) + .first() + .map(|mismatch| mismatch.stored_days) +} diff --git a/crates/sqlbuild-recall/src/clickhouse/tests/test_clickhouse.rs b/crates/sqlbuild-recall/src/clickhouse/tests/test_clickhouse.rs new file mode 100644 index 0000000000..e3f6d0f5a8 --- /dev/null +++ b/crates/sqlbuild-recall/src/clickhouse/tests/test_clickhouse.rs @@ -0,0 +1,285 @@ +use crate::clickhouse::main::error_kind::error_kind; +use crate::clickhouse::main::parse_column_encoding::parse_column_encoding; +use crate::clickhouse::main::stored_rows_sql::stored_rows_sql; +use crate::clickhouse::models::ColumnEncoding; +use crate::clickhouse::tests::helpers::{ + BEAT_KEY, batch, batch_hash, block, encoded_heartbeat, first_mismatch, planned, stored, +}; +use crate::clickhouse::tests::test_types::{ + EncodingTestCase, ErrorKindTestCase, LookupTestCase, PlanTestCase, RowBinaryTestCase, + TtlTestCase, +}; +use crate::schema::models::{DdlOptions, Table}; +use crate::store::errors::StoreErrorKind; + +const OPTIONS: fn() -> DdlOptions = || DdlOptions { + database: Some("observability".to_owned()), + retention_days: 30, + cluster: None, +}; + +#[test] +fn given_column_types_when_parsed_then_recall_types_have_an_encoding() { + let test_cases = [ + EncodingTestCase { + description: "a plain string", + type_text: "String", + expected_encoding: Some((ColumnEncoding::String, false)), + }, + EncodingTestCase { + description: "a nullable integer", + type_text: "Nullable(Int64)", + expected_encoding: Some((ColumnEncoding::Int64, true)), + }, + EncodingTestCase { + description: "a timestamp with its zone", + type_text: "DateTime64(6, 'UTC')", + expected_encoding: Some((ColumnEncoding::DateTime64(6), false)), + }, + EncodingTestCase { + description: "a type Recall never creates", + type_text: "Array(String)", + expected_encoding: None, + }, + ]; + + for test_case in test_cases { + assert_eq!( + parse_column_encoding(test_case.type_text), + test_case.expected_encoding, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_a_heartbeat_when_encoded_then_the_block_is_row_binary_with_ingested_at() { + let beat_at: [u8; 8] = 1_792_368_001_000_000_i64.to_le_bytes(); + let pid: [u8; 8] = 7_i64.to_le_bytes(); + let ingested_millis: [u8; 8] = 1_000_i64.to_le_bytes(); + let test_cases = [ + RowBinaryTestCase { + description: "nullable columns carry a null flag and absent columns are left out", + columns: &[ + ("beat_at", ColumnEncoding::DateTime64(6), false), + ("host_token", ColumnEncoding::String, true), + ("pid", ColumnEncoding::Int64, true), + ("absent_in_rows", ColumnEncoding::String, true), + ("ingested_at", ColumnEncoding::DateTime64(3), false), + ], + expected_block: Ok(( + vec![ + "beat_at".to_owned(), + "host_token".to_owned(), + "pid".to_owned(), + "ingested_at".to_owned(), + ], + block(&[&beat_at, &[0, 4], b"host", &[0], &pid, &ingested_millis]), + )), + }, + RowBinaryTestCase { + description: "a timestamp column refuses an integer", + columns: &[("pid", ColumnEncoding::DateTime64(6), false)], + expected_block: Err( + "Recall column pid holds a value that is not a timestamp".to_owned() + ), + }, + ]; + + for test_case in test_cases { + assert_eq!( + encoded_heartbeat(test_case.columns, 1_000_000), + test_case.expected_block, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_stored_rows_when_planned_then_duplicates_are_skipped_conflicts_kept_and_stale_artifacts_refreshed() + { + let beat_hash: String = batch_hash(Table::Heartbeats); + let artifact_hash: String = batch_hash(Table::Artifacts); + let test_cases = [ + PlanTestCase { + description: "nothing stored writes every row", + stored: vec![], + expected_plan: ( + vec![ + ("recall_artifact_refs", 1), + ("recall_artifacts", 1), + ("recall_heartbeats", 1), + ], + 0, + ), + }, + PlanTestCase { + description: "equal copies and a recently seen artifact are not written again", + stored: vec![ + stored("recall_heartbeats", (BEAT_KEY, &beat_hash), None), + stored( + "recall_artifacts", + ("sha256:abc", &artifact_hash), + Some("2026-10-18T00:00:00.000000Z"), + ), + ], + expected_plan: (vec![("recall_artifact_refs", 1)], 0), + }, + PlanTestCase { + description: "an artifact last seen before the refresh cutoff is written again", + stored: vec![stored( + "recall_artifacts", + ("sha256:abc", &artifact_hash), + Some("2026-10-01T00:00:00.000000Z"), + )], + expected_plan: ( + vec![ + ("recall_artifact_refs", 1), + ("recall_artifacts", 1), + ("recall_heartbeats", 1), + ], + 0, + ), + }, + PlanTestCase { + description: "a key holding other content keeps it and records a conflict", + stored: vec![stored("recall_heartbeats", (BEAT_KEY, "other"), None)], + expected_plan: ( + vec![ + ("recall_artifact_refs", 1), + ("recall_artifacts", 1), + ("recall_conflicts", 1), + ], + 1, + ), + }, + ]; + + for test_case in test_cases { + assert_eq!( + planned(&test_case.stored), + test_case.expected_plan, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_a_batch_when_looked_up_then_each_table_is_filtered_by_its_keys() { + let test_cases = [ + LookupTestCase { + description: "heartbeats and artifacts are looked up by their keys in one query", + sql: stored_rows_sql(&batch(), &OPTIONS()), + expected_fragments: &[ + "FROM `observability`.`recall_heartbeats` WHERE `invocation_id` IN ('inv-1') \ + AND `beat_at` IN (fromUnixTimestamp64Micro(1792368001000000, 'UTC'))", + "FROM `observability`.`recall_artifacts` WHERE `digest` IN ('sha256:abc') \ + GROUP BY record_key", + "\nUNION ALL\n", + ], + }, + LookupTestCase { + description: "an empty batch needs no lookup", + sql: stored_rows_sql(&[], &OPTIONS()), + expected_fragments: &[], + }, + ]; + + for test_case in test_cases { + let sql: String = test_case.sql.clone().unwrap_or_default(); + assert_eq!( + test_case + .expected_fragments + .iter() + .filter(|fragment| !sql.contains(**fragment)) + .collect::>(), + Vec::<&&str>::new(), + "{}: {sql}", + test_case.description + ); + assert_eq!( + test_case.sql.is_some(), + !test_case.expected_fragments.is_empty(), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_failure_messages_when_classified_then_permission_and_transient_codes_are_distinguished() { + let test_cases = [ + ErrorKindTestCase { + description: "access denied", + message: "Code: 497. DB::Exception: default: Not enough privileges. (ACCESS_DENIED)", + expected_kind: StoreErrorKind::Permission, + }, + ErrorKindTestCase { + description: "too many parts is retried", + message: "Received ClickHouse exception, code: 252, server response: Code: 252.", + expected_kind: StoreErrorKind::Busy, + }, + ErrorKindTestCase { + description: "a network failure without a server code is retried", + message: "Error HTTPConnectionPool(host='localhost', port=1): Max retries exceeded", + expected_kind: StoreErrorKind::Busy, + }, + ErrorKindTestCase { + description: "an unknown database is reported", + message: "Code: 81. DB::Exception: Database observability does not exist.", + expected_kind: StoreErrorKind::NotFound, + }, + ErrorKindTestCase { + description: "other server errors are not retried", + message: "Code: 62. DB::Exception: Syntax error", + expected_kind: StoreErrorKind::Other, + }, + ]; + + for test_case in test_cases { + assert_eq!( + error_kind(test_case.message), + test_case.expected_kind, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_stored_ttls_when_compared_then_tables_off_the_configured_retention_are_reported() { + let test_cases = [ + TtlTestCase { + description: "the configured retention matches", + create_query: "CREATE TABLE d.recall_output (`x` String) ENGINE = ReplacingMergeTree \ + ORDER BY x TTL invocation_started_at + toIntervalDay(30) SETTINGS a = 1", + retention_days: 30, + expected_stored_days: None, + }, + TtlTestCase { + description: "a store kept for another retention", + create_query: "CREATE TABLE d.recall_output (`x` String) ENGINE = ReplacingMergeTree \ + ORDER BY x TTL invocation_started_at + toIntervalDay(30) SETTINGS a = 1", + retention_days: 7, + expected_stored_days: Some(Some(30)), + }, + TtlTestCase { + description: "a table without a TTL", + create_query: "CREATE TABLE d.recall_output (`x` String) ENGINE = MergeTree ORDER BY x", + retention_days: 30, + expected_stored_days: Some(None), + }, + ]; + + for test_case in test_cases { + assert_eq!( + first_mismatch(test_case.create_query, test_case.retention_days), + test_case.expected_stored_days, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-recall/src/clickhouse/tests/test_types.rs b/crates/sqlbuild-recall/src/clickhouse/tests/test_types.rs new file mode 100644 index 0000000000..b3aee93253 --- /dev/null +++ b/crates/sqlbuild-recall/src/clickhouse/tests/test_types.rs @@ -0,0 +1,39 @@ +use crate::clickhouse::models::{ColumnEncoding, StoredRow}; +use crate::store::errors::StoreErrorKind; + +pub(crate) struct EncodingTestCase { + pub(crate) description: &'static str, + pub(crate) type_text: &'static str, + pub(crate) expected_encoding: Option<(ColumnEncoding, bool)>, +} + +pub(crate) struct RowBinaryTestCase { + pub(crate) description: &'static str, + pub(crate) columns: &'static [(&'static str, ColumnEncoding, bool)], + pub(crate) expected_block: Result<(Vec, Vec), String>, +} + +pub(crate) struct ErrorKindTestCase { + pub(crate) description: &'static str, + pub(crate) message: &'static str, + pub(crate) expected_kind: StoreErrorKind, +} + +pub(crate) struct PlanTestCase { + pub(crate) description: &'static str, + pub(crate) stored: Vec, + pub(crate) expected_plan: (Vec<(&'static str, usize)>, usize), +} + +pub(crate) struct LookupTestCase { + pub(crate) description: &'static str, + pub(crate) sql: Option, + pub(crate) expected_fragments: &'static [&'static str], +} + +pub(crate) struct TtlTestCase { + pub(crate) description: &'static str, + pub(crate) create_query: &'static str, + pub(crate) retention_days: u32, + pub(crate) expected_stored_days: Option>, +} diff --git a/crates/sqlbuild-recall/src/lib.rs b/crates/sqlbuild-recall/src/lib.rs index 53e5776303..52caf94341 100644 --- a/crates/sqlbuild-recall/src/lib.rs +++ b/crates/sqlbuild-recall/src/lib.rs @@ -2,6 +2,7 @@ #![forbid(unsafe_code)] +pub mod clickhouse; pub mod records; pub mod schema; pub mod settings; diff --git a/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs b/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs index d84e3e1713..f94ab19739 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs @@ -40,12 +40,12 @@ started AS ( ), terminals AS ( SELECT invocation_id, - argMax(event_type, tuple(invocation_sequence, occurred_at)) AS event_type, - argMax(occurred_at, tuple(invocation_sequence, occurred_at)) AS occurred_at, - argMax(exit_code, tuple(invocation_sequence, occurred_at)) AS exit_code, - argMax(duration_ms, tuple(invocation_sequence, occurred_at)) AS duration_ms, - argMax(error_type, tuple(invocation_sequence, occurred_at)) AS error_type, - argMax(error_code, tuple(invocation_sequence, occurred_at)) AS error_code + argMax(event_type, tuple(invocation_sequence, occurred_at)) AS terminal_type, + argMax(occurred_at, tuple(invocation_sequence, occurred_at)) AS terminal_at, + argMax(exit_code, tuple(invocation_sequence, occurred_at)) AS terminal_exit_code, + argMax(duration_ms, tuple(invocation_sequence, occurred_at)) AS terminal_duration_ms, + argMax(error_type, tuple(invocation_sequence, occurred_at)) AS terminal_error_type, + argMax(error_code, tuple(invocation_sequence, occurred_at)) AS terminal_error_code FROM {events} WHERE event_type IN ('invocation_completed', 'invocation_failed') GROUP BY invocation_id @@ -59,21 +59,21 @@ SELECT ids.invocation_started_at AS invocation_started_at, coalesce(started.started_at, ids.invocation_started_at) AS started_at, multiIf( - terminals.event_type = 'invocation_completed', 'completed', - terminals.event_type = 'invocation_failed', 'failed', + terminals.terminal_type = 'invocation_completed', 'completed', + terminals.terminal_type = 'invocation_failed', 'failed', beats.stale_at IS NULL, 'unknown', beats.stale_at >= now64(6), 'running', 'abandoned' ) AS status, - if(terminals.event_type IS NOT NULL, terminals.occurred_at, + if(terminals.terminal_type IS NOT NULL, terminals.terminal_at, if(beats.stale_at < now64(6), beats.last_heartbeat_at, NULL)) AS ended_at, - if(terminals.event_type IS NOT NULL, terminals.duration_ms, + if(terminals.terminal_type IS NOT NULL, terminals.terminal_duration_ms, if(beats.stale_at < now64(6), toFloat64(dateDiff('millisecond', started_at, beats.last_heartbeat_at)), NULL)) AS duration_ms, - terminals.exit_code AS exit_code, - terminals.error_type AS error_type, - terminals.error_code AS error_code, + terminals.terminal_exit_code AS exit_code, + terminals.terminal_error_type AS error_type, + terminals.terminal_error_code AS error_code, coalesce(context.command, started.command) AS command, context.subcommand AS subcommand, context.target AS target, @@ -108,15 +108,15 @@ facts AS ( ), firsts AS ( SELECT statement_id, - argMin(invocation_id, tuple(invocation_sequence, occurred_at)) AS invocation_id, - argMin(run_id, tuple(invocation_sequence, occurred_at)) AS run_id, - argMin(resource_id, tuple(invocation_sequence, occurred_at)) AS resource_id, - argMin(resource_attempt_id, tuple(invocation_sequence, occurred_at)) AS resource_attempt_id, - argMin(operation_id, tuple(invocation_sequence, occurred_at)) AS operation_id, - argMin(intent, tuple(invocation_sequence, occurred_at)) AS intent, - argMin(statement_kind, tuple(invocation_sequence, occurred_at)) AS statement_kind, - argMin(purpose, tuple(invocation_sequence, occurred_at)) AS purpose, - argMin(sql_digest, tuple(invocation_sequence, occurred_at)) AS sql_digest, + argMin(invocation_id, tuple(invocation_sequence, occurred_at)) AS first_invocation_id, + argMin(run_id, tuple(invocation_sequence, occurred_at)) AS first_run_id, + argMin(resource_id, tuple(invocation_sequence, occurred_at)) AS first_resource_id, + argMin(resource_attempt_id, tuple(invocation_sequence, occurred_at)) AS first_resource_attempt_id, + argMin(operation_id, tuple(invocation_sequence, occurred_at)) AS first_operation_id, + argMin(intent, tuple(invocation_sequence, occurred_at)) AS first_intent, + argMin(statement_kind, tuple(invocation_sequence, occurred_at)) AS first_statement_kind, + argMin(purpose, tuple(invocation_sequence, occurred_at)) AS first_purpose, + argMin(sql_digest, tuple(invocation_sequence, occurred_at)) AS first_sql_digest, minIf(occurred_at, event_type = 'statement_started') AS started_at, minIf(occurred_at, event_type = 'statement_submitted') AS submitted_at, max(query_id) AS any_query_id @@ -124,58 +124,58 @@ firsts AS ( ), terminals AS ( SELECT statement_id, - argMax(event_type, tuple(invocation_sequence, occurred_at)) AS event_type, + argMax(event_type, tuple(invocation_sequence, occurred_at)) AS last_event_type, argMax(occurred_at, tuple(invocation_sequence, occurred_at)) AS ended_at, - argMax(query_id, tuple(invocation_sequence, occurred_at)) AS query_id, - argMax(error_type, tuple(invocation_sequence, occurred_at)) AS error_type, - argMax(error_code, tuple(invocation_sequence, occurred_at)) AS error_code, - argMax(duration_ms, tuple(invocation_sequence, occurred_at)) AS duration_ms, - argMax(row_count, tuple(invocation_sequence, occurred_at)) AS row_count, - argMax(affected_rows, tuple(invocation_sequence, occurred_at)) AS affected_rows + argMax(query_id, tuple(invocation_sequence, occurred_at)) AS last_query_id, + argMax(error_type, tuple(invocation_sequence, occurred_at)) AS last_error_type, + argMax(error_code, tuple(invocation_sequence, occurred_at)) AS last_error_code, + argMax(duration_ms, tuple(invocation_sequence, occurred_at)) AS last_duration_ms, + argMax(row_count, tuple(invocation_sequence, occurred_at)) AS last_row_count, + argMax(affected_rows, tuple(invocation_sequence, occurred_at)) AS last_affected_rows FROM facts WHERE event_type IN ('statement_completed', 'statement_failed') GROUP BY statement_id ), resources AS ( - SELECT invocation_id, resource_id, min(resource_name) AS resource_name, - min(resource_kind) AS resource_kind + SELECT invocation_id, resource_id, min(resource_name) AS first_resource_name, + min(resource_kind) AS first_resource_kind FROM {events} WHERE resource_id IS NOT NULL AND resource_name IS NOT NULL GROUP BY invocation_id, resource_id ) SELECT - firsts.invocation_id AS invocation_id, + firsts.first_invocation_id AS invocation_id, firsts.statement_id AS statement_id, - firsts.run_id AS run_id, - firsts.resource_id AS resource_id, - resources.resource_name AS resource_name, - resources.resource_kind AS resource_kind, - firsts.resource_attempt_id AS resource_attempt_id, - firsts.operation_id AS operation_id, - firsts.intent AS intent, - firsts.statement_kind AS statement_kind, - firsts.purpose AS purpose, - coalesce(terminals.query_id, firsts.any_query_id) AS query_id, + firsts.first_run_id AS run_id, + firsts.first_resource_id AS resource_id, + resources.first_resource_name AS resource_name, + resources.first_resource_kind AS resource_kind, + firsts.first_resource_attempt_id AS resource_attempt_id, + firsts.first_operation_id AS operation_id, + firsts.first_intent AS intent, + firsts.first_statement_kind AS statement_kind, + firsts.first_purpose AS purpose, + coalesce(terminals.last_query_id, firsts.any_query_id) AS query_id, firsts.started_at AS started_at, firsts.submitted_at AS submitted_at, terminals.ended_at AS ended_at, multiIf( - terminals.event_type = 'statement_completed', 'completed', - terminals.event_type = 'statement_failed', 'failed', + terminals.last_event_type = 'statement_completed', 'completed', + terminals.last_event_type = 'statement_failed', 'failed', 'no_terminal' ) AS outcome, - terminals.error_type AS error_type, - terminals.error_code AS error_code, - terminals.duration_ms AS duration_ms, - terminals.row_count AS row_count, - terminals.affected_rows AS affected_rows, - firsts.sql_digest AS sql_digest, - concat('sha256:', firsts.sql_digest) IN (SELECT digest FROM {artifacts}) AS sql_available + terminals.last_error_type AS error_type, + terminals.last_error_code AS error_code, + terminals.last_duration_ms AS duration_ms, + terminals.last_row_count AS row_count, + terminals.last_affected_rows AS affected_rows, + firsts.first_sql_digest AS sql_digest, + concat('sha256:', firsts.first_sql_digest) IN (SELECT digest FROM {artifacts}) AS sql_available FROM firsts LEFT JOIN terminals ON terminals.statement_id = firsts.statement_id LEFT JOIN resources - ON resources.invocation_id = firsts.invocation_id - AND resources.resource_id = firsts.resource_id + ON resources.invocation_id = firsts.first_invocation_id + AND resources.resource_id = firsts.first_resource_id SETTINGS join_use_nulls = 1" ) } diff --git a/crates/sqlbuild-recall/src/schema/main/clickhouse_relation.rs b/crates/sqlbuild-recall/src/schema/main/clickhouse_relation.rs new file mode 100644 index 0000000000..93b0d1f532 --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/main/clickhouse_relation.rs @@ -0,0 +1,10 @@ +//! `clickhouse_relation`: a Recall table or view name as ClickHouse SQL refers to it. + +use crate::schema::_helpers::rendering::clickhouse_table; +use crate::schema::models::DdlOptions; + +/// `` `database`.`name` `` with the configured database, else `` `name` ``. +#[must_use] +pub fn clickhouse_relation(name: &str, options: &DdlOptions) -> String { + clickhouse_table(name, options) +} diff --git a/crates/sqlbuild-recall/src/schema/main/mod.rs b/crates/sqlbuild-recall/src/schema/main/mod.rs index 288b8f6614..854c85f2d9 100644 --- a/crates/sqlbuild-recall/src/schema/main/mod.rs +++ b/crates/sqlbuild-recall/src/schema/main/mod.rs @@ -1,5 +1,7 @@ pub mod add_column_statement; +pub mod clickhouse_relation; pub mod ddl_script; +pub mod retention_fix_statements; pub mod sqlite_retention_sql; pub mod table_columns; pub mod table_spec; diff --git a/crates/sqlbuild-recall/src/schema/main/retention_fix_statements.rs b/crates/sqlbuild-recall/src/schema/main/retention_fix_statements.rs new file mode 100644 index 0000000000..bc758e8a04 --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/main/retention_fix_statements.rs @@ -0,0 +1,29 @@ +//! `retention_fix_statements`: what an administrator runs to give a ClickHouse store another retention. + +use crate::schema::_helpers::rendering::{clickhouse_table, clickhouse_ttl}; +use crate::schema::constants::ALL_TABLES; +use crate::schema::main::table_spec::table_spec; +use crate::schema::main::view_statements::view_statements; +use crate::schema::models::{DdlOptions, Dialect, TableSpec}; + +/// `ALTER TABLE … MODIFY TTL` for each table named in `tables`, then the views, which filter by retention; each statement ends with `;`. +#[must_use] +pub fn retention_fix_statements(tables: &[String], options: &DdlOptions) -> String { + let mut statements: Vec = Vec::new(); + for table in ALL_TABLES { + let spec: &TableSpec = table_spec(table); + if tables.iter().any(|name| name == spec.name) { + statements.push(format!( + "ALTER TABLE {} MODIFY TTL {}", + clickhouse_table(spec.name, options), + clickhouse_ttl(spec, options) + )); + } + } + for (_, statement) in view_statements(Dialect::ClickHouse, options) { + statements.push(statement); + } + let mut script: String = statements.join(";\n\n"); + script.push_str(";\n"); + script +} diff --git a/crates/sqlbuild-recall/src/settings/_helpers/layers.rs b/crates/sqlbuild-recall/src/settings/_helpers/layers.rs index 5ea24d617e..f18f117f25 100644 --- a/crates/sqlbuild-recall/src/settings/_helpers/layers.rs +++ b/crates/sqlbuild-recall/src/settings/_helpers/layers.rs @@ -8,6 +8,7 @@ use sqlbuild_config::recall::models::RecallSection; use crate::settings::_helpers::documents::{expand_environment, lookup}; use crate::settings::constants::{ + CLICKHOUSE_ADAPTER, CLICKHOUSE_DATABASE_KEY, CLICKHOUSE_DEFAULT_DATABASE, CONNECTION_ADAPTER_KEY, DEFAULT_STORE_DIRECTORY, DEFAULT_STORE_FILE_NAME, SQLITE_ADAPTER, SQLITE_PATH_KEY, }; @@ -61,6 +62,61 @@ fn connection_entries( merged } +/// `value` with `${env:...}` expanded in every string it holds. +fn expanded(value: ConfigValue, label: &str) -> Result { + Ok(match value { + ConfigValue::String(text) => ConfigValue::String(expand_environment(&text, label)?), + ConfigValue::List(items) => ConfigValue::List( + items + .into_iter() + .map(|item| expanded(item, label)) + .collect::>()?, + ), + ConfigValue::Map(entries) => ConfigValue::Map( + entries + .into_iter() + .map(|(key, item)| Ok((key, expanded(item, label)?))) + .collect::>()?, + ), + other => other, + }) +} + +fn clickhouse_location( + name: &str, + entries: Vec<(String, ConfigValue)>, +) -> Result { + let mut config: Vec<(String, ConfigValue)> = Vec::with_capacity(entries.len()); + for (key, value) in entries { + if key == CONNECTION_ADAPTER_KEY { + continue; + } + let label = format!("connections.{name}.{key}"); + config.push((key, expanded(value, &label)?)); + } + let database: String = match config + .iter() + .find(|(key, _)| key == CLICKHOUSE_DATABASE_KEY) + .map(|(_, value)| value) + { + None => CLICKHOUSE_DEFAULT_DATABASE.to_owned(), + Some(ConfigValue::String(database)) if !database.trim().is_empty() => { + database.trim().to_owned() + } + Some(_) => { + return Err(SettingsError::new(format!( + "connections.{name}.database must name the existing ClickHouse database Recall \ + writes to" + ))); + } + }; + Ok(StoreLocation::ClickHouse { + connection: name.to_owned(), + config, + database, + }) +} + pub(crate) fn store_location( project_dir: &Path, documents: [Option<&ConfigValue>; 2], @@ -93,6 +149,9 @@ pub(crate) fn store_location( } None => project_adapter.unwrap_or_default().to_owned(), }; + if adapter == CLICKHOUSE_ADAPTER { + return clickhouse_location(name, entries); + } if adapter != SQLITE_ADAPTER { return Ok(StoreLocation::Unsupported { connection: name.to_owned(), diff --git a/crates/sqlbuild-recall/src/settings/constants.rs b/crates/sqlbuild-recall/src/settings/constants.rs index 76d7d584e6..97a4ea7b67 100644 --- a/crates/sqlbuild-recall/src/settings/constants.rs +++ b/crates/sqlbuild-recall/src/settings/constants.rs @@ -9,6 +9,11 @@ pub const DEFAULT_DIAGNOSTICS_MIN_SEVERITY: &str = "info"; /// The connection key naming a connection's adapter; never passed to `connect()`. pub const CONNECTION_ADAPTER_KEY: &str = "adapter"; pub const SQLITE_ADAPTER: &str = "sqlite"; +pub const CLICKHOUSE_ADAPTER: &str = "clickhouse"; +/// The ClickHouse connection option naming the database Recall's tables live in. +pub const CLICKHOUSE_DATABASE_KEY: &str = "database"; +/// The database a ClickHouse connection uses when it names none. +pub const CLICKHOUSE_DEFAULT_DATABASE: &str = "default"; pub const SQLITE_PATH_KEY: &str = "path"; pub const PROJECT_FILE_NAME: &str = "sqlbuild_project.toml"; pub const LOCAL_FILE_NAME: &str = "sqlbuild_local.toml"; diff --git a/crates/sqlbuild-recall/src/settings/models.rs b/crates/sqlbuild-recall/src/settings/models.rs index cf9ffa698c..075d16ba15 100644 --- a/crates/sqlbuild-recall/src/settings/models.rs +++ b/crates/sqlbuild-recall/src/settings/models.rs @@ -2,30 +2,39 @@ use std::path::PathBuf; +use sqlbuild_config::models::ConfigValue; + use crate::settings::constants::DEFAULT_SQLITE_MAX_OUTPUT_BYTES; /// Where Recall writes. -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq)] pub enum StoreLocation { /// A SQLite file, written natively. Sqlite { path: PathBuf }, + /// An existing ClickHouse database, written through the ClickHouse adapter's connection. + ClickHouse { + connection: String, + /// The connection options `connect()` receives: `adapter` removed, `${env:...}` expanded. + config: Vec<(String, ConfigValue)>, + database: String, + }, /// A connection whose adapter this build cannot write Recall rows to yet. Unsupported { connection: String, adapter: String }, } impl StoreLocation { - /// The per-stream output limit when none is configured: bounded for the local file, which every command writes; `None` records everything. + /// The per-stream output limit when none is configured: bounded for the local file, which every command writes; a ClickHouse store, sized by its operator, records everything. #[must_use] pub fn default_max_output_bytes(&self) -> Option { match self { Self::Sqlite { .. } => Some(DEFAULT_SQLITE_MAX_OUTPUT_BYTES), - Self::Unsupported { .. } => None, + Self::ClickHouse { .. } | Self::Unsupported { .. } => None, } } } /// The effective settings: target, store and policies. -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq)] pub struct RecallSettings { pub enabled: bool, /// The effective target; `None` for commands without one. diff --git a/crates/sqlbuild-recall/src/settings/tests/test_resolve.rs b/crates/sqlbuild-recall/src/settings/tests/test_resolve.rs index cd7a225b7b..a5c11c43b3 100644 --- a/crates/sqlbuild-recall/src/settings/tests/test_resolve.rs +++ b/crates/sqlbuild-recall/src/settings/tests/test_resolve.rs @@ -1,3 +1,5 @@ +use sqlbuild_config::models::ConfigValue; + use crate::settings::models::{RecallSettings, StoreLocation}; use crate::settings::tests::helpers::{defaults, resolved}; use crate::settings::tests::test_types::{ResolveErrorTestCase, ResolveTestCase}; @@ -14,8 +16,13 @@ const LAYERED_PROJECT: &[(&str, &str)] = &[ [recall]\nretention = \"10d\"\nconnection = \"history\"\n\ [connections.history]\nadapter = \"sqlite\"\npath = \"state/history.sqlite3\"\n\ [connections.central]\nadapter = \"clickhouse\"\nhost = \"localhost\"\n\ + database = \"observability\"\npassword = \"secret\"\n\ + [connections.other]\nadapter = \"postgres\"\nhost = \"localhost\"\n\ [targets.dev]\n[targets.dev.recall]\nenabled = false\n\ - [targets.prod]\n[targets.prod.recall]\nconnection = \"central\"\n", + [targets.prod]\n[targets.prod.recall]\nconnection = \"central\"\n\ + [targets.capped]\n[targets.capped.recall]\nconnection = \"central\"\n\ + max_output_bytes = 4096\n\ + [targets.qa]\n[targets.qa.recall]\nconnection = \"other\"\n", ), ( "sqlbuild_local.toml", @@ -59,14 +66,28 @@ fn given_project_layers_when_resolved_then_target_over_local_over_project_over_d }, }, ResolveTestCase { - description: "a non-SQLite connection is reported as unsupported", + description: "a ClickHouse connection keeps its options and records all output by default", files: LAYERED_PROJECT, target: Some("prod"), expected_settings: RecallSettings { target: Some("prod".to_owned()), - store: StoreLocation::Unsupported { + store: StoreLocation::ClickHouse { connection: "central".to_owned(), - adapter: "clickhouse".to_owned(), + config: vec![ + ( + "host".to_owned(), + ConfigValue::String("localhost".to_owned()), + ), + ( + "database".to_owned(), + ConfigValue::String("observability".to_owned()), + ), + ( + "password".to_owned(), + ConfigValue::String("secret".to_owned()), + ), + ], + database: "observability".to_owned(), }, connection: Some("central".to_owned()), retention_days: 10, @@ -75,6 +96,37 @@ fn given_project_layers_when_resolved_then_target_over_local_over_project_over_d ..defaults() }, }, + ResolveTestCase { + description: "a configured output limit applies to a ClickHouse store", + files: LAYERED_PROJECT, + target: Some("capped"), + expected_settings: RecallSettings { + target: Some("capped".to_owned()), + store: StoreLocation::ClickHouse { + connection: "central".to_owned(), + config: vec![ + ( + "host".to_owned(), + ConfigValue::String("localhost".to_owned()), + ), + ( + "database".to_owned(), + ConfigValue::String("observability".to_owned()), + ), + ( + "password".to_owned(), + ConfigValue::String("secret".to_owned()), + ), + ], + database: "observability".to_owned(), + }, + connection: Some("central".to_owned()), + retention_days: 10, + shutdown_timeout_ms: 1000, + max_output_bytes: Some(4096), + ..defaults() + }, + }, ResolveTestCase { description: "a target raises the output limit over the project's unlimited one", files: OUTPUT_LIMIT_PROJECT, @@ -95,6 +147,23 @@ fn given_project_layers_when_resolved_then_target_over_local_over_project_over_d ..defaults() }, }, + ResolveTestCase { + description: "a connection on another adapter is reported as unsupported", + files: LAYERED_PROJECT, + target: Some("qa"), + expected_settings: RecallSettings { + target: Some("qa".to_owned()), + store: StoreLocation::Unsupported { + connection: "other".to_owned(), + adapter: "postgres".to_owned(), + }, + connection: Some("other".to_owned()), + retention_days: 10, + shutdown_timeout_ms: 1000, + max_output_bytes: None, + ..defaults() + }, + }, ]; for test_case in test_cases { @@ -128,6 +197,17 @@ fn given_invalid_settings_when_resolved_then_the_error_names_the_setting() { target: Some("dev"), expected_message_fragment: "targets.dev.recall.retention", }, + ResolveErrorTestCase { + description: "an unset environment reference in a ClickHouse connection is named", + files: &[( + "sqlbuild_project.toml", + "name = \"x\"\nadapter = \"duckdb\"\n[recall]\nconnection = \"central\"\n\ + [connections.central]\nadapter = \"clickhouse\"\n\ + password = \"${env:SQLBUILD_RECALL_UNSET_FOR_TEST}\"\n", + )], + target: None, + expected_message_fragment: "connections.central.password references unset", + }, ]; for test_case in test_cases { diff --git a/crates/sqlbuild-recall/src/store/mod.rs b/crates/sqlbuild-recall/src/store/mod.rs index 0126341c03..984a13f051 100644 --- a/crates/sqlbuild-recall/src/store/mod.rs +++ b/crates/sqlbuild-recall/src/store/mod.rs @@ -5,6 +5,7 @@ pub mod constants; pub mod errors; pub mod main; pub mod models; +pub mod types; #[cfg(test)] mod tests; diff --git a/crates/sqlbuild-recall/src/store/types.rs b/crates/sqlbuild-recall/src/store/types.rs new file mode 100644 index 0000000000..8789e007cf --- /dev/null +++ b/crates/sqlbuild-recall/src/store/types.rs @@ -0,0 +1,40 @@ +//! The store interfaces the delivery thread writes through. + +use std::time::{Duration, Instant}; + +use crate::records::models::{RecallRow, RowEnvelope}; +use crate::store::errors::StoreError; +use crate::store::models::{InsertOutcome, MaintenanceRequest}; + +/// An open store the delivery thread alone writes to. +pub trait RecallStore: Send { + /// Write `rows` idempotently by key: an equal duplicate is a no-op; a key already stored with different content keeps the stored row and is reported. Waits end by `give_up`. + fn insert(&self, rows: &[RecallRow], give_up: Instant) -> Result; + + /// Retention work after delivery, within `request.budget`; best effort. + fn maintain(&self, _request: &MaintenanceRequest) {} + + /// Notices for the user gathered since the last call, such as a retention conflict. + fn take_notices(&mut self) -> Vec { + Vec::new() + } + + /// Release the connection; called once when delivery ends. + fn close(&mut self) {} +} + +/// Opens a store hosted outside this crate, such as ClickHouse through its adapter, on the delivery thread. +pub trait RecallStoreOpener: Send + Sync + std::fmt::Debug { + /// Connect and create or upgrade the tables, giving up on waits by `give_up`. A `Busy` error is retried until then. + fn open( + &self, + envelope: &RowEnvelope, + give_up: Instant, + ) -> Result, StoreError>; + + /// Most rows one insert batch takes. + fn batch_rows(&self) -> usize; + + /// Longest wait for a batch to fill once its first record is queued. + fn batch_window(&self) -> Duration; +} diff --git a/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs b/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs index 50bd700728..ff24197b78 100644 --- a/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs +++ b/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs @@ -20,6 +20,7 @@ use crate::store::main::quarantine_store::quarantine_store; use crate::store::models::{ InsertOutcome, MaintenanceRequest, OpenMode, QuarantineOutcome, SqliteStore, StoreObservation, }; +use crate::store::types::{RecallStore, RecallStoreOpener}; use crate::writer::_helpers::materialize::{Built, Materializer}; use crate::writer::_helpers::queue::{Queued, WriterState}; use crate::writer::constants::{ @@ -27,7 +28,7 @@ use crate::writer::constants::{ MAX_RETRY_BACKOFF_MS, MINIMUM_MAINTENANCE_MS, OUTPUT_COALESCE_MS, RUNNING_RETRY_LIMIT_MS, SETUP_RETRY_MS, }; -use crate::writer::models::{DeliveryStream, SessionConfig, SetupOutcome}; +use crate::writer::models::{DeliveryStream, SessionConfig, SessionStore, SetupOutcome}; /// State shared by producers, the session handle and the delivery thread. #[derive(Debug)] @@ -68,18 +69,65 @@ fn bound_busy_wait(store: &SqliteStore, give_up: Instant) { drop(store.connection.busy_timeout(remaining)); } -fn open_and_setup(shared: &Shared, config: &SessionConfig) -> Result { +/// The native SQLite store as the delivery thread writes it. +struct NativeSqlite(SqliteStore); + +impl RecallStore for NativeSqlite { + fn insert(&self, rows: &[RecallRow], give_up: Instant) -> Result { + bound_busy_wait(&self.0, give_up); + let now = format_timestamp(SystemTime::now()); + insert_rows(&self.0, rows, &now) + } + + fn maintain(&self, request: &MaintenanceRequest) { + bound_busy_wait(&self.0, Instant::now() + request.budget); + drop(maintain_sqlite_store(&self.0, request)); + } +} + +fn open_hosted( + shared: &Shared, + config: &SessionConfig, + opener: &dyn RecallStoreOpener, +) -> Result, StoreError> { + let started = Instant::now(); + loop { + match opener.open(&config.envelope, retry_limit(shared, started)) { + Err(error) + if error.kind == StoreErrorKind::Busy + && Instant::now() < retry_limit(shared, started) => + { + std::thread::sleep(Duration::from_millis(SETUP_RETRY_MS)); + } + other => return other, + } + } +} + +fn open_store(shared: &Shared, config: &SessionConfig) -> Result, StoreError> { + match &config.store { + SessionStore::Sqlite(path) => open_and_setup(shared, config, path).map(|store| { + drop(checkpoint_sqlite_store(&store)); + Box::new(NativeSqlite(store)) as Box + }), + SessionStore::Hosted(opener) => open_hosted(shared, config, opener.as_ref()), + } +} + +fn open_and_setup( + shared: &Shared, + config: &SessionConfig, + store_path: &std::path::Path, +) -> Result { let started = Instant::now(); let mut quarantined: bool = false; loop { - let observed: Option = - observe_store_file(&config.store_path).unwrap_or(None); - let attempt = - open_sqlite_store(&config.store_path, OpenMode::ReadWrite).and_then(|store| { - bound_busy_wait(&store, retry_limit(shared, started)); - let now = format_timestamp(SystemTime::now()); - ensure_sqlite_schema(&store, &config.envelope, &now).map(|()| store) - }); + let observed: Option = observe_store_file(store_path).unwrap_or(None); + let attempt = open_sqlite_store(store_path, OpenMode::ReadWrite).and_then(|store| { + bound_busy_wait(&store, retry_limit(shared, started)); + let now = format_timestamp(SystemTime::now()); + ensure_sqlite_schema(&store, &config.envelope, &now).map(|()| store) + }); match attempt { Err(error) if error.kind == StoreErrorKind::Busy @@ -93,7 +141,7 @@ fn open_and_setup(shared: &Shared, config: &SessionConfig) -> Result Result Result Result { let mut backoff = Duration::from_millis(10); loop { - bound_busy_wait(store, give_up); - let now = format_timestamp(SystemTime::now()); - match insert_rows(store, rows, &now) { + match store.insert(rows, give_up) { Err(error) if error.kind == StoreErrorKind::Busy && Instant::now() < give_up => { std::thread::sleep(backoff.min(give_up.saturating_duration_since(Instant::now()))); backoff = (backoff * 2).min(Duration::from_millis(MAX_RETRY_BACKOFF_MS)); @@ -154,8 +200,32 @@ pub(crate) fn stream_of(table: Table) -> DeliveryStream { } } -/// Wait until a batch is due: 500 rows, the batch window, closing, or a timed duty. -fn next_batch(shared: &Shared, next_heartbeat: SystemTime) -> (Vec, bool) { +/// How many rows a batch takes and how long it waits to fill. +#[derive(Clone, Copy)] +struct Batching { + rows: usize, + window: Duration, +} + +fn batching(config: &SessionConfig) -> Batching { + match &config.store { + SessionStore::Sqlite(_) => Batching { + rows: BATCH_ROWS, + window: Duration::from_millis(BATCH_WINDOW_MS), + }, + SessionStore::Hosted(opener) => Batching { + rows: opener.batch_rows(), + window: opener.batch_window(), + }, + } +} + +/// Wait until a batch is due: its row limit, the batch window, closing, or a timed duty. +fn next_batch( + shared: &Shared, + next_heartbeat: SystemTime, + batching: Batching, +) -> (Vec, bool) { let mut state = locked(&shared.state); let mut first_seen: Option = None; loop { @@ -170,10 +240,11 @@ fn next_batch(shared: &Shared, next_heartbeat: SystemTime) -> (Vec, bool if state.queued() > 0 && first_seen.is_none() { first_seen = Some(now); } - let window_over = first_seen - .is_some_and(|seen| now.duration_since(seen) >= Duration::from_millis(BATCH_WINDOW_MS)); - if state.queued() >= BATCH_ROWS || (state.queued() > 0 && (window_over || state.closing)) { - return (state.take_batch(BATCH_ROWS), state.closing); + let window_over = + first_seen.is_some_and(|seen| now.duration_since(seen) >= batching.window); + if state.queued() >= batching.rows || (state.queued() > 0 && (window_over || state.closing)) + { + return (state.take_batch(batching.rows), state.closing); } if state.closing && state.queued() == 0 && !state.has_pending_output() { return (Vec::new(), true); @@ -182,14 +253,14 @@ fn next_batch(shared: &Shared, next_heartbeat: SystemTime) -> (Vec, bool .duration_since(SystemTime::now()) .unwrap_or(Duration::ZERO); if until_heartbeat.is_zero() { - return (Vec::new(), state.closing); + return (state.take_batch(batching.rows), state.closing); } let mut wait_until = now + until_heartbeat.min(Duration::from_millis(CLOCK_CHECK_MS)); if let Some(due) = state.output_due(Duration::from_millis(OUTPUT_COALESCE_MS)) { wait_until = wait_until.min(due); } if let Some(seen) = first_seen { - wait_until = wait_until.min(seen + Duration::from_millis(BATCH_WINDOW_MS)); + wait_until = wait_until.min(seen + batching.window); } let timeout = wait_until .saturating_duration_since(now) @@ -226,7 +297,7 @@ fn final_rows(config: &SessionConfig, state: &WriterState) -> Vec { /// The delivery thread body. pub(crate) fn deliver(shared: &Arc, config: &SessionConfig) { - let store = match open_and_setup(shared, config) { + let mut store: Box = match open_store(shared, config) { Ok(store) => { publish_setup(shared, SetupOutcome::Ready); store @@ -246,11 +317,11 @@ pub(crate) fn deliver(shared: &Arc, config: &SessionConfig) { return; } }; - drop(checkpoint_sqlite_store(&store)); let mut next_heartbeat: SystemTime = SystemTime::now(); let mut materializer = Materializer::new(config.envelope.clone()); + let batching: Batching = batching(config); loop { - let (batch, closing) = next_batch(shared, next_heartbeat); + let (batch, closing) = next_batch(shared, next_heartbeat, batching); let built: Built = materializer.build(batch); if !built.settled.is_empty() || !built.unbuildable.is_empty() { let mut state = locked(&shared.state); @@ -270,7 +341,7 @@ pub(crate) fn deliver(shared: &Arc, config: &SessionConfig) { Instant::now() + Duration::from_millis(RUNNING_RETRY_LIMIT_MS) }) }; - let written = write_batch(&store, &rows, give_up); + let written = write_batch(store.as_ref(), &rows, give_up); let mut state = locked(&shared.state); match &written { Ok(outcome) => state.account(&built.written, Ok(outcome)), @@ -291,7 +362,7 @@ pub(crate) fn deliver(shared: &Arc, config: &SessionConfig) { drop(state); let give_up = (Instant::now() + Duration::from_millis(FINAL_ROWS_BUDGET_MS)) .min(retry_limit(shared, Instant::now())); - if let Err(error) = write_batch(&store, &rows, give_up) { + if let Err(error) = write_batch(store.as_ref(), &rows, give_up) { locked(&shared.state).degraded.get_or_insert(error.message); } break; @@ -301,7 +372,6 @@ pub(crate) fn deliver(shared: &Arc, config: &SessionConfig) { .saturating_duration_since(Instant::now()) .min(Duration::from_millis(MAINTENANCE_BUDGET_MS)); if config.maintenance && remaining >= Duration::from_millis(MINIMUM_MAINTENANCE_MS) { - bound_busy_wait(&store, Instant::now() + remaining); let request = MaintenanceRequest { now: SystemTime::now(), retention_days: config.retention_days, @@ -309,10 +379,13 @@ pub(crate) fn deliver(shared: &Arc, config: &SessionConfig) { budget: remaining, envelope: config.envelope.clone(), }; - drop(maintain_sqlite_store(&store, &request)); + store.maintain(&request); } + let notices: Vec = store.take_notices(); + store.close(); drop(store); let mut state = locked(&shared.state); + state.notices.extend(notices); state.finished = true; shared.wake.notify_all(); } diff --git a/crates/sqlbuild-recall/src/writer/models.rs b/crates/sqlbuild-recall/src/writer/models.rs index 4b77260ae5..6e53198889 100644 --- a/crates/sqlbuild-recall/src/writer/models.rs +++ b/crates/sqlbuild-recall/src/writer/models.rs @@ -6,6 +6,7 @@ use std::thread::JoinHandle; use std::time::Duration; use crate::records::models::{DeliveryCounts, InvocationContext, RowEnvelope}; +use crate::store::types::RecallStoreOpener; use crate::writer::_helpers::delivery::Shared; /// The datasets whose delivery is accounted separately. @@ -67,10 +68,19 @@ impl Priority { pub const COUNT: usize = 8; } +/// Where a session writes. +#[derive(Clone, Debug)] +pub enum SessionStore { + /// The native SQLite store at this path. + Sqlite(PathBuf), + /// A store opened through `RecallStoreOpener`. + Hosted(Arc), +} + /// What one writer session needs. -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug)] pub struct SessionConfig { - pub store_path: PathBuf, + pub store: SessionStore, pub envelope: RowEnvelope, pub context: InvocationContext, pub retention_days: u32, diff --git a/crates/sqlbuild-recall/src/writer/tests/helpers.rs b/crates/sqlbuild-recall/src/writer/tests/helpers.rs index eddddbd71d..9c4370107e 100644 --- a/crates/sqlbuild-recall/src/writer/tests/helpers.rs +++ b/crates/sqlbuild-recall/src/writer/tests/helpers.rs @@ -12,7 +12,7 @@ use crate::store::models::OpenMode; use crate::writer::_helpers::queue::WriterState; use crate::writer::main::start_recall_session::start_recall_session; use crate::writer::models::{ - DeliveryStream, DeliverySummary, Priority, SessionConfig, SetupOutcome, + DeliveryStream, DeliverySummary, Priority, SessionConfig, SessionStore, SetupOutcome, }; use crate::writer::tests::test_types::RecordedSession; @@ -73,7 +73,7 @@ pub(crate) fn chunked(writes: &[(OutputStream, Vec)]) -> Vec<[Option SessionConfig { SessionConfig { - store_path: path, + store: SessionStore::Sqlite(path), envelope: envelope(invocation_id), context: InvocationContext { command: Some("build".to_owned()), @@ -321,3 +321,23 @@ pub(crate) fn chunk_facts( })), ] } + +/// Output chunks stored while a session still runs, when heartbeats fall due faster than batches fill. +pub(crate) fn output_stored_while_running( + heartbeat: Duration, + wait: Duration, +) -> Vec> { + let directory = tempfile::tempdir().expect("a temporary directory can be created"); + let path = directory.path().join("logs/recall.sqlite3"); + let session = start_recall_session(SessionConfig { + heartbeat_interval: heartbeat, + ..config(path.clone(), "beating") + }); + session.append_output(OutputStream::Stdout, b"progress\n"); + std::thread::sleep(wait); + let store = open_sqlite_store(&path, OpenMode::ReadOnly).expect("the store opens"); + let stored = query_store(&store, "SELECT COUNT(*) FROM recall_output", None) + .expect("the query is valid"); + drop(session.close()); + stored.rows +} diff --git a/crates/sqlbuild-recall/src/writer/tests/test_session.rs b/crates/sqlbuild-recall/src/writer/tests/test_session.rs index 4bb2d460aa..f5e59501ce 100644 --- a/crates/sqlbuild-recall/src/writer/tests/test_session.rs +++ b/crates/sqlbuild-recall/src/writer/tests/test_session.rs @@ -6,14 +6,14 @@ use crate::writer::models::SetupOutcome; #[cfg(unix)] use crate::writer::tests::helpers::unwritable_session; use crate::writer::tests::helpers::{ - artifact_counts, close_under_held_lock, event_counts, recorded_session, - session_on_corrupt_store, + artifact_counts, close_under_held_lock, event_counts, output_stored_while_running, + recorded_session, session_on_corrupt_store, }; #[cfg(unix)] use crate::writer::tests::test_types::UnwritableTestCase; use crate::writer::tests::test_types::{ ArtifactSizeTestCase, CorruptStoreTestCase, EventBuildTestCase, HeldLockTestCase, - RecordedSession, SessionTestCase, + RecordedSession, SessionTestCase, StoredWhileRunningTestCase, }; const CLOSE_SLACK_SECONDS: f64 = 0.3; @@ -185,3 +185,23 @@ fn given_events_when_built_on_the_delivery_thread_then_unbuildable_ones_count_as ); } } + +#[test] +fn given_heartbeats_due_faster_than_batches_fill_when_running_then_queued_records_are_still_written() + { + let test_cases = [StoredWhileRunningTestCase { + description: "a 50 ms heartbeat still lets queued output reach the store", + heartbeat: Duration::from_millis(50), + wait: Duration::from_millis(1_500), + expected_rows: vec![vec![RecallValue::Integer(1)]], + }]; + + for test_case in test_cases { + assert_eq!( + output_stored_while_running(test_case.heartbeat, test_case.wait), + test_case.expected_rows, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-recall/src/writer/tests/test_types.rs b/crates/sqlbuild-recall/src/writer/tests/test_types.rs index 3aaa2c7048..a1c8ea19f5 100644 --- a/crates/sqlbuild-recall/src/writer/tests/test_types.rs +++ b/crates/sqlbuild-recall/src/writer/tests/test_types.rs @@ -76,3 +76,10 @@ pub(crate) struct CaptureLimitTestCase { pub(crate) writes: Vec<(OutputStream, &'static [u8])>, pub(crate) expected_rows: Vec<[Option; 6]>, } + +pub(crate) struct StoredWhileRunningTestCase { + pub(crate) description: &'static str, + pub(crate) heartbeat: std::time::Duration, + pub(crate) wait: std::time::Duration, + pub(crate) expected_rows: Vec>, +} diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/recall.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/recall.md index 0abc6ee443..c243f37b4e 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/recall.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/recall.md @@ -27,8 +27,9 @@ sqb recall output last ## sqb recall query -Run one read-only SQLite `SELECT` against the Recall views. Statements that write, and multiple -statements, are rejected. +Run one read-only `SELECT` against the Recall views, in the store's dialect: SQLite for the local +store, ClickHouse for a [ClickHouse store](../concepts/observability/recall.md#sharing-history-in-clickhouse). +Statements that write, and multiple statements, are rejected. | Flag | Description | |------|-------------| diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md index eb4a8a3705..72aad75335 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md @@ -13,11 +13,12 @@ Online: https://sqlbuild.com/docs/concepts/observability/recall/ - Output limit - Retention - When the store cannot be written +- Sharing history in ClickHouse - Querying history - Relationship to compute logs -Recall records each `sqb` invocation in a local SQLite store so you can ask what happened after the -fact: which command ran, how it ended, which statements it executed and how long they took, and +Recall records each `sqb` invocation in a local SQLite store, or in a shared +[ClickHouse store](#sharing-history-in-clickhouse), so you can ask what happened after the fact: which command ran, how it ended, which statements it executed and how long they took, and exactly what it printed. Recall is on by default and needs no setup. ## What is recorded @@ -64,7 +65,7 @@ enabled = false # do not record runs against the ci target | Key | Default | Description | |-----|---------|-------------| | `enabled` | `true` | Set to `false` to stop recording | -| `connection` | (none) | A `sqlite` connection whose `path` names the store; defaults to `logs/recall.sqlite3` | +| `connection` | (none) | A `sqlite` connection whose `path` names the store, or a `clickhouse` connection; defaults to `logs/recall.sqlite3` | | `retention` | `"30d"` | How long invocations are kept, in days (`d`) or hours (`h`), up to 3650 days | | `shutdown_timeout` | `"3s"` | The cap on time spent flushing history at exit, in `s` or `ms`, up to 10 minutes | | `diagnostics_min_severity` | `"info"` | `trace`, `debug`, `info`, `warning`, `error`, or `critical` | @@ -115,13 +116,53 @@ repeats at most once a day. If the store file is not a SQLite database, Recall m `recall.sqlite3.corrupt-`, starts a fresh store, and says so once. Commands that find the same corrupt file coordinate through `recall.sqlite3.lock`, so only one of them moves it. -A `connection` that does not use the `sqlite` adapter is not supported yet; Recall prints a notice -and records nothing. +A `connection` on any adapter other than `sqlite` or `clickhouse` is not supported; Recall prints a +notice and records nothing. + +## Sharing history in ClickHouse + +Point `connection` at a ClickHouse connection to keep one history for a whole team. The connection +needs `adapter = "clickhouse"` when the project uses another adapter, and its `database` must +already exist; Recall never creates databases. + +```toml +[connections.history] +adapter = "clickhouse" +host = "${env:CLICKHOUSE_HOST}" +user = "recall_writer" +password = "${env:CLICKHOUSE_PASSWORD}" +database = "observability" + +[recall] +connection = "history" +retention = "30d" +``` + +- **Setup.** On first use Recall creates its `recall_*` MergeTree tables and the views above in that + database, and adds columns that a newer SQLBuild introduces. After that, writing needs only + `INSERT` and `SELECT` on the tables. If the user may not create them, the command still succeeds + and prints the complete DDL for an administrator to run. +- **Writes.** History is sent in batches of up to 5,000 rows about once a second, one insert per + table, with the same queue, shutdown budget and exit-code guarantees as the local store. Many + concurrent commands can write one store. +- **Duplicates.** Every row has a deterministic key. A retried batch is deduplicated by ClickHouse, + and the views keep the first copy of each key, so a repeated delivery is counted once. A key + written again with different content keeps the first row and is listed in `conflicts`. +- **Retention.** Every table carries a TTL of `retention` (SQL text a week longer, and refreshed + while it is still referenced), and the views hide rows past `retention` at once. ClickHouse drops + expired daily parts in the background, so rows may stay on disk for a while after the views stop + showing them. Retention belongs to the store: when a project's `retention` differs from the + tables' TTL, Recall warns once a day with the `ALTER TABLE … MODIFY TTL` and view statements that + would apply it, and never changes the TTL itself. +- **Reading.** `sqb recall query`, `schema` and `output` read the ClickHouse store, so queries use + ClickHouse SQL. `--json` output includes the server's `rows_read`, `bytes_read` and `query_id`. + +Use a single ClickHouse server or ClickHouse Cloud; `ON CLUSTER` stores are not supported yet. ## Querying history -Use [`sqb recall`](../../cli/recall.md) to read the store. Queries are read-only SQLite `SELECT` -statements against these views: +Use [`sqb recall`](../../cli/recall.md) to read the store. Queries are read-only `SELECT` statements, +in the store's SQL dialect, against these views: | View | One row per | |------|-------------| diff --git a/src/sqlbuild/_native_host.py b/src/sqlbuild/_native_host.py index 634ab0f4fe..a7fd8fdeb1 100644 --- a/src/sqlbuild/_native_host.py +++ b/src/sqlbuild/_native_host.py @@ -98,6 +98,23 @@ "sqlbuild.adapter.type_system.main.safe_cast_nullability", "safe_cast_nullability", ), + "connect_recall_store": ( + "sqlbuild.adapters.clickhouse._helpers.recall_store", + "connect_recall_store", + ), + "run_recall_query": ("sqlbuild.adapters.clickhouse._helpers.recall_store", "run_recall_query"), + "run_recall_command": ( + "sqlbuild.adapters.clickhouse._helpers.recall_store", + "run_recall_command", + ), + "insert_recall_rows": ( + "sqlbuild.adapters.clickhouse._helpers.recall_store", + "insert_recall_rows", + ), + "close_recall_store": ( + "sqlbuild.adapters.clickhouse._helpers.recall_store", + "close_recall_store", + ), "python_resource_rows": ( "sqlbuild.compiler.compile._helpers.diagnostics.python_resources", "python_resource_rows", diff --git a/src/sqlbuild/_native_host.pyi b/src/sqlbuild/_native_host.pyi index 08c7f6dee1..3f51f2462b 100644 --- a/src/sqlbuild/_native_host.pyi +++ b/src/sqlbuild/_native_host.pyi @@ -75,6 +75,19 @@ from sqlbuild.adapter.type_system.main.normalize_type import normalize_type as n from sqlbuild.adapter.type_system.main.safe_cast_nullability import ( safe_cast_nullability as safe_cast_nullability, ) +from sqlbuild.adapters.clickhouse._helpers.recall_store import ( + connect_recall_store as connect_recall_store, +) +from sqlbuild.adapters.clickhouse._helpers.recall_store import run_recall_query as run_recall_query +from sqlbuild.adapters.clickhouse._helpers.recall_store import ( + run_recall_command as run_recall_command, +) +from sqlbuild.adapters.clickhouse._helpers.recall_store import ( + insert_recall_rows as insert_recall_rows, +) +from sqlbuild.adapters.clickhouse._helpers.recall_store import ( + close_recall_store as close_recall_store, +) from sqlbuild.cli.commands.main.entrypoint._dispatch_with_compute_logs import ( dispatch_with_compute_logs as dispatch_with_compute_logs, ) diff --git a/src/sqlbuild/adapters/clickhouse/_helpers/recall_store.py b/src/sqlbuild/adapters/clickhouse/_helpers/recall_store.py new file mode 100644 index 0000000000..e7182cbfc6 --- /dev/null +++ b/src/sqlbuild/adapters/clickhouse/_helpers/recall_store.py @@ -0,0 +1,136 @@ +"""Recall store operations on ClickHouse connections, reached by native code through handles.""" + +from __future__ import annotations + +import datetime +import json +import threading +from typing import Any + +from sqlbuild.adapters.clickhouse.classes.clickhouse_adapter import ClickHouseAdapter +from sqlbuild.adapters.clickhouse.classes.clickhouse_connection import _ClickHouseConnection +from sqlbuild.adapters.clickhouse.types import RecallCell, RecallFailure, RecallStatistics + +_CLIENT_MODULE: str = "clickhouse_connect" +_CONNECT_TIMEOUT_KEY: str = "connect_timeout" +_DEFAULT_CONNECT_TIMEOUT_SECONDS: int = 3 +_TIMESTAMP_FORMAT: str = "%Y-%m-%dT%H:%M:%S.%fZ" +_BYTES_FORMATS: dict[str, str] = {"String": "bytes", "FixedString": "bytes"} +_ROW_BINARY: str = "RowBinary" +_INT64_MIN: int = -(2**63) +_INT64_MAX: int = 2**63 - 1 + + +def _failure(error: Exception) -> tuple[bool, str]: + return (type(error).__module__.startswith(_CLIENT_MODULE), str(error)) + + +def _cell(value: object) -> RecallCell: + if isinstance(value, bool): + return int(value) + if isinstance(value, int): + return value if _INT64_MIN <= value <= _INT64_MAX else str(value) + if value is None or isinstance(value, (float, str)): + return value + if isinstance(value, bytes): + try: + return value.decode("utf-8") + except UnicodeDecodeError: + return value + if isinstance(value, datetime.datetime): + return value.strftime(_TIMESTAMP_FORMAT) + if isinstance(value, (list, tuple, dict)): + return json.dumps(value, default=_text) + return str(value) + + +def _text(value: object) -> object: + cell: RecallCell = _cell(value) + return cell.hex() if isinstance(cell, bytes) else cell + + +def _row(row: Any) -> list[RecallCell]: + return [_cell(value) for value in row] + + +def _number(*, summary: dict[str, Any], key: str) -> int | None: + value: object = summary.get(key) + return int(str(value)) if value is not None else None + + +def connect_recall_store( + *, config: dict[str, object], quiet_thread_attribute: str +) -> tuple[RecallFailure, int]: + """Connect for Recall with a short connect timeout and return the connection's handle.""" + + setattr(threading.current_thread(), quiet_thread_attribute, True) + options: dict[str, object] = {_CONNECT_TIMEOUT_KEY: _DEFAULT_CONNECT_TIMEOUT_SECONDS, **config} + try: + adapter: ClickHouseAdapter = ClickHouseAdapter() + connection: _ClickHouseConnection = adapter.connect(options) + except Exception as error: + return _failure(error), 0 + return None, adapter.recall_connections.add(connection=connection) + + +def run_recall_query( + *, handle: int, sql: str, settings: dict[str, int] +) -> tuple[RecallFailure, tuple[list[str], list[list[RecallCell]], RecallStatistics] | None]: + """Columns, rows of plain values and server statistics; strings not in UTF-8 stay bytes.""" + + try: + result: Any = ClickHouseAdapter.recall_connections.get(handle=handle).client.query( + sql, settings=settings, query_formats=_BYTES_FORMATS + ) + except Exception as error: + return _failure(error), None + summary: dict[str, Any] = dict(result.summary or {}) + statistics: RecallStatistics = ( + _number(summary=summary, key="read_rows"), + _number(summary=summary, key="read_bytes"), + summary.get("query_id"), + ) + rows: list[list[RecallCell]] = [_row(row) for row in result.result_rows] + return None, (list(result.column_names), rows, statistics) + + +def run_recall_command(*, handle: int, sql: str) -> tuple[RecallFailure, None]: + """Run one DDL statement.""" + + try: + ClickHouseAdapter.recall_connections.get(handle=handle).client.command(sql) + except Exception as error: + return _failure(error), None + return None, None + + +def insert_recall_rows( + *, handle: int, table: str, column_names: list[str], payload: bytes, deduplication_token: str +) -> tuple[RecallFailure, None]: + """Insert one `RowBinary` block; a retried block with the same token is deduplicated.""" + + try: + ClickHouseAdapter.recall_connections.get(handle=handle).client.raw_insert( + table, + column_names=column_names, + insert_block=payload, + settings={"insert_deduplication_token": deduplication_token}, + fmt=_ROW_BINARY, + ) + except Exception as error: + return _failure(error), None + return None, None + + +def close_recall_store(*, handle: int) -> None: + """Close and forget the connection; a failure to close is ignored.""" + + connection: _ClickHouseConnection | None = ClickHouseAdapter.recall_connections.pop( + handle=handle + ) + if connection is None: + return + try: + connection.close() + except Exception: + return diff --git a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py index bcf53d1924..158f0bf427 100644 --- a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py +++ b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py @@ -108,6 +108,7 @@ ) from sqlbuild.adapters.clickhouse.classes.clickhouse_connection import _ClickHouseConnection from sqlbuild.adapters.clickhouse.classes.partition_locks import PartitionLocks +from sqlbuild.adapters.clickhouse.classes.recall_connections import RecallConnections from sqlbuild.adapters.clickhouse.classes.relation_write_locks import RelationWriteLocks from sqlbuild.adapters.clickhouse.constants import ( CONNECTION_OPTION_KEYS, @@ -181,6 +182,7 @@ class ClickHouseAdapter(MicrobatchMixin, UnkeyedDiffMixin, BaseAdapter): ) _partition_locks: ClassVar[PartitionLocks] = PartitionLocks() _relation_locks: ClassVar[RelationWriteLocks] = RelationWriteLocks() + recall_connections: ClassVar[RecallConnections] = RecallConnections() def connect(self, config: dict[str, Any]) -> _ClickHouseConnection: try: diff --git a/src/sqlbuild/adapters/clickhouse/classes/recall_connections.py b/src/sqlbuild/adapters/clickhouse/classes/recall_connections.py new file mode 100644 index 0000000000..0eeea1f566 --- /dev/null +++ b/src/sqlbuild/adapters/clickhouse/classes/recall_connections.py @@ -0,0 +1,37 @@ +"""Open Recall store connections, kept in Python and named to native code by integer handles.""" + +from __future__ import annotations + +import itertools +import threading +from collections.abc import Iterator + +from sqlbuild.adapters.clickhouse.classes.clickhouse_connection import _ClickHouseConnection + + +class RecallConnections: + """Hand out a handle per open connection and look connections up by handle.""" + + def __init__(self) -> None: + self._guard: threading.Lock = threading.Lock() + self._handles: Iterator[int] = itertools.count(1) + self._connections: dict[int, _ClickHouseConnection] = {} + + def add(self, *, connection: _ClickHouseConnection) -> int: + """Keep ``connection`` and return its new handle.""" + + with self._guard: + handle: int = next(self._handles) + self._connections[handle] = connection + return handle + + def get(self, *, handle: int) -> _ClickHouseConnection: + """The connection ``handle`` names.""" + + return self._connections[handle] + + def pop(self, *, handle: int) -> _ClickHouseConnection | None: + """Forget ``handle`` and return its connection, if it is still open.""" + + with self._guard: + return self._connections.pop(handle, None) diff --git a/src/sqlbuild/adapters/clickhouse/types.py b/src/sqlbuild/adapters/clickhouse/types.py new file mode 100644 index 0000000000..9281aad5de --- /dev/null +++ b/src/sqlbuild/adapters/clickhouse/types.py @@ -0,0 +1,5 @@ +"""ClickHouse adapter type declarations.""" + +type RecallFailure = tuple[bool, str] | None +type RecallCell = None | int | float | str | bytes +type RecallStatistics = tuple[int | None, int | None, str | None] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py index f46daea1cb..166d882b71 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py @@ -95,3 +95,62 @@ class ClickHouseOutputColumnNameE2ETestCase: expected_return_code: int expected_output_fragments: tuple[str, ...] expected_columns: tuple[tuple[object, ...], ...] + + +@dataclass(frozen=True) +class ClickHouseRecallRetentionTestCase: + description: str + recall_settings: str + expected_ttl_days: tuple[tuple[str, int], ...] + + +@dataclass(frozen=True) +class ClickHouseRecallExpiryTestCase: + description: str + created_with: str + recall_settings: str + aged_days: int + expected_fix_statements: int + + +@dataclass(frozen=True) +class ClickHouseRecallDuplicateTestCase: + description: str + view_counts_sql: str + expected_physical_factor: int + + +@dataclass(frozen=True) +class ClickHouseRecallPermissionTestCase: + description: str + connection_settings: str + expected_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class ClickHouseRecallConcurrencyTestCase: + description: str + targets: tuple[str, ...] + expected_status: str + + +@dataclass(frozen=True) +class ClickHouseRecallAbandonedTestCase: + description: str + heartbeat_seconds: str + expected_status: str + + +@dataclass(frozen=True) +class ClickHouseRecallOutputTestCase: + description: str + args: tuple[str, ...] + expected_stream: str + + +@dataclass(frozen=True) +class ClickHouseRecallRetentionConflictTestCase: + description: str + created_with: str + built_with: str + expected_fragments: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/conftest.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/conftest.py index 531933d58e..359deff916 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/conftest.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/conftest.py @@ -6,7 +6,11 @@ import pytest -from tests.e2e.src.sqlbuild.cli.commands.main.clickhouse.helpers import CLICKHOUSE_IMAGE +from tests.e2e.src.sqlbuild.cli.commands.main.clickhouse.helpers import ( + CLICKHOUSE_IMAGE, + build_unique_schema_name, + execute_clickhouse_sql, +) def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: @@ -41,3 +45,15 @@ def clickhouse_e2e_config() -> Iterator[dict[str, object]]: } except Exception as exc: pytest.skip(f"Docker not available for ClickHouse testcontainer: {exc}") + + +@pytest.fixture +def recall_database(clickhouse_e2e_config: dict[str, object]) -> Iterator[str]: + database: str = build_unique_schema_name(prefix="recall") + execute_clickhouse_sql( + config=clickhouse_e2e_config, statements=(f"CREATE DATABASE {database}",) + ) + yield database + execute_clickhouse_sql( + config=clickhouse_e2e_config, statements=(f"DROP DATABASE IF EXISTS {database} SYNC",) + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py index b74d2dfc8c..91d1a61d15 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py @@ -10,6 +10,7 @@ from sqlbuild.adapters.clickhouse.classes.clickhouse_adapter import ClickHouseAdapter from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_waffle_shop +from tests.integration.src.sqlbuild.cli.recall.helpers import write_project CLICKHOUSE_IMAGE: str = "clickhouse/clickhouse-server:26.1" @@ -311,3 +312,166 @@ def _alias_qualified_columns(model_path: Path) -> None: ), encoding="utf-8", ) + + +RECALL_TABLES: tuple[str, ...] = ( + "recall_meta", + "recall_invocations", + "recall_output", + "recall_events", + "recall_diagnostics", + "recall_artifacts", + "recall_artifact_refs", + "recall_heartbeats", + "recall_delivery", + "recall_conflicts", +) +_TTL_DAYS: re.Pattern[str] = re.compile(r" TTL \w+ \+ toIntervalDay\((\d+)\)") +_AGED_TABLES: tuple[tuple[str, str], ...] = ( + ("recall_invocations", ""), + ("recall_output", ""), + ("recall_events", ", concat('aged-', event_id) AS event_id"), + ("recall_diagnostics", ""), + ("recall_artifact_refs", ""), + ("recall_heartbeats", ""), + ("recall_delivery", ""), + ("recall_conflicts", ""), +) + + +def recall_store_toml( + *, + config: dict[str, object], + database: str, + recall_settings: str = "", + connection_settings: str = "", +) -> str: + """Project configuration sending Recall history to a ClickHouse database.""" + + return ( + "\n[connections.history]\n" + 'adapter = "clickhouse"\n' + f'host = "{config["host"]}"\n' + f"port = {config['port']}\n" + f'user = "{config["user"]}"\n' + f'password = "{config["password"]}"\n' + f'database = "{database}"\n' + f"{connection_settings}" + "\n[recall]\n" + 'connection = "history"\n' + f"{recall_settings}" + ) + + +def stored_ttl_days(*, config: dict[str, object], database: str) -> tuple[tuple[str, int], ...]: + """Each Recall table's TTL in days, by table name.""" + + rows: tuple[tuple[object, ...], ...] = fetch_clickhouse_rows( + config=config, + sql=( + "SELECT name, create_table_query FROM system.tables " + f"WHERE database = '{database}' AND startsWith(name, 'recall_') ORDER BY name" + ), + ) + return tuple( + (str(name), int(next(iter(_TTL_DAYS.findall(str(query))), -1))) for name, query in rows + ) + + +def physical_rows(*, config: dict[str, object], database: str, where: str = "1") -> int: + """Rows stored across every Recall table, before any deduplication or retention filter.""" + + total: int = 0 + for table in RECALL_TABLES: + rows: tuple[tuple[object, ...], ...] = fetch_clickhouse_rows( + config=config, sql=f"SELECT count() FROM {database}.{table} WHERE {where}" + ) + total += int(cast(int, rows[0][0])) + return total + + +def age_recorded_invocations(*, config: dict[str, object], database: str, days: int) -> None: + """Store a copy of every invocation's rows started `days` earlier, under `aged-` IDs.""" + + statements: list[str] = [] + for table, extra in _AGED_TABLES: + replaced: str = ( + "concat('aged-', invocation_id) AS invocation_id, " + f"invocation_started_at - INTERVAL {days} DAY AS invocation_started_at{extra}" + ) + statements.append( + f"INSERT INTO {database}.{table} SELECT * REPLACE ({replaced}) FROM " + f"(SELECT * FROM {database}.{table} WHERE NOT startsWith(invocation_id, 'aged-'))" + ) + execute_clickhouse_sql(config=config, statements=tuple(statements)) + + +def expire_recall_parts(*, config: dict[str, object], database: str) -> None: + """Apply every Recall table's TTL now, as background TTL merges eventually would.""" + + execute_clickhouse_sql( + config=config, + statements=tuple( + f"ALTER TABLE {database}.{table} MATERIALIZE TTL SETTINGS mutations_sync = 2" + for table in RECALL_TABLES + ), + ) + + +def apply_printed_retention_fix(*, config: dict[str, object], stderr: str) -> int: + """Run the statements a retention notice printed without expiring stored parts.""" + + script: str = stderr.split("configured retention with:\n\n", 1)[1].split("\nsqb ", 1)[0] + statements: list[str] = [ + statement.strip().removesuffix(";") for statement in script.split(";\n\n") + ] + import clickhouse_connect + + client: Any = clickhouse_connect.get_client( + host=str(config["host"]), + port=int(cast(int, config["port"])), + username=str(config["user"]), + password=str(config["password"]), + ) + try: + for statement in statements: + client.command(statement, settings={"materialize_ttl_after_modify": 0}) + finally: + client.close() + return len(statements) + + +def duplicate_recall_rows(*, config: dict[str, object], database: str) -> None: + """Deliver every stored Recall row a second time, as a retried insert would.""" + + execute_clickhouse_sql( + config=config, + statements=tuple( + f"INSERT INTO {database}.{table} SETTINGS insert_deduplicate = 0 " + f"SELECT * FROM {database}.{table}" + for table in RECALL_TABLES + ), + ) + + +def write_recall_project( + *, + parent: Path, + config: dict[str, object], + database: str, + settings: tuple[str, str] = ("", ""), + targets: int = 1, +) -> Path: + """Write the local Recall test project, sending its history to `database`.""" + + recall_settings, connection_settings = settings + return write_project( + parent / "orders_shop", + extra_toml=recall_store_toml( + config=config, + database=database, + recall_settings=recall_settings, + connection_settings=connection_settings, + ), + targets=targets, + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py new file mode 100644 index 0000000000..ea468cc3d8 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py @@ -0,0 +1,425 @@ +from __future__ import annotations + +import json +import subprocess +from pathlib import Path +from typing import cast + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.clickhouse._test_types import ( + ClickHouseRecallAbandonedTestCase, + ClickHouseRecallConcurrencyTestCase, + ClickHouseRecallDuplicateTestCase, + ClickHouseRecallExpiryTestCase, + ClickHouseRecallOutputTestCase, + ClickHouseRecallPermissionTestCase, + ClickHouseRecallRetentionConflictTestCase, + ClickHouseRecallRetentionTestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.clickhouse.helpers import ( + age_recorded_invocations, + apply_printed_retention_fix, + duplicate_recall_rows, + expire_recall_parts, + fetch_clickhouse_rows, + physical_rows, + stored_ttl_days, + write_recall_project, +) +from tests.integration.src.sqlbuild.cli.recall.helpers import ( + concurrent_builds, + killed_build, + recall_rows, + run, +) + + +@pytest.mark.parametrize( + "test_case", + [ + ClickHouseRecallRetentionTestCase( + description="the default keeps 30 days and artifacts a week longer", + recall_settings="", + expected_ttl_days=( + ("recall_artifact_refs", 30), + ("recall_artifacts", 37), + ("recall_conflicts", 30), + ("recall_delivery", 30), + ("recall_diagnostics", 30), + ("recall_events", 30), + ("recall_heartbeats", 30), + ("recall_invocations", 30), + ("recall_meta", 30), + ("recall_output", 30), + ), + ), + ClickHouseRecallRetentionTestCase( + description="a configured retention sets every table's TTL", + recall_settings='retention = "7d"\n', + expected_ttl_days=( + ("recall_artifact_refs", 7), + ("recall_artifacts", 14), + ("recall_conflicts", 7), + ("recall_delivery", 7), + ("recall_diagnostics", 7), + ("recall_events", 7), + ("recall_heartbeats", 7), + ("recall_invocations", 7), + ("recall_meta", 7), + ("recall_output", 7), + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_retention_settings_when_building_then_recall_tables_expire_by_matching_ttls( + test_case: ClickHouseRecallRetentionTestCase, + tmp_path: Path, + clickhouse_e2e_config: dict[str, object], + recall_database: str, +) -> None: + project_dir: Path = write_recall_project( + parent=tmp_path, + config=clickhouse_e2e_config, + database=recall_database, + settings=(test_case.recall_settings, ""), + ) + + built: subprocess.CompletedProcess[bytes] = run(project_dir, "build") + + assert built.returncode == 0, built.stderr + assert b"Recall" not in built.stderr + assert stored_ttl_days(config=clickhouse_e2e_config, database=recall_database) == ( + test_case.expected_ttl_days + ) + assert recall_rows(project_dir, "SELECT status, exit_code FROM invocations") == [ + ["completed", 0] + ] + + +@pytest.mark.parametrize( + "test_case", + [ + ClickHouseRecallExpiryTestCase( + description="a shortened retention hides old runs at once and TTL drops them later", + created_with="", + recall_settings='retention = "7d"\n', + aged_days=10, + expected_fix_statements=20, + ), + ], + ids=lambda case: case.description, +) +def test_given_rows_past_retention_when_queried_then_views_hide_them_before_ttl_drops_them( + test_case: ClickHouseRecallExpiryTestCase, + tmp_path: Path, + clickhouse_e2e_config: dict[str, object], + recall_database: str, +) -> None: + creator: Path = write_recall_project( + parent=tmp_path / "creator", + config=clickhouse_e2e_config, + database=recall_database, + settings=(test_case.created_with, ""), + ) + assert run(creator, "build").returncode == 0 + age_recorded_invocations( + config=clickhouse_e2e_config, database=recall_database, days=test_case.aged_days + ) + project_dir: Path = write_recall_project( + parent=tmp_path, + config=clickhouse_e2e_config, + database=recall_database, + settings=(test_case.recall_settings, ""), + ) + aged: str = "startsWith(invocation_id, 'aged-')" + visible_sql: str = ( + f"SELECT (SELECT count() FROM invocations WHERE {aged}) " + f"+ (SELECT count() FROM events WHERE {aged}) " + f"+ (SELECT count() FROM output_transcripts WHERE {aged})" + ) + visible_before: list[list[object]] = recall_rows(project_dir, visible_sql) + built: subprocess.CompletedProcess[bytes] = run(project_dir, "build") + + applied: int = apply_printed_retention_fix( + config=clickhouse_e2e_config, stderr=built.stderr.decode(errors="replace") + ) + visible_after: list[list[object]] = recall_rows(project_dir, visible_sql) + stored_before_ttl: int = physical_rows( + config=clickhouse_e2e_config, database=recall_database, where=aged + ) + expire_recall_parts(config=clickhouse_e2e_config, database=recall_database) + stored_after_ttl: int = physical_rows( + config=clickhouse_e2e_config, database=recall_database, where=aged + ) + + assert built.returncode == 0 + assert applied == test_case.expected_fix_statements + assert int(cast(int, visible_before[0][0])) > 0 + assert visible_after == [[0]] + assert stored_before_ttl > 0 + assert stored_after_ttl == 0 + assert recall_rows(project_dir, "SELECT count() FROM invocations") == [[2]] + + +@pytest.mark.parametrize( + "test_case", + [ + ClickHouseRecallDuplicateTestCase( + description="a second delivery of every row changes no view", + view_counts_sql=( + "SELECT (SELECT count() FROM invocations), (SELECT count() FROM statements), " + "(SELECT count() FROM events), (SELECT count() FROM diagnostics), " + "(SELECT count() FROM artifacts), (SELECT sum(chunks) FROM output_invocations), " + "(SELECT min(complete) FROM output_invocations), " + "(SELECT count() FROM conflicts)" + ), + expected_physical_factor=2, + ), + ], + ids=lambda case: case.description, +) +def test_given_duplicate_delivery_when_queried_then_views_count_each_record_once( + test_case: ClickHouseRecallDuplicateTestCase, + tmp_path: Path, + clickhouse_e2e_config: dict[str, object], + recall_database: str, +) -> None: + project_dir: Path = write_recall_project( + parent=tmp_path, config=clickhouse_e2e_config, database=recall_database + ) + built: subprocess.CompletedProcess[bytes] = run(project_dir, "build") + before: list[list[object]] = recall_rows(project_dir, test_case.view_counts_sql) + stored_before: int = physical_rows(config=clickhouse_e2e_config, database=recall_database) + + duplicate_recall_rows(config=clickhouse_e2e_config, database=recall_database) + + assert built.returncode == 0, built.stderr + assert recall_rows(project_dir, test_case.view_counts_sql) == before + assert before[0][-2:] == [1, 0] + assert physical_rows(config=clickhouse_e2e_config, database=recall_database) == ( + stored_before * test_case.expected_physical_factor + ) + assert run(project_dir, "recall", "output", "last", "--stream", "stdout").stdout == ( + built.stdout + ) + + +@pytest.mark.parametrize( + "test_case", + [ + ClickHouseRecallPermissionTestCase( + description="a read-only user gets the administrator's DDL and a successful build", + connection_settings="settings = { readonly = 2 }\n", + expected_fragments=( + "Recall history is not being written", + "Cannot execute query in readonly mode", + "CREATE TABLE IF NOT EXISTS `{database}`.`recall_meta`", + "CREATE OR REPLACE VIEW `{database}`.`invocations`", + "completed successfully; only its Recall history is affected", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_store_without_create_permission_when_building_then_ddl_is_printed_and_exit_is_zero( + test_case: ClickHouseRecallPermissionTestCase, + tmp_path: Path, + clickhouse_e2e_config: dict[str, object], + recall_database: str, +) -> None: + project_dir: Path = write_recall_project( + parent=tmp_path, + config=clickhouse_e2e_config, + database=recall_database, + settings=("", test_case.connection_settings), + ) + + built: subprocess.CompletedProcess[bytes] = run(project_dir, "build") + + stderr: str = built.stderr.decode(errors="replace") + assert built.returncode == 0, stderr + assert [ + fragment.format(database=recall_database) in stderr + for fragment in test_case.expected_fragments + ] == [True for _ in test_case.expected_fragments] + assert b"Completed successfully" in built.stdout + assert fetch_clickhouse_rows( + config=clickhouse_e2e_config, + sql=f"SELECT count() FROM system.tables WHERE database = '{recall_database}'", + ) == ((0,),) + + +@pytest.mark.parametrize( + "test_case", + [ + ClickHouseRecallConcurrencyTestCase( + description="four builds and a polling reader share one ClickHouse store", + targets=("dev", "dev_1", "dev_2", "dev_3"), + expected_status="completed", + ), + ], + ids=lambda case: case.description, +) +def test_given_concurrent_builds_when_writing_one_store_then_every_run_is_recorded_once( + test_case: ClickHouseRecallConcurrencyTestCase, + tmp_path: Path, + clickhouse_e2e_config: dict[str, object], + recall_database: str, +) -> None: + project_dir: Path = write_recall_project( + parent=tmp_path, + config=clickhouse_e2e_config, + database=recall_database, + targets=len(test_case.targets), + ) + + exit_codes, reads, unexpected_failures = concurrent_builds(project_dir, test_case.targets) + recorded: list[list[object]] = recall_rows( + project_dir, + "SELECT i.target, i.status, o.complete FROM invocations i " + "JOIN output_invocations o USING (invocation_id) ORDER BY i.target", + ) + + assert exit_codes == [0 for _ in test_case.targets] + assert reads > 0 + assert unexpected_failures == 0 + assert recorded == [ + [target, test_case.expected_status, 1] for target in sorted(test_case.targets) + ] + assert recall_rows( + project_dir, + "SELECT (SELECT count() FROM conflicts), (SELECT sum(conflicts) FROM recall_delivery)", + ) == [[0, 0]] + + +@pytest.mark.parametrize( + "test_case", + [ + ClickHouseRecallAbandonedTestCase( + description="a SIGKILLed build is abandoned once its heartbeats go stale", + heartbeat_seconds="0.5", + expected_status="abandoned", + ), + ], + ids=lambda case: case.description, +) +def test_given_a_killed_build_when_heartbeats_stop_then_the_invocation_is_abandoned( + test_case: ClickHouseRecallAbandonedTestCase, + tmp_path: Path, + clickhouse_e2e_config: dict[str, object], + recall_database: str, +) -> None: + project_dir: Path = write_recall_project( + parent=tmp_path, config=clickhouse_e2e_config, database=recall_database + ) + + running_id, recorded = killed_build(project_dir, heartbeat_seconds=test_case.heartbeat_seconds) + + assert recorded == [[running_id, test_case.expected_status, 1, 0]] + + +@pytest.mark.parametrize( + "test_case", + [ + ClickHouseRecallOutputTestCase( + description="stdout alone", + args=("--stream", "stdout"), + expected_stream="stdout", + ), + ClickHouseRecallOutputTestCase( + description="stderr alone", + args=("--stream", "stderr"), + expected_stream="stderr", + ), + ], + ids=lambda case: case.description, +) +def test_given_a_build_when_recalled_from_clickhouse_then_output_is_byte_exact( + test_case: ClickHouseRecallOutputTestCase, + tmp_path: Path, + clickhouse_e2e_config: dict[str, object], + recall_database: str, +) -> None: + project_dir: Path = write_recall_project( + parent=tmp_path, config=clickhouse_e2e_config, database=recall_database + ) + built: subprocess.CompletedProcess[bytes] = run(project_dir, "build") + + recalled: subprocess.CompletedProcess[bytes] = run( + project_dir, "recall", "output", "last", *test_case.args + ) + queried: subprocess.CompletedProcess[bytes] = run( + project_dir, + "recall", + "query", + "--json", + "SELECT outcome, sql_available FROM statements WHERE outcome = 'completed' LIMIT 1", + ) + + assert recalled.returncode == 0, recalled.stderr + assert recalled.stdout == getattr(built, test_case.expected_stream) + assert recalled.stderr == b"" + document: dict[str, object] = json.loads(queried.stdout) + assert document["store"] == {"adapter": "clickhouse", "database": recall_database} + assert document["rows"] == [["completed", 1]] + statistics: dict[str, object] = document["statistics"] # type: ignore[assignment] + assert isinstance(statistics["rows_read"], int) + assert isinstance(statistics["query_id"], str) + + +@pytest.mark.parametrize( + "test_case", + [ + ClickHouseRecallRetentionConflictTestCase( + description="a store created for 7 days is reported, never altered, for a 30-day project", + created_with='retention = "7d"\n', + built_with='retention = "30d"\n', + expected_fragments=( + "keeps history for 7 days by its table TTLs", + "configured for 30 days", + "MODIFY TTL recorded_at + INTERVAL 30 DAY", + "completed successfully; only its Recall history is affected", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_store_with_another_retention_when_building_then_the_conflict_is_reported_once( + test_case: ClickHouseRecallRetentionConflictTestCase, + tmp_path: Path, + clickhouse_e2e_config: dict[str, object], + recall_database: str, +) -> None: + creator: Path = write_recall_project( + parent=tmp_path / "creator", + config=clickhouse_e2e_config, + database=recall_database, + settings=(test_case.created_with, ""), + ) + assert run(creator, "build").returncode == 0 + project_dir: Path = write_recall_project( + parent=tmp_path, + config=clickhouse_e2e_config, + database=recall_database, + settings=(test_case.built_with, ""), + ) + + built: subprocess.CompletedProcess[bytes] = run(project_dir, "build") + repeated: subprocess.CompletedProcess[bytes] = run(project_dir, "build") + + stderr: str = built.stderr.decode(errors="replace") + assert (built.returncode, repeated.returncode) == (0, 0) + assert [fragment in stderr for fragment in test_case.expected_fragments] == [ + True for _ in test_case.expected_fragments + ] + assert b"MODIFY TTL" not in repeated.stderr + assert stored_ttl_days(config=clickhouse_e2e_config, database=recall_database)[0] == ( + "recall_artifact_refs", + 7, + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/website/src/content/docs/docs/cli/recall.mdx b/website/src/content/docs/docs/cli/recall.mdx index 3879c74d27..c436103d4f 100644 --- a/website/src/content/docs/docs/cli/recall.mdx +++ b/website/src/content/docs/docs/cli/recall.mdx @@ -18,8 +18,9 @@ sqb recall output last ## sqb recall query -Run one read-only SQLite `SELECT` against the Recall views. Statements that write, and multiple -statements, are rejected. +Run one read-only `SELECT` against the Recall views, in the store's dialect: SQLite for the local +store, ClickHouse for a [ClickHouse store](/docs/concepts/observability/recall#sharing-history-in-clickhouse). +Statements that write, and multiple statements, are rejected. | Flag | Description | |------|-------------| diff --git a/website/src/content/docs/docs/concepts/observability/recall.mdx b/website/src/content/docs/docs/concepts/observability/recall.mdx index 959319b3b8..70ec088e2e 100644 --- a/website/src/content/docs/docs/concepts/observability/recall.mdx +++ b/website/src/content/docs/docs/concepts/observability/recall.mdx @@ -3,8 +3,8 @@ title: "Recall" description: "The local history of every command: what ran, what it executed, and what it printed." --- -Recall records each `sqb` invocation in a local SQLite store so you can ask what happened after the -fact: which command ran, how it ended, which statements it executed and how long they took, and +Recall records each `sqb` invocation in a local SQLite store, or in a shared +[ClickHouse store](#sharing-history-in-clickhouse), so you can ask what happened after the fact: which command ran, how it ended, which statements it executed and how long they took, and exactly what it printed. Recall is on by default and needs no setup. ## What is recorded @@ -51,7 +51,7 @@ enabled = false # do not record runs against the ci target | Key | Default | Description | |-----|---------|-------------| | `enabled` | `true` | Set to `false` to stop recording | -| `connection` | (none) | A `sqlite` connection whose `path` names the store; defaults to `logs/recall.sqlite3` | +| `connection` | (none) | A `sqlite` connection whose `path` names the store, or a `clickhouse` connection; defaults to `logs/recall.sqlite3` | | `retention` | `"30d"` | How long invocations are kept, in days (`d`) or hours (`h`), up to 3650 days | | `shutdown_timeout` | `"3s"` | The cap on time spent flushing history at exit, in `s` or `ms`, up to 10 minutes | | `diagnostics_min_severity` | `"info"` | `trace`, `debug`, `info`, `warning`, `error`, or `critical` | @@ -102,13 +102,53 @@ repeats at most once a day. If the store file is not a SQLite database, Recall m `recall.sqlite3.corrupt-`, starts a fresh store, and says so once. Commands that find the same corrupt file coordinate through `recall.sqlite3.lock`, so only one of them moves it. -A `connection` that does not use the `sqlite` adapter is not supported yet; Recall prints a notice -and records nothing. +A `connection` on any adapter other than `sqlite` or `clickhouse` is not supported; Recall prints a +notice and records nothing. + +## Sharing history in ClickHouse + +Point `connection` at a ClickHouse connection to keep one history for a whole team. The connection +needs `adapter = "clickhouse"` when the project uses another adapter, and its `database` must +already exist; Recall never creates databases. + +```toml +[connections.history] +adapter = "clickhouse" +host = "${env:CLICKHOUSE_HOST}" +user = "recall_writer" +password = "${env:CLICKHOUSE_PASSWORD}" +database = "observability" + +[recall] +connection = "history" +retention = "30d" +``` + +- **Setup.** On first use Recall creates its `recall_*` MergeTree tables and the views above in that + database, and adds columns that a newer SQLBuild introduces. After that, writing needs only + `INSERT` and `SELECT` on the tables. If the user may not create them, the command still succeeds + and prints the complete DDL for an administrator to run. +- **Writes.** History is sent in batches of up to 5,000 rows about once a second, one insert per + table, with the same queue, shutdown budget and exit-code guarantees as the local store. Many + concurrent commands can write one store. +- **Duplicates.** Every row has a deterministic key. A retried batch is deduplicated by ClickHouse, + and the views keep the first copy of each key, so a repeated delivery is counted once. A key + written again with different content keeps the first row and is listed in `conflicts`. +- **Retention.** Every table carries a TTL of `retention` (SQL text a week longer, and refreshed + while it is still referenced), and the views hide rows past `retention` at once. ClickHouse drops + expired daily parts in the background, so rows may stay on disk for a while after the views stop + showing them. Retention belongs to the store: when a project's `retention` differs from the + tables' TTL, Recall warns once a day with the `ALTER TABLE … MODIFY TTL` and view statements that + would apply it, and never changes the TTL itself. +- **Reading.** `sqb recall query`, `schema` and `output` read the ClickHouse store, so queries use + ClickHouse SQL. `--json` output includes the server's `rows_read`, `bytes_read` and `query_id`. + +Use a single ClickHouse server or ClickHouse Cloud; `ON CLUSTER` stores are not supported yet. ## Querying history -Use [`sqb recall`](/docs/cli/recall) to read the store. Queries are read-only SQLite `SELECT` -statements against these views: +Use [`sqb recall`](/docs/cli/recall) to read the store. Queries are read-only `SELECT` statements, +in the store's SQL dialect, against these views: | View | One row per | |------|-------------| From 8eac28f0ba9fcb458d9bc9cb302ccc20a018e010 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 10:18:27 +0100 Subject: [PATCH 02/19] perf: read ClickHouse Recall views with FINAL so filters reach the tables Every Recall table the views read is now a ReplacingMergeTree ordered by its key, and the views read it with FINAL instead of deduplicating with LIMIT BY. Filters on invocation_id now reach the primary key: on a store of about 3,000 invocations and 2.2 million events, a per-invocation artifact lookup reads 185 thousand rows instead of 507 thousand, and sqb recall output drops from about 2.8 s to 0.5 s. --- .../src/schema/_helpers/clickhouse_views.rs | 12 +++++++++-- .../sqlbuild-recall/src/schema/constants.rs | 20 ++++++++++--------- .../src/schema/tests/test_ddl.rs | 12 +++++++++-- .../docs/concepts/observability/recall.md | 5 +++-- .../docs/concepts/observability/recall.mdx | 5 +++-- 5 files changed, 37 insertions(+), 17 deletions(-) diff --git a/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs b/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs index f94ab19739..ba54bed59e 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs @@ -1,13 +1,21 @@ //! The bodies of the supported views in ClickHouse: rows deduplicated by key, within retention. use crate::schema::_helpers::rendering::clickhouse_table; -use crate::schema::constants::OUTPUT_TAIL_RECORDS; +use crate::schema::constants::{OUTPUT_TAIL_RECORDS, REPLACING_ENGINE}; use crate::schema::main::table_spec::table_spec; use crate::schema::models::{DdlOptions, Table}; -/// `table`'s rows within retention, first ingested copy per key. +/// `table`'s rows within retention, one copy per key: `FINAL` for replacing tables, so filters on the sorting key reach the read, else the first ingested copy. fn deduplicated(table: Table, options: &DdlOptions) -> String { let spec = table_spec(table); + if spec.clickhouse_engine == REPLACING_ENGINE { + return format!( + "(SELECT * FROM {name} FINAL WHERE {expiry} >= now64(6) - INTERVAL {days} DAY)", + name = clickhouse_table(spec.name, options), + expiry = spec.expiry_column, + days = options.retention_days, + ); + } format!( "(SELECT * FROM {name} WHERE {expiry} >= now64(6) - INTERVAL {days} DAY \ ORDER BY ingested_at ASC LIMIT 1 BY {key})", diff --git a/crates/sqlbuild-recall/src/schema/constants.rs b/crates/sqlbuild-recall/src/schema/constants.rs index 0da7cf7f80..e3bf8ed3c9 100644 --- a/crates/sqlbuild-recall/src/schema/constants.rs +++ b/crates/sqlbuild-recall/src/schema/constants.rs @@ -178,7 +178,7 @@ pub(crate) const INVOCATIONS_TABLE: TableSpec = TableSpec { columns: &INVOCATION_COLUMNS, key: &["invocation_id"], sqlite_indexes: &[("started", &["invocation_started_at"])], - clickhouse_engine: "ReplacingMergeTree", + clickhouse_engine: REPLACING_ENGINE, clickhouse_order_by: &["invocation_started_at", "invocation_id"], ttl_extra_days: 0, expiry_column: "invocation_started_at", @@ -189,7 +189,7 @@ pub(crate) const OUTPUT_TABLE: TableSpec = TableSpec { columns: &OUTPUT_COLUMNS, key: &["invocation_id", "sequence"], sqlite_indexes: &[("expiry", &["invocation_started_at"])], - clickhouse_engine: "ReplacingMergeTree", + clickhouse_engine: REPLACING_ENGINE, clickhouse_order_by: &["invocation_id", "sequence"], ttl_extra_days: 0, expiry_column: "invocation_started_at", @@ -205,7 +205,7 @@ pub(crate) const EVENTS_TABLE: TableSpec = TableSpec { ("statement", &["statement_id"]), ("expiry", &["invocation_started_at"]), ], - clickhouse_engine: "ReplacingMergeTree", + clickhouse_engine: REPLACING_ENGINE, clickhouse_order_by: &["invocation_id", "event_id"], ttl_extra_days: 0, expiry_column: "invocation_started_at", @@ -219,7 +219,7 @@ pub(crate) const DIAGNOSTICS_TABLE: TableSpec = TableSpec { ("severity", &["severity"]), ("expiry", &["invocation_started_at"]), ], - clickhouse_engine: "ReplacingMergeTree", + clickhouse_engine: REPLACING_ENGINE, clickhouse_order_by: &["invocation_id", "sequence"], ttl_extra_days: 0, expiry_column: "invocation_started_at", @@ -230,7 +230,7 @@ pub(crate) const ARTIFACTS_TABLE: TableSpec = TableSpec { columns: &ARTIFACT_COLUMNS, key: &["digest"], sqlite_indexes: &[("seen", &["seen_at"])], - clickhouse_engine: "ReplacingMergeTree", + clickhouse_engine: REPLACING_ENGINE, clickhouse_order_by: &["digest"], ttl_extra_days: ARTIFACT_REFRESH_DAYS, expiry_column: "seen_at", @@ -241,7 +241,7 @@ pub(crate) const ARTIFACT_REFS_TABLE: TableSpec = TableSpec { columns: &ARTIFACT_REF_COLUMNS, key: &["invocation_id", "digest", "ref_id"], sqlite_indexes: &[("digest", &["digest"])], - clickhouse_engine: "MergeTree", + clickhouse_engine: REPLACING_ENGINE, clickhouse_order_by: &["invocation_id", "digest", "ref_id"], ttl_extra_days: 0, expiry_column: "invocation_started_at", @@ -252,7 +252,7 @@ pub(crate) const HEARTBEATS_TABLE: TableSpec = TableSpec { columns: &HEARTBEAT_COLUMNS, key: &["invocation_id", "beat_at"], sqlite_indexes: &[], - clickhouse_engine: "MergeTree", + clickhouse_engine: REPLACING_ENGINE, clickhouse_order_by: &["invocation_id", "beat_at"], ttl_extra_days: 0, expiry_column: "invocation_started_at", @@ -263,7 +263,7 @@ pub(crate) const DELIVERY_TABLE: TableSpec = TableSpec { columns: &DELIVERY_COLUMNS, key: &["invocation_id", "stream"], sqlite_indexes: &[], - clickhouse_engine: "MergeTree", + clickhouse_engine: REPLACING_ENGINE, clickhouse_order_by: &["invocation_id", "stream"], ttl_extra_days: 0, expiry_column: "invocation_started_at", @@ -285,7 +285,7 @@ pub(crate) const CONFLICTS_TABLE: TableSpec = TableSpec { columns: &CONFLICT_COLUMNS, key: &["dataset", "record_key", "rejected_sha256"], sqlite_indexes: &[("invocation", &["invocation_id"])], - clickhouse_engine: "MergeTree", + clickhouse_engine: REPLACING_ENGINE, clickhouse_order_by: &["dataset", "record_key", "rejected_sha256"], ttl_extra_days: 0, expiry_column: "invocation_started_at", @@ -328,6 +328,8 @@ pub const VIEW_NAMES: [&str; 10] = [ "artifacts", "conflicts", ]; +/// The ClickHouse engine that collapses copies of a key; views read it with `FINAL`. +pub(crate) const REPLACING_ENGINE: &str = "ReplacingMergeTree"; /// The number of records `output_tail` keeps per invocation. pub const OUTPUT_TAIL_RECORDS: u32 = 200; /// Invocations deleted per maintenance transaction. diff --git a/crates/sqlbuild-recall/src/schema/tests/test_ddl.rs b/crates/sqlbuild-recall/src/schema/tests/test_ddl.rs index b59d10139a..1010e1fe76 100644 --- a/crates/sqlbuild-recall/src/schema/tests/test_ddl.rs +++ b/crates/sqlbuild-recall/src/schema/tests/test_ddl.rs @@ -67,8 +67,16 @@ fn given_clickhouse_tables_when_rendered_then_engines_ttl_and_partitions_follow_ ], }, FragmentTestCase { - description: "heartbeats are append-only", + description: "heartbeats collapse repeated deliveries of a beat", statement: clickhouse_table("recall_heartbeats", None), + expected_fragments: &[ + "ENGINE = ReplacingMergeTree()", + "ORDER BY (invocation_id, beat_at)", + ], + }, + FragmentTestCase { + description: "schema records are append-only", + statement: clickhouse_table("recall_meta", None), expected_fragments: &["ENGINE = MergeTree()"], }, FragmentTestCase { @@ -115,7 +123,7 @@ fn given_clickhouse_statement_sets_when_rendered_then_every_statement_has_the_sh StatementSetTestCase { description: "views deduplicate rows and filter by retention", statements: views(None), - required_fragments: &["now64(6) - INTERVAL 14 DAY", "LIMIT 1 BY"], + required_fragments: &["now64(6) - INTERVAL 14 DAY", " FINAL WHERE "], expected_count: VIEW_NAMES.len(), }, ]; diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md index 72aad75335..761517b7cf 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md @@ -146,8 +146,9 @@ retention = "30d" table, with the same queue, shutdown budget and exit-code guarantees as the local store. Many concurrent commands can write one store. - **Duplicates.** Every row has a deterministic key. A retried batch is deduplicated by ClickHouse, - and the views keep the first copy of each key, so a repeated delivery is counted once. A key - written again with different content keeps the first row and is listed in `conflicts`. + and the views read each key once (`FINAL` on `ReplacingMergeTree` tables), so a repeated delivery + is counted once. Before writing, Recall looks up the batch's keys: a key already stored with + different content keeps the stored row, and the rejected copy is listed in `conflicts`. - **Retention.** Every table carries a TTL of `retention` (SQL text a week longer, and refreshed while it is still referenced), and the views hide rows past `retention` at once. ClickHouse drops expired daily parts in the background, so rows may stay on disk for a while after the views stop diff --git a/website/src/content/docs/docs/concepts/observability/recall.mdx b/website/src/content/docs/docs/concepts/observability/recall.mdx index 70ec088e2e..6334b6d269 100644 --- a/website/src/content/docs/docs/concepts/observability/recall.mdx +++ b/website/src/content/docs/docs/concepts/observability/recall.mdx @@ -132,8 +132,9 @@ retention = "30d" table, with the same queue, shutdown budget and exit-code guarantees as the local store. Many concurrent commands can write one store. - **Duplicates.** Every row has a deterministic key. A retried batch is deduplicated by ClickHouse, - and the views keep the first copy of each key, so a repeated delivery is counted once. A key - written again with different content keeps the first row and is listed in `conflicts`. + and the views read each key once (`FINAL` on `ReplacingMergeTree` tables), so a repeated delivery + is counted once. Before writing, Recall looks up the batch's keys: a key already stored with + different content keeps the stored row, and the rejected copy is listed in `conflicts`. - **Retention.** Every table carries a TTL of `retention` (SQL text a week longer, and refreshed while it is still referenced), and the views hide rows past `retention` at once. ClickHouse drops expired daily parts in the background, so rows may stay on disk for a while after the views stop From 21af0987212dc3df72573c29fac02d99f50c2db1 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 10:37:58 +0100 Subject: [PATCH 03/19] refactor: share Recall timestamp conversions with the ClickHouse store --- .../bindings/_helpers/recall/clickhouse_store.rs | 7 ++----- .../src/clickhouse/_helpers/literals.rs | 16 +--------------- .../src/clickhouse/_helpers/planning.rs | 2 +- .../src/clickhouse/_helpers/row_binary.rs | 2 +- .../src/records/main/epoch_micros.rs | 11 +++++++++++ crates/sqlbuild-recall/src/records/main/mod.rs | 2 ++ .../src/records/main/timestamp_micros.rs | 9 +++++++++ 7 files changed, 27 insertions(+), 22 deletions(-) create mode 100644 crates/sqlbuild-recall/src/records/main/epoch_micros.rs create mode 100644 crates/sqlbuild-recall/src/records/main/timestamp_micros.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs index 8cdf5bb2ee..070b1dc834 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs @@ -15,6 +15,7 @@ use sqlbuild_recall::clickhouse::main::store_state_sql::store_state_sql; use sqlbuild_recall::clickhouse::main::stored_rows_sql::stored_rows_sql; use sqlbuild_recall::clickhouse::main::ttl_mismatches::ttl_mismatches; use sqlbuild_recall::clickhouse::models::{StoredColumn, StoredRow, TtlMismatch}; +use sqlbuild_recall::records::main::epoch_micros::epoch_micros; use sqlbuild_recall::records::main::format_timestamp::format_timestamp; use sqlbuild_recall::records::main::meta_row::meta_row; use sqlbuild_recall::records::models::{OutputStream, RecallRow, RecallValue, RowEnvelope}; @@ -287,11 +288,7 @@ impl ClickHouseStore { now: SystemTime, ) -> Result<(), StoreError> { let columns: &[StoredColumn] = self.columns.get(&table).map_or(&[], Vec::as_slice); - let micros: i64 = now - .duration_since(SystemTime::UNIX_EPOCH) - .map_or(0, |elapsed| { - i64::try_from(elapsed.as_micros()).unwrap_or(i64::MAX) - }); + let micros: i64 = epoch_micros(now); let (names, block) = encode_rows(rows, columns, micros).map_err(|message| StoreError { kind: StoreErrorKind::Other, message, diff --git a/crates/sqlbuild-recall/src/clickhouse/_helpers/literals.rs b/crates/sqlbuild-recall/src/clickhouse/_helpers/literals.rs index 5b8457bf76..9863904c39 100644 --- a/crates/sqlbuild-recall/src/clickhouse/_helpers/literals.rs +++ b/crates/sqlbuild-recall/src/clickhouse/_helpers/literals.rs @@ -1,27 +1,13 @@ //! ClickHouse literals and key expressions for the store's own queries. -use std::time::{SystemTime, UNIX_EPOCH}; - use crate::clickhouse::constants::TIMESTAMP_PATTERN; use crate::records::main::format_timestamp::format_timestamp; use crate::records::main::parse_timestamp::parse_timestamp; +use crate::records::main::timestamp_micros::timestamp_micros; use crate::records::models::RecallValue; use crate::schema::main::table_columns::table_columns; use crate::schema::models::{ColumnType, Table}; -/// Microseconds since the epoch; times before it count back. -pub(crate) fn epoch_micros(time: SystemTime) -> i64 { - match time.duration_since(UNIX_EPOCH) { - Ok(after) => i64::try_from(after.as_micros()).unwrap_or(i64::MAX), - Err(before) => -i64::try_from(before.duration().as_micros()).unwrap_or(i64::MAX), - } -} - -/// The microseconds of Recall timestamp text, `None` when it is not one. -pub(crate) fn timestamp_micros(text: &str) -> Option { - parse_timestamp(text).map(epoch_micros) -} - /// A single-quoted ClickHouse string literal. pub(crate) fn string_literal(text: &str) -> String { let mut quoted = String::with_capacity(text.len() + 2); diff --git a/crates/sqlbuild-recall/src/clickhouse/_helpers/planning.rs b/crates/sqlbuild-recall/src/clickhouse/_helpers/planning.rs index b026cac795..0ee4da1649 100644 --- a/crates/sqlbuild-recall/src/clickhouse/_helpers/planning.rs +++ b/crates/sqlbuild-recall/src/clickhouse/_helpers/planning.rs @@ -2,11 +2,11 @@ use std::collections::{BTreeMap, HashMap}; -use crate::clickhouse::_helpers::literals::timestamp_micros; use crate::clickhouse::_helpers::record_key::record_key; use crate::clickhouse::constants::UNCHECKED_TABLES; use crate::clickhouse::models::{InsertPlan, StoredRow}; use crate::records::main::conflict_row::conflict_row; +use crate::records::main::timestamp_micros::timestamp_micros; use crate::records::models::{RecallRow, RecallValue, RowEnvelope}; use crate::schema::main::table_spec::table_spec; use crate::schema::models::Table; diff --git a/crates/sqlbuild-recall/src/clickhouse/_helpers/row_binary.rs b/crates/sqlbuild-recall/src/clickhouse/_helpers/row_binary.rs index 1f15a0d968..de301e52fa 100644 --- a/crates/sqlbuild-recall/src/clickhouse/_helpers/row_binary.rs +++ b/crates/sqlbuild-recall/src/clickhouse/_helpers/row_binary.rs @@ -1,10 +1,10 @@ //! `RowBinary` encoding of one Recall value; each function takes the block and returns it extended. -use crate::clickhouse::_helpers::literals::timestamp_micros; use crate::clickhouse::constants::{ DECIMAL_SCALE, MICROS_DIGITS, VARINT_CONTINUE, VARINT_MASK, VARINT_PAYLOAD_BITS, }; use crate::clickhouse::models::{ColumnEncoding, StoredColumn}; +use crate::records::main::timestamp_micros::timestamp_micros; use crate::records::models::RecallValue; fn varint(mut block: Vec, mut value: u64) -> Vec { diff --git a/crates/sqlbuild-recall/src/records/main/epoch_micros.rs b/crates/sqlbuild-recall/src/records/main/epoch_micros.rs new file mode 100644 index 0000000000..e746e76514 --- /dev/null +++ b/crates/sqlbuild-recall/src/records/main/epoch_micros.rs @@ -0,0 +1,11 @@ +//! `epoch_micros`: a system time as microseconds since the epoch. + +use std::time::SystemTime; + +use crate::records::_helpers::time::system_micros; + +/// Microseconds since the epoch; times before it count back. +#[must_use] +pub fn epoch_micros(time: SystemTime) -> i64 { + system_micros(time) +} diff --git a/crates/sqlbuild-recall/src/records/main/mod.rs b/crates/sqlbuild-recall/src/records/main/mod.rs index 2e466594b9..b16f7deafb 100644 --- a/crates/sqlbuild-recall/src/records/main/mod.rs +++ b/crates/sqlbuild-recall/src/records/main/mod.rs @@ -3,6 +3,7 @@ pub mod conflict_row; pub mod delivery_row; pub mod diagnostic_input_from_json; pub mod diagnostic_row; +pub mod epoch_micros; pub mod event_object_row; pub mod event_severity; pub mod format_timestamp; @@ -13,3 +14,4 @@ pub mod output_row; pub mod parse_event; pub mod parse_timestamp; pub mod sha256_hex; +pub mod timestamp_micros; diff --git a/crates/sqlbuild-recall/src/records/main/timestamp_micros.rs b/crates/sqlbuild-recall/src/records/main/timestamp_micros.rs new file mode 100644 index 0000000000..22baaa39b7 --- /dev/null +++ b/crates/sqlbuild-recall/src/records/main/timestamp_micros.rs @@ -0,0 +1,9 @@ +//! `timestamp_micros`: Recall timestamp text as microseconds since the epoch. + +use crate::records::_helpers::time::parse_micros; + +/// The microseconds of an ISO-8601 UTC timestamp, `None` when `text` is not one. +#[must_use] +pub fn timestamp_micros(text: &str) -> Option { + parse_micros(text) +} From fbcb9ea1d26c61827946d93d86fe7b58358c2529 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 10:46:08 +0100 Subject: [PATCH 04/19] fix: report a ClickHouse Recall store still being created as not recorded yet A reader that ran while the first writer was still creating the views failed with an unknown-table query error; it now gets the coded store error that says no history is recorded yet. --- .../src/bindings/_helpers/recall/clickhouse_store.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs index 070b1dc834..399f0c8f98 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs @@ -447,13 +447,14 @@ impl ClickHouseReader { columns: HashMap::new(), notices: Vec::new(), }; - if store.state(py)?.tables.is_empty() { + let state: StoreState = store.state(py)?; + if state.tables.len() < ALL_TABLES.len() || state.views.len() < VIEW_NAMES.len() { store.close(); return Err(StoreError { kind: StoreErrorKind::NotFound, message: format!( - "no Recall history is recorded yet in ClickHouse database `{}`; run a command \ - first", + "no Recall history is recorded yet in ClickHouse database `{}`, or its tables \ + are still being created; run a command first", location.database() ), }); From 6f22348ff881568fad735bf7134d18396aac250b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 11:55:14 +0100 Subject: [PATCH 05/19] feat: let a sink taking a RecallWriter decide what Recall stores A @lifecycle_event_sink or @command_output_sink with a parameter annotated sqlbuild.recall.RecallWriter replaces Recall's default for its stream: Recall stores only what it publishes. Discovery allows one replacement per stream and reports D015 naming every path:function otherwise, rejects a sink with two writers or with providers, and reserves the provider name recall_writer (D012). Replacements are not run as ordinary sinks; other sinks still see every record. Lifecycle events reach the replacement as they are published, filtered by its event kinds and minimum severity. Output chunks reach it on the delivery thread as schema version 2 CommandOutputRecords carrying the exact bytes in data; record.replace() keeps the ID and sequence. An unpublished record is counted as filtered, a raising sink as failed, never falling back to the unfiltered record and never changing the exit code or output. The invocation row now waits for discovery so filter_active records whether a replacement ran. --- .../src/command_output/constants.rs | 2 + .../src/command_output/main/ansi_free_text.rs | 9 + .../src/command_output/main/mod.rs | 1 + .../main/validate_output_record.rs | 13 +- .../src/command_output/models.rs | 1 + .../src/bindings/_helpers/cli/entry.rs | 46 ++- .../bindings/_helpers/cli/observability.rs | 24 ++ .../_helpers/compile_records/constants.rs | 2 + .../_helpers/discovery/imported_providers.rs | 61 +++- .../_helpers/native_host/user_code.rs | 6 + .../_helpers/output_capture/records.rs | 3 +- .../src/bindings/_helpers/recall/constants.rs | 7 + .../src/bindings/_helpers/recall/mod.rs | 2 + .../bindings/_helpers/recall/registration.rs | 4 +- .../bindings/_helpers/recall/replacements.rs | 267 ++++++++++++++++++ .../src/bindings/_helpers/recall/session.rs | 8 + .../src/bindings/_helpers/recall/sinks.rs | 31 +- .../src/bindings/_helpers/recall/writer.rs | 139 +++++++++ .../src/writer/_helpers/delivery.rs | 12 +- .../src/writer/_helpers/materialize.rs | 72 ++++- .../src/writer/_helpers/mod.rs | 1 + .../src/writer/_helpers/output_chunk.rs | 45 +++ .../src/writer/_helpers/queue.rs | 23 +- .../src/writer/_helpers/session.rs | 15 + .../src/writer/main/start_recall_session.rs | 19 +- crates/sqlbuild-recall/src/writer/mod.rs | 1 + crates/sqlbuild-recall/src/writer/models.rs | 13 + .../src/writer/tests/helpers.rs | 33 ++- .../src/writer/tests/test_session.rs | 37 ++- .../src/writer/tests/test_types.rs | 28 +- crates/sqlbuild-recall/src/writer/types.rs | 10 + fensu.toml | 5 +- python-boundary.toml | 15 + .../docs/concepts/observability/recall.md | 31 ++ .../docs/concepts/observability/sinks.md | 11 + src/sqlbuild/_native.pyi | 10 + src/sqlbuild/_native_host.py | 4 + src/sqlbuild/_native_host.pyi | 11 +- .../filesystem/command_output_sinks.py | 18 +- .../_helpers/filesystem/recall_sinks.py | 67 +++++ .../discovery/main/runtime_extensions.py | 20 +- src/sqlbuild/recall.py | 5 + .../main/_invoke_recall_sink.py | 13 + src/sqlbuild/runtime/output_capture/models.py | 15 +- .../src/sqlbuild/cli/recall/_test_types.py | 21 ++ .../src/sqlbuild/cli/recall/helpers.py | 68 +++++ .../recall/test_recall_replacement_sinks.py | 144 ++++++++++ .../compiler/discovery/_test_types.py | 17 ++ .../sqlbuild/compiler/discovery/helpers.py | 45 +++ .../test_recall_replacement_sinks.py | 145 ++++++++++ .../output_capture/main/_test_types.py | 15 + .../main/test_command_output_record.py | 74 +++++ .../docs/concepts/observability/recall.mdx | 31 ++ .../docs/concepts/observability/sinks.mdx | 4 + 54 files changed, 1659 insertions(+), 65 deletions(-) create mode 100644 crates/sqlbuild-core/src/command_output/main/ansi_free_text.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/recall/replacements.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/recall/writer.rs create mode 100644 crates/sqlbuild-recall/src/writer/_helpers/output_chunk.rs create mode 100644 crates/sqlbuild-recall/src/writer/types.rs create mode 100644 src/sqlbuild/compiler/discovery/_helpers/filesystem/recall_sinks.py create mode 100644 src/sqlbuild/recall.py create mode 100644 src/sqlbuild/runtime/event_exporting/main/_invoke_recall_sink.py create mode 100644 tests/integration/src/sqlbuild/cli/recall/test_recall_replacement_sinks.py create mode 100644 tests/unit/src/sqlbuild/compiler/discovery/test_recall_replacement_sinks.py create mode 100644 tests/unit/src/sqlbuild/runtime/output_capture/main/test_command_output_record.py diff --git a/crates/sqlbuild-core/src/command_output/constants.rs b/crates/sqlbuild-core/src/command_output/constants.rs index 57c7e05962..a80ca72aa9 100644 --- a/crates/sqlbuild-core/src/command_output/constants.rs +++ b/crates/sqlbuild-core/src/command_output/constants.rs @@ -9,6 +9,8 @@ pub const MIN_SHUTDOWN_TIMEOUT_SECONDS: f64 = 0.0; pub const COMMAND_OUTPUT_RECORD_TYPE: &str = "command_output"; pub const COMMAND_OUTPUT_LOSS_RECORD_TYPE: &str = "command_output_loss"; pub const CURRENT_SCHEMA_VERSION: i64 = 1; +/// The schema of records a Recall replacement sink receives: version 1 plus the exact `data` bytes. +pub const RECALL_REPLACEMENT_SCHEMA_VERSION: i64 = 2; pub const MAX_INVOCATION_CONTEXT_BYTES: usize = 16 * 1024; pub const MAX_INVOCATION_CONTEXT_DEPTH: usize = 8; pub const INVOCATION_CONTEXT_ENV: &str = "SQLBUILD_INVOCATION_CONTEXT_JSON"; diff --git a/crates/sqlbuild-core/src/command_output/main/ansi_free_text.rs b/crates/sqlbuild-core/src/command_output/main/ansi_free_text.rs new file mode 100644 index 0000000000..70769a5336 --- /dev/null +++ b/crates/sqlbuild-core/src/command_output/main/ansi_free_text.rs @@ -0,0 +1,9 @@ +//! `ansi_free_text`: output bytes as the text a command-output record's `message` holds. + +use crate::command_output::_helpers::text::strip_ansi; + +/// `data` without terminal control sequences, decoded as UTF-8 with invalid bytes replaced. +#[must_use] +pub fn ansi_free_text(data: &[u8]) -> String { + String::from_utf8_lossy(&strip_ansi(data)).into_owned() +} diff --git a/crates/sqlbuild-core/src/command_output/main/mod.rs b/crates/sqlbuild-core/src/command_output/main/mod.rs index 9b7cef61f9..473c11c79b 100644 --- a/crates/sqlbuild-core/src/command_output/main/mod.rs +++ b/crates/sqlbuild-core/src/command_output/main/mod.rs @@ -1,3 +1,4 @@ +pub mod ansi_free_text; pub mod append_output; pub mod chunk_output_text; pub mod close_output; diff --git a/crates/sqlbuild-core/src/command_output/main/validate_output_record.rs b/crates/sqlbuild-core/src/command_output/main/validate_output_record.rs index 5cb51ba1a1..a214bbbceb 100644 --- a/crates/sqlbuild-core/src/command_output/main/validate_output_record.rs +++ b/crates/sqlbuild-core/src/command_output/main/validate_output_record.rs @@ -2,6 +2,7 @@ use crate::command_output::constants::{ COMMAND_OUTPUT_LOSS_RECORD_TYPE, COMMAND_OUTPUT_RECORD_TYPE, CURRENT_SCHEMA_VERSION, + RECALL_REPLACEMENT_SCHEMA_VERSION, }; use crate::command_output::models::RecordShape; @@ -35,6 +36,9 @@ pub fn validate_output_record(shape: &RecordShape) -> Result<(), String> { if !shape.message_is_text { return Err("message must be a string".to_owned()); } + if !shape.data_is_bytes { + return Err("data must be bytes".to_owned()); + } let chunk_index = at_least("chunk_index", shape.chunk_index, 0)?; let chunk_count = at_least("chunk_count", shape.chunk_count, 1)?; let dropped = at_least("dropped_records", shape.dropped_records, 0)?; @@ -54,8 +58,13 @@ pub fn validate_output_record(shape: &RecordShape) -> Result<(), String> { Some(COMMAND_OUTPUT_RECORD_TYPE | COMMAND_OUTPUT_LOSS_RECORD_TYPE) => {} _ => return Err("record_type is unknown".to_owned()), } - if shape.schema_version != Some(CURRENT_SCHEMA_VERSION) { - return Err(format!("schema_version must be {CURRENT_SCHEMA_VERSION}")); + if !matches!( + shape.schema_version, + Some(CURRENT_SCHEMA_VERSION | RECALL_REPLACEMENT_SCHEMA_VERSION) + ) { + return Err(format!( + "schema_version must be {CURRENT_SCHEMA_VERSION} or {RECALL_REPLACEMENT_SCHEMA_VERSION}" + )); } Ok(()) } diff --git a/crates/sqlbuild-core/src/command_output/models.rs b/crates/sqlbuild-core/src/command_output/models.rs index 3edf0fa627..c765f80ef2 100644 --- a/crates/sqlbuild-core/src/command_output/models.rs +++ b/crates/sqlbuild-core/src/command_output/models.rs @@ -116,6 +116,7 @@ pub struct RecordShape { pub occurred_at_is_utc: bool, pub stream_is_known: bool, pub message_is_text: bool, + pub data_is_bytes: bool, pub chunk_index: Option, pub chunk_count: Option, pub dropped_records: Option, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index 1d6ec0a116..92c5486ca8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -1,5 +1,6 @@ //! The `sqb` entry: invocation scopes, parsing, observed dispatch and the error projection. +use std::sync::Arc; use std::time::Instant; use pyo3::exceptions::PySystemExit; @@ -16,7 +17,9 @@ use crate::bindings::_helpers::build_progress::transient_lines::shared_transient use crate::bindings::_helpers::cli::argv::parse_cli_invocation; use crate::bindings::_helpers::cli::dispatch::{command_facts, dispatch_command}; use crate::bindings::_helpers::cli::launch_state::offer_launch_record; -use crate::bindings::_helpers::cli::observability::cli_observability_scope; +use crate::bindings::_helpers::cli::observability::{ + CliObservabilityScope, cli_observability_scope, +}; use crate::bindings::_helpers::compile_command::report::format_compile_error_json; use crate::bindings::_helpers::compile_driver::python_calls::{ call_with, imported, keywords, within, @@ -397,12 +400,27 @@ fn lifecycle_event<'py>( ) } -/// Subscribe the invocation's Recall session to every lifecycle fact and SQL artifact. -fn subscribe_recall(dispatcher: &Bound<'_, PyAny>, recall: &InvocationRecall) -> PyResult<()> { +/// Install the discovered Recall replacement sinks, release the context row, and subscribe the session to every lifecycle fact and SQL artifact. +fn subscribe_recall( + (scope, dispatcher): (&Bound<'_, CliObservabilityScope>, &Bound<'_, PyAny>), + recall: &InvocationRecall, +) -> PyResult<()> { let Some(session) = recall.session.clone() else { return Ok(()); }; - let sink = Bound::new(dispatcher.py(), RecallLifecycleSink { session })?; + let py: Python<'_> = dispatcher.py(); + let (lifecycle, output) = scope.get().recall_sinks(py)?; + let filter_active: bool = recall + .replacements + .configure(&session, (&lifecycle, &output))?; + session.publish_context(filter_active); + let sink = Bound::new( + py, + RecallLifecycleSink { + session, + replacements: Arc::clone(&recall.replacements), + }, + )?; dispatcher.call_method1("subscribe_history", (sink,))?; Ok(()) } @@ -432,6 +450,9 @@ fn observed_dispatch<'py>( ) -> PyResult> { let py: Python<'py> = invocation.py(); if !policy.observed { + if let Some(session) = &invocation.recall.session { + session.publish_context(false); + } return dispatch_command(&invocation.args, invocation.handlers.clone()); } let command: String = invocation.command_name(); @@ -441,20 +462,20 @@ fn observed_dispatch<'py>( .unwrap_or(u64::MAX) .into_bound_py_any(py) }; - let scope: Bound<'py, PyAny> = Bound::new( + let observability: Bound<'py, CliObservabilityScope> = Bound::new( py, cli_observability_scope( invocation.args.clone().unbind(), invocation.project_dir()?.unbind(), ), - )? - .into_any(); + )?; + let scope: Bound<'py, PyAny> = observability.clone().into_any(); within(&scope, |dispatcher| { let publish = |event: Bound<'py, PyAny>| -> PyResult<()> { dispatcher.call_method1("publish_lifecycle", (event,))?; Ok(()) }; - subscribe_recall(dispatcher, &invocation.recall)?; + subscribe_recall((&observability, dispatcher), &invocation.recall)?; let payload: Bound<'py, PyDict> = PyDict::new(py); payload.set_item("command", &command)?; add_invocation_context(invocation, &payload)?; @@ -551,7 +572,14 @@ fn parsed_invocation<'py>( InvocationRecall::default() }; let recall_sink: Bound<'py, PyAny> = match recall.session.clone() { - Some(session) => Bound::new(py, RecallLifecycleSink { session })?.into_any(), + Some(session) => Bound::new( + py, + RecallLifecycleSink { + session, + replacements: Arc::clone(&recall.replacements), + }, + )? + .into_any(), None => py.None().into_bound(py), }; let closure_recall: InvocationRecall = recall.clone(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs index b6b8066a02..f6c98f448d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs @@ -150,6 +150,8 @@ struct Observed { exporter_scope: Option>, unsubscribe_exporters: Option>, unsubscribe_outcome: Option>, + /// The discovered Recall lifecycle and command-output replacement sinks, each `(sink, writer_parameter)` or `None`. + recall_sinks: Option<(Py, Py)>, /// Context managers entered, innermost last. entered: Vec>, } @@ -332,6 +334,10 @@ impl Observed { &host::entry(py, UserCodeHost::DiscoverRuntimeExtensions)?, &[("project_dir", project_dir.clone())], )?; + self.recall_sinks = Some(( + native_field(&extensions, "recall_lifecycle_sink")?.unbind(), + native_field(&extensions, "recall_output_sink")?.unbind(), + )); let event_exporters: Bound<'_, PyAny> = native_field(&extensions, "event_exporters")?; let sinks: Bound<'_, PyAny> = native_field(&extensions, "command_output_sinks")?; command_output_sinks = sinks.is_truthy()?; @@ -432,6 +438,23 @@ pub(crate) struct CliObservabilityScope { } impl CliObservabilityScope { + /// The Recall lifecycle and command-output replacement sinks discovery found, each `(sink, writer_parameter)` or `None`. + pub(crate) fn recall_sinks<'py>( + &self, + py: Python<'py>, + ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + let observed = self.locked()?; + Ok( + match observed + .as_ref() + .and_then(|observed| observed.recall_sinks.as_ref()) + { + Some((lifecycle, output)) => (lifecycle.bind(py).clone(), output.bind(py).clone()), + None => (py.None().into_bound(py), py.None().into_bound(py)), + }, + ) + } + fn locked(&self) -> PyResult>> { self.observed .lock() @@ -510,6 +533,7 @@ impl CliObservabilityScope { exporter_scope: None, unsubscribe_exporters: None, unsubscribe_outcome: None, + recall_sinks: None, entered: Vec::new(), }; if let Err(error) = observed.install(py, (args, self.project_dir.bind(py)), &terminal_index) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs index 4de06e17e0..cc28328048 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs @@ -1191,6 +1191,8 @@ const R_DISCOVERED_RUNTIME_EXTENSIONS: RecordSpec = RecordSpec { FieldSpec::new("event_exporters", FieldDefault::EmptyTuple), FieldSpec::new("command_output_sinks", FieldDefault::EmptyTuple), FieldSpec::new("lifecycle_shutdown_timeout_seconds", FieldDefault::None), + FieldSpec::new("recall_lifecycle_sink", FieldDefault::None), + FieldSpec::new("recall_output_sink", FieldDefault::None), ], }; const R_COMPILER_DIAGNOSTIC: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_providers.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_providers.rs index 853202768d..e6b381bc4d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_providers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_providers.rs @@ -15,12 +15,15 @@ use crate::bindings::_helpers::discovery::imported_modules::{ }; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; +use crate::bindings::_helpers::recall::writer::RecallWriter; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_get_or; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; const PROVIDER_ERROR: ErrorId = ErrorId::ProviderDiscovery; const SINK_ERROR: ErrorId = ErrorId::EventExporterDiscovery; +/// The provider name kept free so a sink's `RecallWriter` parameter can never be mistaken for a provider. +const RESERVED_PROVIDER_NAME: &str = "recall_writer"; const PROVIDER_INPUT_ERROR: UserCodeHost = UserCodeHost::ProviderInputError; const VALIDATION_ERROR: (&str, &str) = ("pydantic", "ValidationError"); const SINK_DEFINITION: UserCodeHost = UserCodeHost::GetLifecycleEventSinkDefinition; @@ -157,6 +160,17 @@ pub(crate) fn discover_provider_classes<'py>( let relative_path: Bound<'py, PyAny> = project.relative(&file_path)?; let name: Bound<'py, PyAny> = provider_name(&provider_class, &relative_path)?; let name_text: String = name.extract()?; + if name_text == RESERVED_PROVIDER_NAME { + return Err(discovery_error( + py, + PROVIDER_ERROR, + format!( + "Provider name '{RESERVED_PROVIDER_NAME}' in {} is reserved for the \ + RecallWriter of Recall replacement sinks; rename the provider", + project.relative_to(&file_path)? + ), + )); + } if let Some((_, existing)) = seen.iter().find(|(listed, _)| *listed == name_text) { return Err(discovery_error( py, @@ -267,6 +281,33 @@ pub(crate) fn discover_event_exporter_declarations<'py>( tuple_of(py, declarations) } +/// Each parameter of `function` with its annotation, and whether that annotation is `RecallWriter`. +fn annotated_parameters<'py>( + function: &Bound<'py, PyAny>, +) -> PyResult, bool)>> { + let py: Python<'py> = function.py(); + let writer = py.get_type::(); + let mut parameters: Vec<(String, Bound<'py, PyAny>, bool)> = Vec::new(); + for pair in shim(py, UserCodeHost::ParameterAnnotations)? + .call1((function,))? + .try_iter()? + { + let (parameter, annotation): (String, Bound<'py, PyAny>) = pair?.extract()?; + let is_writer: bool = annotation.is(writer.as_any()); + parameters.push((parameter, annotation, is_writer)); + } + Ok(parameters) +} + +/// Whether `declaration`'s function takes a `RecallWriter`, making it a Recall replacement sink. +fn takes_recall_writer(declaration: &Bound<'_, PyAny>) -> PyResult { + Ok( + annotated_parameters(&native_field(declaration, "function")?)? + .iter() + .any(|(_, _, is_writer)| *is_writer), + ) +} + /// Provider parameters after the event, each naming a known provider of the right class. fn exporter_provider_usages<'py>( declaration: &Bound<'py, PyAny>, @@ -280,13 +321,10 @@ fn exporter_provider_usages<'py>( .str()? .extract()?; let empty: Bound<'py, PyAny> = imported(py, "inspect", "Parameter")?.getattr("empty")?; - let annotations: Bound<'py, PyAny> = - shim(py, UserCodeHost::ParameterAnnotations)?.call1((&function,))?; - for (index, pair) in annotations.try_iter()?.enumerate() { - if index == 0 { + for (parameter, annotation, is_writer) in annotated_parameters(&function)?.into_iter().skip(1) { + if is_writer { continue; } - let (parameter, annotation): (String, Bound<'py, PyAny>) = pair?.extract()?; let Some(provider) = providers.get_item(¶meter)? else { return Err(discovery_error( py, @@ -344,7 +382,7 @@ pub(crate) fn bind_event_exporter_declarations<'py>( tuple_of(py, bound) } -/// Validated lifecycle-event sink declarations under sinks/, bound to providers. +/// Validated lifecycle-event sink declarations under sinks/, bound to providers; Recall replacement sinks are left out. #[pyfunction] #[pyo3(signature = (*, project_dir, providers = None))] pub(crate) fn discover_event_exporter_functions<'py>( @@ -358,10 +396,15 @@ pub(crate) fn discover_event_exporter_functions<'py>( if let Some(cached) = project.cached_extensions()? { return cached.get_item(1); } - let declarations: Bound<'py, PyAny> = - discover_event_exporter_declarations(project_dir.clone())?; + let mut ordinary: Vec> = Vec::new(); + for declaration in discover_event_exporter_declarations(project_dir.clone())?.try_iter()? { + let declaration: Bound<'py, PyAny> = declaration?; + if !takes_recall_writer(&declaration)? { + ordinary.push(declaration); + } + } bind_event_exporter_declarations( - declarations, + tuple_of(py, ordinary)?, providers.map_or_else(|| tuple_of(py, Vec::new()), Ok)?, project_dir, ) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/user_code.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/user_code.rs index 30ede99a40..20586f35a0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/user_code.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/user_code.rs @@ -57,6 +57,7 @@ pub(crate) enum UserCodeHost { InvokeLifecycleEventSink, RunCommandOutputSinks, GetLifecycleEventSinkDefinition, + InvokeRecallSink, } /// Every user-code entry, in `UserCodeHost` order. @@ -321,6 +322,11 @@ pub(crate) const ENTRIES: &[HostEntry] = &[ kind: HostKind::Function, edge: HostEdge::UserCode, }, + HostEntry { + name: "invoke_recall_sink", + kind: HostKind::Function, + edge: HostEdge::UserCode, + }, ]; impl UserCodeHost { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/output_capture/records.rs b/crates/sqlbuild-python/src/bindings/_helpers/output_capture/records.rs index 2470bf0640..5f902b9587 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/output_capture/records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/output_capture/records.rs @@ -5,7 +5,7 @@ use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, pyfunction, wrap_pyfunction, }; -use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods}; +use pyo3::types::{PyBytes, PyDict, PyDictMethods, PyString, PyStringMethods}; use sqlbuild_core::command_output::constants::{ COMMAND_OUTPUT_LOSS_RECORD_TYPE, COMMAND_OUTPUT_RECORD_TYPE, CURRENT_SCHEMA_VERSION, INVOCATION_CONTEXT_ENV, MAX_INVOCATION_CONTEXT_BYTES, MAX_INVOCATION_CONTEXT_DEPTH, @@ -101,6 +101,7 @@ fn record_shape(record: &Bound<'_, PyAny>) -> PyResult { stream_is_known: field("stream")? .is_instance(&host::entry(py, ExecutorHost::CommandOutputStream)?)?, message_is_text: field("message")?.is_instance_of::(), + data_is_bytes: field("data")?.is_instance_of::(), chunk_index: plain_int(&field("chunk_index")?), chunk_count: plain_int(&field("chunk_count")?), dropped_records: plain_int(&field("dropped_records")?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs index e1093942ed..114976c1d5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs @@ -91,3 +91,10 @@ pub(crate) const CLOSE_POLL_MS: u64 = 50; pub(crate) const QUARANTINE_START_WAIT_MS: u64 = 200; /// Seconds in a day, for the artifact refresh cutoff. pub(crate) const SECONDS_PER_DAY: u64 = 24 * 60 * 60; +/// The parameter a lifecycle-event sink takes its event in. +pub(crate) const EVENT_PARAMETER: &str = "event"; +/// The parameter a command-output sink takes its record in. +pub(crate) const RECORD_PARAMETER: &str = "record"; +/// How Recall timestamps mark UTC, and the offset `datetime.fromisoformat` reads. +pub(crate) const UTC_SUFFIX: &str = "Z"; +pub(crate) const UTC_OFFSET: &str = "+00:00"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/mod.rs index 1baa25d9fd..3cf9b77191 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/mod.rs @@ -6,5 +6,7 @@ pub(crate) mod constants; pub(crate) mod events; pub(crate) mod registration; pub(crate) mod rendering; +pub(crate) mod replacements; pub(crate) mod session; pub(crate) mod sinks; +pub(crate) mod writer; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/registration.rs index 6e7568ce35..fc10453d94 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/registration.rs @@ -3,7 +3,9 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; use crate::bindings::_helpers::recall::sinks::RecallLifecycleSink; +use crate::bindings::_helpers::recall::writer::RecallWriter; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::() + module.add_class::()?; + module.add_class::() } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/replacements.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/replacements.rs new file mode 100644 index 0000000000..f2f5752f2d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/replacements.rs @@ -0,0 +1,267 @@ +//! Recall replacement sinks: the one lifecycle and one command-output sink that decide what Recall stores. + +use std::cell::Cell; +use std::sync::{Arc, OnceLock}; + +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyBytes, PyDict, PyDictMethods, PyString}; +use sqlbuild_core::command_output::constants::RECALL_REPLACEMENT_SCHEMA_VERSION; +use sqlbuild_core::command_output::main::ansi_free_text::ansi_free_text; +use sqlbuild_core::event_export::main::route_lifecycle_event::route_lifecycle_event; +use sqlbuild_core::event_export::models::{ExporterCounts, ExporterFilter}; +use sqlbuild_recall::records::models::{OutputChunk, OutputStream}; +use sqlbuild_recall::writer::models::{DeliveryStream, RecallSession, ReplacedOutput}; +use sqlbuild_recall::writer::types::OutputReplacement; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::event_exporting::models::BoundEventExporter; +use crate::bindings::_helpers::event_exporting::values::export_policy; +use crate::bindings::_helpers::native_host::executor::ExecutorHost; +use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::native_host::user_code::UserCodeHost; +use crate::bindings::_helpers::native_threads::constants::OUTPUT_SUPPRESSED_ATTRIBUTE; +use crate::bindings::_helpers::output_capture::records::invocation_context; +use crate::bindings::_helpers::recall::constants::{ + EVENT_PARAMETER, RECORD_PARAMETER, UTC_OFFSET, UTC_SUFFIX, +}; +use crate::bindings::_helpers::recall::writer::RecallWriter; +use crate::bindings::_helpers::records::fields::field as native_field; + +thread_local! { + /// Set while this thread runs the output replacement sink, whose own writes Recall never records. + static IN_OUTPUT_REPLACEMENT: Cell = const { Cell::new(false) }; +} + +/// Whether the current thread is running the output replacement sink. +pub(crate) fn in_output_replacement() -> bool { + IN_OUTPUT_REPLACEMENT.with(Cell::get) +} + +/// The lifecycle replacement: the sink function, the parameter its writer goes in, and its event filter. +struct LifecycleReplacement { + function: Py, + writer_parameter: String, + filter: ExporterFilter, +} + +/// The replacement sinks of one invocation, configured once discovery has run. +#[derive(Default)] +pub(crate) struct RecallReplacements { + lifecycle: OnceLock, +} + +impl std::fmt::Debug for RecallReplacements { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("RecallReplacements") + .field("lifecycle", &self.lifecycle.get().is_some()) + .finish() + } +} + +/// `function(**{parameter: record, writer_parameter: writer})` through the user-code shim; whether it raised. +fn invoke( + py: Python<'_>, + function: &Py, + (parameter, record): (&str, &Bound<'_, PyAny>), + (writer_parameter, writer): (&str, Bound<'_, PyAny>), +) -> PyResult { + let arguments = PyDict::new(py); + arguments.set_item(parameter, record)?; + arguments.set_item(writer_parameter, writer)?; + call_with( + &host::entry(py, UserCodeHost::InvokeRecallSink)?, + &[ + ("function", function.bind(py).clone()), + ("arguments", arguments.into_any()), + ], + )? + .extract() +} + +/// The `(sink, writer_parameter)` pair discovery found, or `None`. +fn discovered<'py>(pair: &Bound<'py, PyAny>) -> PyResult, String)>> { + if pair.is_none() { + return Ok(None); + } + let (sink, writer_parameter): (Bound<'py, PyAny>, String) = pair.extract()?; + Ok(Some((sink, writer_parameter))) +} + +impl RecallReplacements { + /// Install the discovered replacements, each `(sink, writer_parameter)` or `None`; whether any filters Recall history. + pub(crate) fn configure( + &self, + session: &Arc, + (lifecycle, output): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + ) -> PyResult { + let mut active: bool = false; + if let Some((sink, writer_parameter)) = discovered(lifecycle)? { + let bound = BoundEventExporter::create( + native_field(&sink, "name")?.extract()?, + ( + native_field(&sink, "function")?.unbind(), + PyDict::new(sink.py()).into_any().unbind(), + ), + ( + Some(&native_field(&sink, "event_kinds")?), + Some(&native_field(&sink, "min_severity")?), + ), + )?; + drop(self.lifecycle.set(LifecycleReplacement { + function: bound.function, + writer_parameter, + filter: bound.filter, + })); + active = true; + } + if let Some((sink, writer_parameter)) = discovered(output)? { + let mut streams: [bool; 2] = [false; 2]; + for stream in native_field(&sink, "streams")?.try_iter()? { + let name: String = native_field(&stream?, "value")?.extract()?; + if let Some(stream) = OutputStream::from_name(&name) { + streams[usize::from(stream == OutputStream::Stderr)] = true; + } + } + session.set_output_replacement(Arc::new(OutputSink { + function: native_field(&sink, "function")?.unbind(), + writer_parameter, + streams, + invocation_id: session.invocation_id(), + })); + active = true; + } + Ok(active) + } + + /// Hand `event` to the lifecycle replacement and account for what it kept; `false` when there is none. + pub(crate) fn deliver_lifecycle( + &self, + py: Python<'_>, + session: &Arc, + event: &Bound<'_, PyAny>, + ) -> PyResult { + let Some(replacement) = self.lifecycle.get() else { + return Ok(false); + }; + let event_type: String = native_field(event, "event_type")?.extract()?; + let eligible: bool = export_policy(py, &event_type).is_ok_and(|policy| { + let (_, eligible) = route_lifecycle_event( + &policy, + std::slice::from_ref(&replacement.filter), + vec![ExporterCounts::default()], + ); + !eligible.is_empty() + }); + if !eligible { + session.record_filtered(DeliveryStream::Events, 1); + return Ok(true); + } + let writer = Bound::new(py, RecallWriter::lifecycle(Arc::clone(session)))?; + let raised: PyResult = invoke( + py, + &replacement.function, + (EVENT_PARAMETER, event), + (&replacement.writer_parameter, writer.clone().into_any()), + ); + let (published, _) = writer.get().finish(); + match raised { + Ok(false) if published == 0 => session.record_filtered(DeliveryStream::Events, 1), + Ok(false) => {} + Ok(true) | Err(_) => session.record_failed(DeliveryStream::Events, 1), + } + Ok(true) + } +} + +/// The output replacement, called on the Recall delivery thread for each chunk. +struct OutputSink { + function: Py, + writer_parameter: String, + /// Whether the sink takes stdout and stderr chunks; others are filtered. + streams: [bool; 2], + invocation_id: String, +} + +impl std::fmt::Debug for OutputSink { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("OutputSink") + .field("writer_parameter", &self.writer_parameter) + .field("streams", &self.streams) + .finish_non_exhaustive() + } +} + +impl OutputSink { + /// The schema-2 `CommandOutputRecord` of `chunk`. + fn record<'py>(&self, py: Python<'py>, chunk: &OutputChunk) -> PyResult> { + let occurred_at: Bound<'py, PyAny> = imported(py, "datetime", "datetime")?.call_method1( + "fromisoformat", + (chunk.occurred_at.replace(UTC_SUFFIX, UTC_OFFSET),), + )?; + let context: Bound<'py, PyAny> = + invocation_context(py, None).unwrap_or_else(|_| PyDict::new(py).into_any()); + call_with( + &host::entry(py, ExecutorHost::CommandOutputRecord)?, + &[ + ( + "invocation_id", + PyString::new(py, &self.invocation_id).into_any(), + ), + ("sequence", chunk.sequence.into_bound_py_any(py)?), + ("occurred_at", occurred_at), + ( + "stream", + host::entry(py, ExecutorHost::CommandOutputStream)? + .call1((chunk.stream.as_str(),))?, + ), + ( + "message", + PyString::new(py, &ansi_free_text(&chunk.data)).into_any(), + ), + ("external_context", context), + ("data", PyBytes::new(py, &chunk.data).into_any()), + ( + "schema_version", + RECALL_REPLACEMENT_SCHEMA_VERSION.into_bound_py_any(py)?, + ), + ], + ) + } + + fn call(&self, py: Python<'_>, chunk: &OutputChunk) -> PyResult { + if let Ok(thread) = + imported(py, "threading", "current_thread").and_then(|current| current.call0()) + { + thread.setattr(OUTPUT_SUPPRESSED_ATTRIBUTE, true)?; + } + let record = self.record(py, chunk)?; + let writer = Bound::new(py, RecallWriter::output(chunk.sequence))?; + let raised: bool = invoke( + py, + &self.function, + (RECORD_PARAMETER, &record), + (&self.writer_parameter, writer.clone().into_any()), + )?; + let (_, published) = writer.get().finish(); + Ok(match (raised, published) { + (true, _) => ReplacedOutput::Failed, + (false, Some(data)) => ReplacedOutput::Published(data), + (false, None) => ReplacedOutput::Filtered, + }) + } +} + +impl OutputReplacement for OutputSink { + fn replace(&self, chunk: &OutputChunk) -> ReplacedOutput { + if !self.streams[usize::from(chunk.stream == OutputStream::Stderr)] { + return ReplacedOutput::Filtered; + } + IN_OUTPUT_REPLACEMENT.with(|flag| flag.set(true)); + let replaced = Python::attach(|py| self.call(py, chunk)).unwrap_or(ReplacedOutput::Failed); + IN_OUTPUT_REPLACEMENT.with(|flag| flag.set(false)); + replaced + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs index 6c0c3bad72..e5cb911f75 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs @@ -37,6 +37,7 @@ use crate::bindings::_helpers::recall::constants::{ STDOUT_STREAM_INDEX, }; use crate::bindings::_helpers::recall::events::escaped_text; +use crate::bindings::_helpers::recall::replacements::{RecallReplacements, in_output_replacement}; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::rules::config_json::dumped_json; @@ -50,6 +51,8 @@ pub(crate) struct InvocationRecall { pub(crate) command: String, /// The resolved settings and project directory, for notices at the end of the run. pub(crate) settings: Option<(RecallSettings, PathBuf)>, + /// The Recall replacement sinks, configured once runtime extensions are discovered. + pub(crate) replacements: Arc, } /// The text of an optional attribute value. @@ -343,6 +346,7 @@ fn started_session( heartbeat_interval: heartbeat_interval(), setup_budget: Duration::ZERO, maintenance: true, + defer_context: true, }); Ok(Some(Arc::new(session))) } @@ -374,6 +378,7 @@ pub(crate) fn start_invocation_recall( adapter: settings.project_adapter.clone(), command: facts.command.clone().unwrap_or_default(), settings: Some((settings, project_dir)), + replacements: Arc::default(), }) } @@ -457,6 +462,9 @@ pub(crate) fn forward_output( let Some(session) = session.filter(|session| session.invocation_id() == invocation_id) else { return; }; + if in_output_replacement() { + return; + } let stream = if stream == STDOUT_STREAM_INDEX { OutputStream::Stdout } else { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs index 7373814272..5511815af6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs @@ -10,11 +10,29 @@ use sqlbuild_recall::writer::models::RecallSession; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::recall::events::{escaped_text, event_object}; +use crate::bindings::_helpers::recall::replacements::RecallReplacements; -/// Publishes lifecycle events and SQL artifacts of the active session; delivery is best effort. +/// Queue `event` in `session`; an event that cannot be encoded is skipped. +pub(crate) fn publish_lifecycle_event( + py: Python<'_>, + session: &RecallSession, + event: &Bound<'_, PyAny>, +) -> PyResult<()> { + if let Some(object) = event_object(event)? { + session.publish_event_object(object); + return Ok(()); + } + let encoded = host::entry(py, ExecutorHost::LifecycleEventToJson)?.call1((event,))?; + let text = escaped_text(encoded.downcast::()?)?; + drop(session.publish_event(&text)); + Ok(()) +} + +/// Publishes lifecycle events, through the Recall replacement sink when one is configured, and SQL artifacts of the active session; delivery is best effort. #[pyclass(module = "sqlbuild._native", frozen)] pub(crate) struct RecallLifecycleSink { pub(crate) session: Arc, + pub(crate) replacements: Arc, } #[pymethods] @@ -52,14 +70,13 @@ impl RecallLifecycleSink { impl RecallLifecycleSink { fn publish(&self, py: Python<'_>, event: &Bound<'_, PyAny>) -> PyResult<()> { - if let Some(object) = event_object(event)? { - self.session.publish_event_object(object); + if self + .replacements + .deliver_lifecycle(py, &self.session, event)? + { return Ok(()); } - let encoded = host::entry(py, ExecutorHost::LifecycleEventToJson)?.call1((event,))?; - let text = escaped_text(encoded.downcast::()?)?; - drop(self.session.publish_event(&text)); - Ok(()) + publish_lifecycle_event(py, &self.session, event) } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/writer.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/writer.rs new file mode 100644 index 0000000000..db228b2e63 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/writer.rs @@ -0,0 +1,139 @@ +//! `sqlbuild.recall.RecallWriter`: what a Recall replacement sink publishes through. + +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, PoisonError}; + +use pyo3::exceptions::{PyTypeError, PyValueError}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyclass, pymethods}; +use pyo3::types::{PyBytes, PyBytesMethods}; +use sqlbuild_recall::writer::models::RecallSession; + +use crate::bindings::_helpers::native_host::executor::ExecutorHost; +use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::recall::sinks::publish_lifecycle_event; +use crate::bindings::_helpers::records::fields::field as native_field; + +/// The one stream a writer publishes to. +#[derive(Debug)] +enum Target { + Lifecycle(Arc), + /// The chunk the sink was handed, by sequence, and what it published for it. + Output { + sequence: u64, + published: Mutex>>, + }, +} + +/// Publishes the records a Recall replacement sink keeps; valid only while the sink call runs. +#[pyclass(module = "sqlbuild._native", name = "RecallWriter", frozen)] +pub(crate) struct RecallWriter { + target: Target, + open: AtomicBool, + published: AtomicU64, +} + +impl RecallWriter { + /// A writer for one lifecycle event handed to the lifecycle replacement. + pub(crate) fn lifecycle(session: Arc) -> Self { + Self::new(Target::Lifecycle(session)) + } + + /// A writer for the output chunk with `sequence` handed to the output replacement. + pub(crate) fn output(sequence: u64) -> Self { + Self::new(Target::Output { + sequence, + published: Mutex::new(None), + }) + } + + fn new(target: Target) -> Self { + Self { + target, + open: AtomicBool::new(true), + published: AtomicU64::new(0), + } + } + + /// End the sink call: later publishing raises, and the records published are returned. + pub(crate) fn finish(&self) -> (u64, Option>) { + self.open.store(false, Ordering::SeqCst); + let data = match &self.target { + Target::Lifecycle(_) => None, + Target::Output { published, .. } => published + .lock() + .unwrap_or_else(PoisonError::into_inner) + .take(), + }; + (self.published.load(Ordering::SeqCst), data) + } + + fn ensure_open(&self) -> PyResult<()> { + if self.open.load(Ordering::SeqCst) { + return Ok(()); + } + Err(PyValueError::new_err( + "a RecallWriter can publish only while the sink call it was passed to runs", + )) + } +} + +#[pymethods] +impl RecallWriter { + /// Record `event` in Recall; for the lifecycle replacement sink. + fn publish_lifecycle(&self, py: Python<'_>, event: &Bound<'_, PyAny>) -> PyResult<()> { + self.ensure_open()?; + let Target::Lifecycle(session) = &self.target else { + return Err(PyValueError::new_err( + "this RecallWriter belongs to a command-output sink; use publish_command_output", + )); + }; + if !event.is_instance(&host::entry(py, ExecutorHost::LifecycleEvent)?)? { + return Err(PyTypeError::new_err( + "publish_lifecycle expects a LifecycleEvent", + )); + } + publish_lifecycle_event(py, session, event)?; + self.published.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + + /// Record `record.data` in Recall as the output chunk the sink was passed; at most once per call. + fn publish_command_output(&self, py: Python<'_>, record: &Bound<'_, PyAny>) -> PyResult<()> { + self.ensure_open()?; + let Target::Output { + sequence, + published, + } = &self.target + else { + return Err(PyValueError::new_err( + "this RecallWriter belongs to a lifecycle-event sink; use publish_lifecycle", + )); + }; + if !record.is_instance(&host::entry(py, ExecutorHost::CommandOutputRecord)?)? { + return Err(PyTypeError::new_err( + "publish_command_output expects a CommandOutputRecord", + )); + } + let record_sequence: u64 = native_field(record, "sequence")?.extract()?; + if record_sequence != *sequence { + return Err(PyValueError::new_err(format!( + "publish_command_output expects the record passed to this call (sequence \ + {sequence}) or a copy made with record.replace(), not sequence {record_sequence}" + ))); + } + let data: Vec = native_field(record, "data")? + .downcast::() + .map_err(|_| PyTypeError::new_err("record.data must be bytes"))? + .as_bytes() + .to_vec(); + let mut slot = published.lock().unwrap_or_else(PoisonError::into_inner); + if slot.is_some() { + return Err(PyValueError::new_err( + "a command-output record can be published once per sink call", + )); + } + *slot = Some(data); + self.published.fetch_add(1, Ordering::SeqCst); + Ok(()) + } +} diff --git a/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs b/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs index ff24197b78..b2ecc16eb3 100644 --- a/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs +++ b/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs @@ -1,6 +1,6 @@ //! The delivery thread: table setup, batched writes with retries, heartbeats, the final delivery rows and maintenance. It alone opens and uses the store connection. -use std::sync::{Arc, Condvar, Mutex, MutexGuard}; +use std::sync::{Arc, Condvar, Mutex, MutexGuard, OnceLock}; use std::time::{Duration, Instant, SystemTime}; use crate::records::main::delivery_row::delivery_row; @@ -29,6 +29,7 @@ use crate::writer::constants::{ SETUP_RETRY_MS, }; use crate::writer::models::{DeliveryStream, SessionConfig, SessionStore, SetupOutcome}; +use crate::writer::types::OutputReplacement; /// State shared by producers, the session handle and the delivery thread. #[derive(Debug)] @@ -37,6 +38,8 @@ pub(crate) struct Shared { pub(crate) wake: Condvar, pub(crate) setup: Mutex>, pub(crate) setup_done: Condvar, + /// The replacement sink output passes through, once one is configured. + pub(crate) output_replacement: OnceLock>, } pub(crate) fn locked(mutex: &Mutex) -> MutexGuard<'_, WriterState> { @@ -322,11 +325,14 @@ pub(crate) fn deliver(shared: &Arc, config: &SessionConfig) { let batching: Batching = batching(config); loop { let (batch, closing) = next_batch(shared, next_heartbeat, batching); - let built: Built = materializer.build(batch); - if !built.settled.is_empty() || !built.unbuildable.is_empty() { + let built: Built = + materializer.build(batch, shared.output_replacement.get().map(Arc::as_ref)); + if !built.settled.is_empty() || !built.unbuildable.is_empty() || !built.filtered.is_empty() + { let mut state = locked(&shared.state); state.account(&built.settled, Ok(&InsertOutcome::default())); state.account_unbuildable(&built.unbuildable); + state.account_filtered(&built.filtered); } let mut rows: Vec = built.rows; let now: SystemTime = SystemTime::now(); diff --git a/crates/sqlbuild-recall/src/writer/_helpers/materialize.rs b/crates/sqlbuild-recall/src/writer/_helpers/materialize.rs index 7038867b34..f03c812a3e 100644 --- a/crates/sqlbuild-recall/src/writer/_helpers/materialize.rs +++ b/crates/sqlbuild-recall/src/writer/_helpers/materialize.rs @@ -8,10 +8,13 @@ use crate::records::constants::{ use crate::records::main::artifact_rows::artifact_rows; use crate::records::main::event_object_row::event_object_row; use crate::records::main::format_timestamp::format_timestamp; +use crate::records::main::output_row::output_row; use crate::records::main::sha256_hex::sha256_hex; -use crate::records::models::{ArtifactInput, RecallRow, RowEnvelope}; +use crate::records::models::{ArtifactInput, OutputChunk, OutputStream, RecallRow, RowEnvelope}; +use crate::writer::_helpers::output_chunk::output_chunk; use crate::writer::_helpers::queue::{Payload, Queued, SqlInput}; -use crate::writer::models::DeliveryStream; +use crate::writer::models::{DeliveryStream, ReplacedOutput}; +use crate::writer::types::OutputReplacement; /// A batch ready to write: its rows, the streams of the records they came from, and records already settled. #[derive(Debug, Default)] @@ -21,8 +24,10 @@ pub(crate) struct Built { pub(crate) written: Vec, /// Records that are already stored, such as SQL this invocation recorded before. pub(crate) settled: Vec, - /// Records whose rows could not be built. + /// Records whose rows could not be built, or that a replacement sink failed on. pub(crate) unbuildable: Vec, + /// Records a replacement sink chose not to store. + pub(crate) filtered: Vec, } /// Builds rows for one session; remembers which SQL it has stored. @@ -30,6 +35,8 @@ pub(crate) struct Built { pub(crate) struct Materializer { envelope: RowEnvelope, artifacts_seen: HashSet, + /// Output bytes stored per stream, for the offsets of replaced chunks. + output_bytes: [u64; 2], } impl Materializer { @@ -37,9 +44,52 @@ impl Materializer { Self { envelope, artifacts_seen: HashSet::new(), + output_bytes: [0; 2], } } + /// The output rows to store for `rows`, passed through `replacement`: what it published, under the chunk's sequence. + fn output_rows( + &mut self, + rows: Vec, + replacement: Option<&dyn OutputReplacement>, + ) -> Result, ReplacedOutput> { + let mut stored: Vec = Vec::with_capacity(rows.len()); + for row in rows { + let Some(chunk) = output_chunk(&row) else { + stored.push(row); + continue; + }; + let index = usize::from(chunk.stream == OutputStream::Stderr); + let Some(replacement) = replacement else { + self.output_bytes[index] += chunk.data.len() as u64; + stored.push(row); + continue; + }; + if chunk.omitted.is_some() { + let gap = OutputChunk { + byte_offset: self.output_bytes[index], + ..chunk + }; + stored.push(output_row(&self.envelope, &gap)); + continue; + } + match replacement.replace(&chunk) { + ReplacedOutput::Published(data) => { + let replaced = OutputChunk { + byte_offset: self.output_bytes[index], + data, + ..chunk + }; + self.output_bytes[index] += replaced.data.len() as u64; + stored.push(output_row(&self.envelope, &replaced)); + } + other => return Err(other), + } + } + Ok(stored) + } + fn sql_rows(&mut self, input: SqlInput) -> Option> { let digest = format!( "{ARTIFACT_DIGEST_PREFIX}{}", @@ -63,11 +113,25 @@ impl Materializer { } /// The rows of `batch`, with each record's stream sorted by how it is accounted. - pub(crate) fn build(&mut self, batch: Vec) -> Built { + pub(crate) fn build( + &mut self, + batch: Vec, + replacement: Option<&dyn OutputReplacement>, + ) -> Built { let mut built = Built::default(); for record in batch { let stream: DeliveryStream = record.stream; match record.payload { + Payload::Rows(rows) if stream == DeliveryStream::Output => { + match self.output_rows(rows, replacement) { + Ok(rows) => { + built.rows.extend(rows); + built.written.push(stream); + } + Err(ReplacedOutput::Filtered) => built.filtered.push(stream), + Err(_) => built.unbuildable.push(stream), + } + } Payload::Rows(rows) => { built.rows.extend(rows); built.written.push(stream); diff --git a/crates/sqlbuild-recall/src/writer/_helpers/mod.rs b/crates/sqlbuild-recall/src/writer/_helpers/mod.rs index 2115f0b0b2..d36a2aa4d3 100644 --- a/crates/sqlbuild-recall/src/writer/_helpers/mod.rs +++ b/crates/sqlbuild-recall/src/writer/_helpers/mod.rs @@ -1,5 +1,6 @@ pub(crate) mod delivery; pub(crate) mod materialize; +pub(crate) mod output_chunk; pub(crate) mod queue; pub(crate) mod session; pub(crate) mod tail; diff --git a/crates/sqlbuild-recall/src/writer/_helpers/output_chunk.rs b/crates/sqlbuild-recall/src/writer/_helpers/output_chunk.rs new file mode 100644 index 0000000000..4e196c4088 --- /dev/null +++ b/crates/sqlbuild-recall/src/writer/_helpers/output_chunk.rs @@ -0,0 +1,45 @@ +//! The output chunk a queued `recall_output` row was built from. + +use crate::records::constants::{ + DROPPED_BEFORE_COLUMN, OMITTED_BYTES_COLUMN, OMITTED_SEQUENCES_COLUMN, +}; +use crate::records::models::{OmittedOutput, OutputChunk, OutputStream, RecallRow, RecallValue}; +use crate::schema::models::Table; + +fn count(row: &RecallRow, column: &str) -> Option { + match row.value(column)? { + RecallValue::Integer(number) if *number >= 0 => Some(number.unsigned_abs()), + _ => None, + } +} + +fn text<'a>(row: &'a RecallRow, column: &str) -> Option<&'a str> { + match row.value(column)? { + RecallValue::Text(text) => Some(text), + _ => None, + } +} + +/// The chunk `row` stores, `None` for a row of another table. +pub(crate) fn output_chunk(row: &RecallRow) -> Option { + if row.table != Table::Output { + return None; + } + let data: Vec = match row.value("data")? { + RecallValue::Blob(data) => data.clone(), + RecallValue::Text(text) => text.as_bytes().to_vec(), + _ => return None, + }; + Some(OutputChunk { + sequence: count(row, "sequence")?, + stream: OutputStream::from_name(text(row, "stream")?)?, + occurred_at: text(row, "occurred_at")?.to_owned(), + data, + byte_offset: count(row, "byte_offset")?, + dropped_before: count(row, DROPPED_BEFORE_COLUMN)?, + omitted: count(row, OMITTED_BYTES_COLUMN).map(|bytes| OmittedOutput { + bytes, + sequences: count(row, OMITTED_SEQUENCES_COLUMN).unwrap_or(1), + }), + }) +} diff --git a/crates/sqlbuild-recall/src/writer/_helpers/queue.rs b/crates/sqlbuild-recall/src/writer/_helpers/queue.rs index ed710a05bf..c9d9648d02 100644 --- a/crates/sqlbuild-recall/src/writer/_helpers/queue.rs +++ b/crates/sqlbuild-recall/src/writer/_helpers/queue.rs @@ -7,9 +7,11 @@ use serde_json::{Map, Value}; use crate::records::constants::{DROPPED_BEFORE_COLUMN, MAX_OUTPUT_CHUNK_BYTES}; use crate::records::main::format_timestamp::format_timestamp; +use crate::records::main::invocation_row::invocation_row; use crate::records::main::output_row::output_row; use crate::records::models::{ - DeliveryCounts, OutputChunk, OutputStream, RecallRow, RecallValue, RowEnvelope, + DeliveryCounts, InvocationContext, OutputChunk, OutputStream, RecallRow, RecallValue, + RowEnvelope, }; use crate::store::models::InsertOutcome; use crate::writer::_helpers::delivery::stream_of; @@ -97,6 +99,8 @@ pub(crate) struct WriterState { pub(crate) degraded: Option, /// Store notices the user should see, such as a corrupt store moved aside. pub(crate) notices: Vec, + /// The context row's content while its replacement-filter flag is still unknown. + pub(crate) held_context: Option, capacity: (usize, usize), } @@ -141,6 +145,7 @@ impl WriterState { rejecting: false, degraded: None, notices: Vec::new(), + held_context: None, capacity, } } @@ -421,6 +426,22 @@ impl WriterState { self.pending.is_some() } + /// Count records a replacement sink chose not to store as filtered. + pub(crate) fn account_filtered(&mut self, streams: &[DeliveryStream]) { + for stream in streams { + self.counts_mut(*stream).filtered += 1; + } + } + + /// Queue the held context row, recording whether a replacement sink filters this invocation. + pub(crate) fn release_context(&mut self, filter_active: bool) { + if let Some(mut context) = self.held_context.take() { + context.recall_filter_active = filter_active; + let row = invocation_row(&self.envelope, &context); + self.push(vec![row], DeliveryStream::Invocation, Priority::Context); + } + } + /// Count records the delivery thread could not turn into rows as failed. pub(crate) fn account_unbuildable(&mut self, streams: &[DeliveryStream]) { for stream in streams { diff --git a/crates/sqlbuild-recall/src/writer/_helpers/session.rs b/crates/sqlbuild-recall/src/writer/_helpers/session.rs index f656bb31cc..8a4b470c87 100644 --- a/crates/sqlbuild-recall/src/writer/_helpers/session.rs +++ b/crates/sqlbuild-recall/src/writer/_helpers/session.rs @@ -1,5 +1,6 @@ //! The session handle: producers queue records; `close` flushes within the shutdown budget. +use std::sync::Arc; use std::time::{Duration, Instant, SystemTime}; use serde_json::{Map, Value}; @@ -17,6 +18,7 @@ use crate::writer::constants::{JSON_SCALAR_BYTES, MAX_ARTIFACT_BYTES}; use crate::writer::models::{ DeliveryStream, DeliverySummary, Priority, RecallSession, SetupOutcome, }; +use crate::writer::types::OutputReplacement; pub(crate) fn severity_rank(severity: &str) -> usize { SEVERITY_ORDER @@ -152,6 +154,18 @@ impl RecallSession { self.shared.wake.notify_one(); } + /// Queue the context row held by `SessionConfig::defer_context`, with whether a Recall replacement sink filters this invocation; later calls do nothing. + pub fn publish_context(&self, filter_active: bool) { + let mut state = locked(&self.shared.state); + state.release_context(filter_active); + self.shared.wake.notify_one(); + } + + /// Pass every output chunk produced from now on through `replacement` before it is stored; only the first replacement is kept. + pub fn set_output_replacement(&self, replacement: Arc) { + drop(self.shared.output_replacement.set(replacement)); + } + /// Count records a replacement sink filtered out of `stream`. pub fn record_filtered(&self, stream: DeliveryStream, count: u64) { let mut state = locked(&self.shared.state); @@ -186,6 +200,7 @@ impl RecallSession { pub fn begin_close(&self) { let mut state = locked(&self.shared.state); if !state.closing { + state.release_context(false); state.closing = true; state.deadline = Some(Instant::now() + self.shutdown_timeout); state.flush_tails(); diff --git a/crates/sqlbuild-recall/src/writer/main/start_recall_session.rs b/crates/sqlbuild-recall/src/writer/main/start_recall_session.rs index e388056c3c..e1ecf8d7da 100644 --- a/crates/sqlbuild-recall/src/writer/main/start_recall_session.rs +++ b/crates/sqlbuild-recall/src/writer/main/start_recall_session.rs @@ -1,6 +1,6 @@ //! `start_recall_session`: begin recording one invocation. -use std::sync::{Arc, Condvar, Mutex}; +use std::sync::{Arc, Condvar, Mutex, OnceLock}; use crate::records::main::invocation_row::invocation_row; use crate::writer::_helpers::delivery::{Shared, deliver, locked}; @@ -8,21 +8,26 @@ use crate::writer::_helpers::queue::WriterState; use crate::writer::_helpers::session::severity_rank; use crate::writer::models::{DeliveryStream, Priority, RecallSession, SessionConfig}; -/// Start the delivery thread and queue the invocation's context row first. +/// Start the delivery thread and queue the invocation's context row first, unless the config defers it. #[must_use] pub fn start_recall_session(config: SessionConfig) -> RecallSession { let mut state = WriterState::new(config.envelope.clone()).with_output_limit(config.max_output_bytes); - state.push( - vec![invocation_row(&config.envelope, &config.context)], - DeliveryStream::Invocation, - Priority::Context, - ); + if config.defer_context { + state.held_context = Some(config.context.clone()); + } else { + state.push( + vec![invocation_row(&config.envelope, &config.context)], + DeliveryStream::Invocation, + Priority::Context, + ); + } let shared = Arc::new(Shared { state: Mutex::new(state), wake: Condvar::new(), setup: Mutex::new(None), setup_done: Condvar::new(), + output_replacement: OnceLock::new(), }); let session = RecallSession { shared: Arc::clone(&shared), diff --git a/crates/sqlbuild-recall/src/writer/mod.rs b/crates/sqlbuild-recall/src/writer/mod.rs index 9a334c477b..1a5c762182 100644 --- a/crates/sqlbuild-recall/src/writer/mod.rs +++ b/crates/sqlbuild-recall/src/writer/mod.rs @@ -4,6 +4,7 @@ pub(crate) mod _helpers; pub mod constants; pub mod main; pub mod models; +pub mod types; #[cfg(test)] mod tests; diff --git a/crates/sqlbuild-recall/src/writer/models.rs b/crates/sqlbuild-recall/src/writer/models.rs index 6e53198889..3ba49c8f4e 100644 --- a/crates/sqlbuild-recall/src/writer/models.rs +++ b/crates/sqlbuild-recall/src/writer/models.rs @@ -68,6 +68,17 @@ impl Priority { pub const COUNT: usize = 8; } +/// What a Recall replacement made of one output chunk. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ReplacedOutput { + /// Store these bytes under the chunk's sequence. + Published(Vec), + /// Store nothing; counted as filtered. + Filtered, + /// Store nothing; counted as failed. + Failed, +} + /// Where a session writes. #[derive(Clone, Debug)] pub enum SessionStore { @@ -92,6 +103,8 @@ pub struct SessionConfig { pub setup_budget: Duration, /// Whether this session runs retention maintenance after delivery. pub maintenance: bool, + /// Hold the context row until `RecallSession::publish_context` says whether a replacement filters history. + pub defer_context: bool, pub maintenance_owner: String, } diff --git a/crates/sqlbuild-recall/src/writer/tests/helpers.rs b/crates/sqlbuild-recall/src/writer/tests/helpers.rs index 9c4370107e..9eda3f65f4 100644 --- a/crates/sqlbuild-recall/src/writer/tests/helpers.rs +++ b/crates/sqlbuild-recall/src/writer/tests/helpers.rs @@ -1,4 +1,5 @@ use std::path::PathBuf; +use std::sync::Arc; use std::time::{Duration, SystemTime}; use crate::records::main::invocation_row::invocation_row; @@ -14,7 +15,7 @@ use crate::writer::main::start_recall_session::start_recall_session; use crate::writer::models::{ DeliveryStream, DeliverySummary, Priority, SessionConfig, SessionStore, SetupOutcome, }; -use crate::writer::tests::test_types::RecordedSession; +use crate::writer::tests::test_types::{PrefixReplacement, RecordedSession}; /// The envelope of invocation `invocation_id`. pub(crate) fn envelope(invocation_id: &str) -> RowEnvelope { @@ -88,6 +89,7 @@ pub(crate) fn config(path: PathBuf, invocation_id: &str) -> SessionConfig { setup_budget: Duration::from_secs(2), maintenance: true, maintenance_owner: "test:42".to_owned(), + defer_context: false, } } @@ -341,3 +343,32 @@ pub(crate) fn output_stored_while_running( drop(session.close()); stored.rows } + +/// The transcript and output accounting a session with a deferred context and `PrefixReplacement` stored for `output`. +pub(crate) fn replaced_session( + output: &[(OutputStream, &[u8])], +) -> (Vec, Vec>) { + let directory = tempfile::tempdir().expect("a temporary directory can be created"); + let path = directory.path().join("logs/recall.sqlite3"); + let session = start_recall_session(SessionConfig { + defer_context: true, + ..config(path.clone(), "replaced") + }); + session.set_output_replacement(Arc::new(PrefixReplacement)); + session.publish_context(true); + for (stream, data) in output { + session.append_output(*stream, data); + } + drop(session.close()); + let store = open_sqlite_store(&path, OpenMode::ReadOnly).expect("the store opens"); + let transcript = + read_output_transcript(&store, "replaced", None).expect("the transcript can be read"); + let facts = query_store( + &store, + "SELECT filtered, failed, missing_sequences, filter_active, gap_reason \ + FROM output_invocations", + None, + ) + .expect("the query is valid"); + (transcript.data, facts.rows) +} diff --git a/crates/sqlbuild-recall/src/writer/tests/test_session.rs b/crates/sqlbuild-recall/src/writer/tests/test_session.rs index f5e59501ce..f215cd128d 100644 --- a/crates/sqlbuild-recall/src/writer/tests/test_session.rs +++ b/crates/sqlbuild-recall/src/writer/tests/test_session.rs @@ -7,13 +7,13 @@ use crate::writer::models::SetupOutcome; use crate::writer::tests::helpers::unwritable_session; use crate::writer::tests::helpers::{ artifact_counts, close_under_held_lock, event_counts, output_stored_while_running, - recorded_session, session_on_corrupt_store, + recorded_session, replaced_session, session_on_corrupt_store, }; #[cfg(unix)] use crate::writer::tests::test_types::UnwritableTestCase; use crate::writer::tests::test_types::{ ArtifactSizeTestCase, CorruptStoreTestCase, EventBuildTestCase, HeldLockTestCase, - RecordedSession, SessionTestCase, StoredWhileRunningTestCase, + RecordedSession, ReplacementTestCase, SessionTestCase, StoredWhileRunningTestCase, }; const CLOSE_SLACK_SECONDS: f64 = 0.3; @@ -205,3 +205,36 @@ fn given_heartbeats_due_faster_than_batches_fill_when_running_then_queued_record ); } } + +#[test] +fn given_an_output_replacement_when_closed_then_published_filtered_and_failed_chunks_are_accounted() +{ + let test_cases = [ReplacementTestCase { + description: "one chunk published changed, one filtered, one failed", + output: &[ + (OutputStream::Stderr, b"drop that\n"), + (OutputStream::Stdout, b"boom\n"), + (OutputStream::Stderr, b"keep this\n"), + ], + expected_transcript: b"KEEP THIS\n", + expected_accounting: vec![vec![ + RecallValue::Integer(1), + RecallValue::Integer(1), + RecallValue::Integer(2), + RecallValue::Integer(1), + RecallValue::Text("dropped".to_owned()), + ]], + }]; + + for test_case in test_cases { + assert_eq!( + replaced_session(test_case.output), + ( + test_case.expected_transcript.to_vec(), + test_case.expected_accounting + ), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-recall/src/writer/tests/test_types.rs b/crates/sqlbuild-recall/src/writer/tests/test_types.rs index a1c8ea19f5..bd727343b8 100644 --- a/crates/sqlbuild-recall/src/writer/tests/test_types.rs +++ b/crates/sqlbuild-recall/src/writer/tests/test_types.rs @@ -1,5 +1,6 @@ -use crate::records::models::{OutputStream, RecallValue}; -use crate::writer::models::{DeliveryStream, Priority, SetupOutcome}; +use crate::records::models::{OutputChunk, OutputStream, RecallValue}; +use crate::writer::models::{DeliveryStream, Priority, ReplacedOutput, SetupOutcome}; +use crate::writer::types::OutputReplacement; /// What one recorded session left in the store. #[derive(Debug, PartialEq)] @@ -83,3 +84,26 @@ pub(crate) struct StoredWhileRunningTestCase { pub(crate) wait: std::time::Duration, pub(crate) expected_rows: Vec>, } + +pub(crate) struct ReplacementTestCase { + pub(crate) description: &'static str, + pub(crate) output: &'static [(OutputStream, &'static [u8])], + pub(crate) expected_transcript: &'static [u8], + pub(crate) expected_accounting: Vec>, +} + +/// Publishes `keep…` chunks upper-cased, filters `drop…` chunks and fails on anything else. +#[derive(Debug)] +pub(crate) struct PrefixReplacement; + +impl OutputReplacement for PrefixReplacement { + fn replace(&self, chunk: &OutputChunk) -> ReplacedOutput { + if chunk.data.starts_with(b"keep") { + ReplacedOutput::Published(chunk.data.to_ascii_uppercase()) + } else if chunk.data.starts_with(b"drop") { + ReplacedOutput::Filtered + } else { + ReplacedOutput::Failed + } + } +} diff --git a/crates/sqlbuild-recall/src/writer/types.rs b/crates/sqlbuild-recall/src/writer/types.rs new file mode 100644 index 0000000000..b1b7defa5d --- /dev/null +++ b/crates/sqlbuild-recall/src/writer/types.rs @@ -0,0 +1,10 @@ +//! The hook a Recall replacement sink plugs into delivery. + +use crate::records::models::OutputChunk; +use crate::writer::models::ReplacedOutput; + +/// Decides, on the delivery thread, what Recall stores for each output chunk. +pub trait OutputReplacement: Send + Sync + std::fmt::Debug { + /// What to store for `chunk`: its replacement bytes, nothing, or nothing because the replacement failed. + fn replace(&self, chunk: &OutputChunk) -> ReplacedOutput; +} diff --git a/fensu.toml b/fensu.toml index 8a3dfb12ec..103bb85c38 100644 --- a/fensu.toml +++ b/fensu.toml @@ -961,8 +961,10 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/query_command/query_values.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/recall/console.rs=10", + "crates/sqlbuild-python/src/bindings/_helpers/recall/replacements.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/recall/writer.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/auditing.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/compiled.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/cursors.rs=1", @@ -1130,7 +1132,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/cli/argv.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs=11", + "crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs=12", "crates/sqlbuild-python/src/bindings/_helpers/cli/progress.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/cli/progress_reporters.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs=4", @@ -1292,6 +1294,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/models.rs=15", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/recall/replacements.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/records/error_classes.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/records/models.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_compare.rs=2", diff --git a/python-boundary.toml b/python-boundary.toml index 5857186522..8fbb3795d5 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -302,6 +302,11 @@ path = "runtime/event_exporting/main/_run_command_output_sinks.py" rule = 2 note = "Rule 2 shim: target of each dispatcher's single command-output sink thread; waits on a Python queue and calls user sinks in order on that one thread (provider thread affinity), reporting each outcome through a Python queue, so no native frame is on that thread when finalization can end it (event_exporting/dispatcher.rs)." +[[stays]] +path = "runtime/event_exporting/main/_invoke_recall_sink.py" +rule = 2 +note = "Thin shim that calls a user Recall replacement sink with its record and RecallWriter and reports whether it raised." + [[stays]] path = "runtime/event_exporting/models.py" rule = 3 @@ -482,6 +487,11 @@ path = "compiler/discovery/_helpers/filesystem/command_output_sinks.py" rule = 2 note = "Inspects loaded user sink functions." +[[stays]] +path = "compiler/discovery/_helpers/filesystem/recall_sinks.py" +rule = 2 +note = "Inspects loaded user sink functions for a RecallWriter parameter." + [[stays]] path = "compiler/pipeline/_helpers/materializations.py" rule = 2 @@ -622,6 +632,11 @@ path = "retries.py" rule = 3 note = "Public facade: re-exports only." +[[stays]] +path = "recall.py" +rule = 3 +note = "Public facade: re-exports only." + [[stays]] path = "sinks.py" rule = 3 diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md index 761517b7cf..f995b1aac5 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md @@ -14,6 +14,7 @@ Online: https://sqlbuild.com/docs/concepts/observability/recall/ - Retention - When the store cannot be written - Sharing history in ClickHouse +- Filtering what Recall stores - Querying history - Relationship to compute logs @@ -160,6 +161,36 @@ retention = "30d" Use a single ClickHouse server or ClickHouse Cloud; `ON CLUSTER` stores are not supported yet. +## Filtering what Recall stores + +By default Recall stores every lifecycle event and every output byte. To decide yourself, write a +[sink](sinks.md) with a parameter annotated `RecallWriter`: it replaces +Recall's default for its stream, and Recall stores only what it publishes. + +```python +from sqlbuild.recall import RecallWriter +from sqlbuild.sinks import CommandOutputRecord, command_output_sink + +@command_output_sink +def recall_output(record: CommandOutputRecord, writer: RecallWriter) -> None: + redacted: bytes = record.data.replace(b"secret-token", b"[redacted]") + writer.publish_command_output(record.replace(data=redacted)) +``` + +- A `@command_output_sink` replacement receives each output chunk with its exact bytes in + `record.data` (record schema version 2). `record.replace(data=…, message=…)` returns a copy that + keeps the record ID and sequence; publish at most one record per call. +- A `@lifecycle_event_sink` replacement receives each event its `event_kinds` and `min_severity` + accept and keeps it with `writer.publish_lifecycle(event)`. +- A record the sink does not publish is counted as `filtered`; a sink that raises has its record + counted as `failed`, and Recall never falls back to storing the unfiltered record. The command's + exit code and output are unaffected. `output_invocations.filter_active` is `1` for invocations + that ran a replacement. +- There is at most one replacement per stream; discovery fails with `D015` naming each + `path:function` otherwise. A replacement takes only its record and the writer, not providers, and + the provider name `recall_writer` is reserved. +- Other sinks keep receiving every record. With `enabled = false`, no replacement runs. + ## Querying history Use [`sqb recall`](../../cli/recall.md) to read the store. Queries are read-only `SELECT` statements, diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/sinks.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/sinks.md index 328cdc4752..aa0c9d1058 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/sinks.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/sinks.md @@ -6,6 +6,13 @@ Online: https://sqlbuild.com/docs/concepts/observability/sinks/ +## Contents + +- Lifecycle sink +- Command-output sink +- Configuration +- Delivery behavior + Project sinks consume one declared record type. Lifecycle and command-output streams remain separate even when they share a provider and destination. @@ -59,6 +66,10 @@ Command output groups adjacent text into bounded byte chunks and flushes on elap change, size, or close. A loss record reports bounded-queue drops. This stream is useful for remote transcripts but is not lifecycle evidence. +A sink with a parameter annotated `sqlbuild.recall.RecallWriter` is not an ordinary sink: it +decides what [Recall](recall.md#filtering-what-recall-stores) stores for +its stream. + ## Configuration Runtime configuration can narrow declaration filters: diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index f163e88da3..19bf31f222 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -3331,6 +3331,12 @@ class RecallLifecycleSink: self, *, sql: str, statement_id: str | None = None, resource_id: str | None = None ) -> None: ... +class RecallWriter: + """Publishes the records a Recall replacement sink keeps; valid only during the sink call.""" + + def publish_lifecycle(self, event: LifecycleEvent) -> None: ... + def publish_command_output(self, record: CommandOutputRecord) -> None: ... + class LocalFilesystemComputeLogStorage: """Ordered fsync capture storage subject to underlying filesystem and hardware guarantees.""" @@ -16925,12 +16931,16 @@ class DiscoveredRuntimeExtensions(NativeRecord): event_exporters: tuple[DiscoveredEventExporter, ...] command_output_sinks: tuple[DiscoveredCommandOutputSink, ...] lifecycle_shutdown_timeout_seconds: int | None + recall_lifecycle_sink: tuple[DiscoveredEventExporter, str] | None + recall_output_sink: tuple[DiscoveredCommandOutputSink, str] | None def __init__( self, providers: tuple[DiscoveredProvider, ...] = ..., event_exporters: tuple[DiscoveredEventExporter, ...] = ..., command_output_sinks: tuple[DiscoveredCommandOutputSink, ...] = ..., lifecycle_shutdown_timeout_seconds: int | None = ..., + recall_lifecycle_sink: tuple[DiscoveredEventExporter, str] | None = ..., + recall_output_sink: tuple[DiscoveredCommandOutputSink, str] | None = ..., ) -> None: ... class ProjectGraph: diff --git a/src/sqlbuild/_native_host.py b/src/sqlbuild/_native_host.py index a7fd8fdeb1..537b5a86e5 100644 --- a/src/sqlbuild/_native_host.py +++ b/src/sqlbuild/_native_host.py @@ -288,6 +288,10 @@ "sqlbuild.runtime.event_exporting.main._invoke_lifecycle_event_sink", "invoke_lifecycle_event_sink", ), + "invoke_recall_sink": ( + "sqlbuild.runtime.event_exporting.main._invoke_recall_sink", + "invoke_recall_sink", + ), "run_command_output_sinks": ( "sqlbuild.runtime.event_exporting.main._run_command_output_sinks", "run_command_output_sinks", diff --git a/src/sqlbuild/_native_host.pyi b/src/sqlbuild/_native_host.pyi index 3f51f2462b..cdd3aea788 100644 --- a/src/sqlbuild/_native_host.pyi +++ b/src/sqlbuild/_native_host.pyi @@ -76,18 +76,18 @@ from sqlbuild.adapter.type_system.main.safe_cast_nullability import ( safe_cast_nullability as safe_cast_nullability, ) from sqlbuild.adapters.clickhouse._helpers.recall_store import ( - connect_recall_store as connect_recall_store, + close_recall_store as close_recall_store, ) -from sqlbuild.adapters.clickhouse._helpers.recall_store import run_recall_query as run_recall_query from sqlbuild.adapters.clickhouse._helpers.recall_store import ( - run_recall_command as run_recall_command, + connect_recall_store as connect_recall_store, ) from sqlbuild.adapters.clickhouse._helpers.recall_store import ( insert_recall_rows as insert_recall_rows, ) from sqlbuild.adapters.clickhouse._helpers.recall_store import ( - close_recall_store as close_recall_store, + run_recall_command as run_recall_command, ) +from sqlbuild.adapters.clickhouse._helpers.recall_store import run_recall_query as run_recall_query from sqlbuild.cli.commands.main.entrypoint._dispatch_with_compute_logs import ( dispatch_with_compute_logs as dispatch_with_compute_logs, ) @@ -508,6 +508,9 @@ from sqlbuild.runtime.event_exporting.exceptions import ( from sqlbuild.runtime.event_exporting.main._invoke_lifecycle_event_sink import ( invoke_lifecycle_event_sink as invoke_lifecycle_event_sink, ) +from sqlbuild.runtime.event_exporting.main._invoke_recall_sink import ( + invoke_recall_sink as invoke_recall_sink, +) from sqlbuild.runtime.event_exporting.main._run_command_output_sinks import ( run_command_output_sinks as run_command_output_sinks, ) diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/command_output_sinks.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/command_output_sinks.py index 45f154939a..b2604f76b5 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/command_output_sinks.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/command_output_sinks.py @@ -16,11 +16,13 @@ DiscoveredProvider, DiscoveredProviderUsage, EventExporterDiscoveryError, + RecallWriter, ) from sqlbuild.compiler.discovery._helpers.filesystem.project_modules import ( load_sink_module, provider_usages, ) +from sqlbuild.compiler.discovery._helpers.filesystem.recall_sinks import split_recall_sinks from sqlbuild.runtime.event_exporting.main.get_command_output_sink_definition import ( get_command_output_sink_definition, ) @@ -31,7 +33,7 @@ def discover_command_output_sink_functions( *, project_dir: Path, providers: tuple[DiscoveredProvider, ...] = () ) -> tuple[DiscoveredCommandOutputSink, ...]: - """Discover validated command-output sink declarations under sinks/.""" + """Discover validated command-output sink declarations under sinks/, without Recall's.""" cached: ( tuple[ @@ -46,11 +48,15 @@ def discover_command_output_sink_functions( declarations: tuple[DiscoveredCommandOutputSinkDeclaration, ...] = ( discover_command_output_sink_declarations(project_dir=project_dir) ) - return bind_command_output_sink_declarations( - declarations=declarations, - providers=providers, - project_dir=project_dir, + _, ordinary = split_recall_sinks( + sinks=bind_command_output_sink_declarations( + declarations=declarations, + providers=providers, + project_dir=project_dir, + ), + history="command-output", ) + return ordinary def discover_command_output_sink_declarations( @@ -243,6 +249,8 @@ def _bind_sink_provider_usages( ) type_hints: dict[str, object] = get_type_hints(function) for parameter in parameters[1:]: + if type_hints.get(parameter.name, parameter.annotation) is RecallWriter: + continue provider: DiscoveredProvider | None = provider_by_name.get(parameter.name) if provider is None: raise EventExporterDiscoveryError( diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/recall_sinks.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/recall_sinks.py new file mode 100644 index 0000000000..c832bc13dc --- /dev/null +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/recall_sinks.py @@ -0,0 +1,67 @@ +"""Recall replacement sinks: sinks that take a ``RecallWriter`` and decide what Recall stores.""" + +from __future__ import annotations + +import inspect +from collections.abc import Callable +from typing import get_type_hints + +from sqlbuild._native import ( + DiscoveredCommandOutputSink, + DiscoveredEventExporter, + EventExporterDiscoveryError, + RecallWriter, +) + + +def _writer_parameters(function: Callable[..., object]) -> tuple[str, ...]: + try: + type_hints: dict[str, object] = get_type_hints(function) + except (NameError, TypeError): + type_hints = {} + return tuple( + name + for name, parameter in inspect.signature(function).parameters.items() + if type_hints.get(name, parameter.annotation) is RecallWriter + ) + + +def _location(sink: DiscoveredEventExporter | DiscoveredCommandOutputSink) -> str: + return f"{sink.relative_path}:{getattr(sink.function, '__name__', sink.name)}" + + +def _writer_parameter(sink: DiscoveredEventExporter | DiscoveredCommandOutputSink) -> str | None: + writers: tuple[str, ...] = _writer_parameters(sink.function) + if not writers: + return None + if len(writers) > 1: + raise EventExporterDiscoveryError( + f"Recall replacement sink {_location(sink)} declares more than one RecallWriter " + f"parameter: {', '.join(writers)}" + ) + if sink.provider_usages: + raise EventExporterDiscoveryError( + f"Recall replacement sink {_location(sink)} must not declare provider parameters" + ) + return writers[0] + + +def split_recall_sinks[SinkT: (DiscoveredEventExporter, DiscoveredCommandOutputSink)]( + *, sinks: tuple[SinkT, ...], history: str +) -> tuple[tuple[SinkT, str] | None, tuple[SinkT, ...]]: + """The sink replacing Recall's ``history`` and its writer parameter, then the other sinks.""" + + replacements: list[tuple[SinkT, str]] = [] + ordinary: list[SinkT] = [] + for sink in sinks: + writer: str | None = _writer_parameter(sink) + if writer is None: + ordinary.append(sink) + else: + replacements.append((sink, writer)) + if len(replacements) > 1: + raise EventExporterDiscoveryError( + f"Only one sink may replace Recall {history} history; found " + + ", ".join(_location(sink) for sink, _ in replacements) + ) + return (replacements[0] if replacements else None), tuple(ordinary) diff --git a/src/sqlbuild/compiler/discovery/main/runtime_extensions.py b/src/sqlbuild/compiler/discovery/main/runtime_extensions.py index 8a41d0579b..197605d5e8 100644 --- a/src/sqlbuild/compiler/discovery/main/runtime_extensions.py +++ b/src/sqlbuild/compiler/discovery/main/runtime_extensions.py @@ -23,10 +23,11 @@ bind_command_output_sink_declarations, discover_command_output_sink_declarations, ) +from sqlbuild.compiler.discovery._helpers.filesystem.recall_sinks import split_recall_sinks def discover_runtime_extensions(*, project_dir: Path) -> DiscoveredRuntimeExtensions: - """Discover providers, typed sinks, and lifecycle delivery settings at command startup.""" + """Discover providers, typed sinks, Recall replacement sinks and lifecycle delivery settings.""" declarations: tuple[DiscoveredEventExporterDeclaration, ...] = ( discover_event_exporter_declarations(project_dir=project_dir) @@ -62,13 +63,22 @@ def discover_runtime_extensions(*, project_dir: Path) -> DiscoveredRuntimeExtens project_dir=project_dir, providers=providers, ) - return DiscoveredRuntimeExtensions( - providers=providers, - event_exporters=tuple( + recall_lifecycle_sink, lifecycle_sinks = split_recall_sinks( + sinks=tuple( _apply_runtime_filter(exporter=exporter, config=config) for exporter in exporters ), - command_output_sinks=output_sinks, + history="lifecycle", + ) + recall_output_sink, ordinary_output_sinks = split_recall_sinks( + sinks=output_sinks, history="command-output" + ) + return DiscoveredRuntimeExtensions( + providers=providers, + event_exporters=lifecycle_sinks, + command_output_sinks=ordinary_output_sinks, lifecycle_shutdown_timeout_seconds=config.shutdown_timeout_seconds, + recall_lifecycle_sink=recall_lifecycle_sink, + recall_output_sink=recall_output_sink, ) diff --git a/src/sqlbuild/recall.py b/src/sqlbuild/recall.py new file mode 100644 index 0000000000..b26756f2b6 --- /dev/null +++ b/src/sqlbuild/recall.py @@ -0,0 +1,5 @@ +"""Public Recall sink authoring: a sink taking a ``RecallWriter`` replaces Recall's default.""" + +from sqlbuild._native import RecallWriter as RecallWriter + +__all__ = ("RecallWriter",) diff --git a/src/sqlbuild/runtime/event_exporting/main/_invoke_recall_sink.py b/src/sqlbuild/runtime/event_exporting/main/_invoke_recall_sink.py new file mode 100644 index 0000000000..4a973c62af --- /dev/null +++ b/src/sqlbuild/runtime/event_exporting/main/_invoke_recall_sink.py @@ -0,0 +1,13 @@ +"""Call a Recall replacement sink with one record and its writer.""" + +from collections.abc import Callable, Mapping + + +def invoke_recall_sink(*, function: Callable[..., object], arguments: Mapping[str, object]) -> bool: + """Call a user replacement sink and report whether it raised.""" + + try: + _ = function(**arguments) + except BaseException: + return True + return False diff --git a/src/sqlbuild/runtime/output_capture/models.py b/src/sqlbuild/runtime/output_capture/models.py index a3e508bbc3..f371b82625 100644 --- a/src/sqlbuild/runtime/output_capture/models.py +++ b/src/sqlbuild/runtime/output_capture/models.py @@ -2,6 +2,7 @@ from __future__ import annotations +import dataclasses from collections.abc import Callable, Mapping from dataclasses import dataclass, field from datetime import datetime @@ -17,7 +18,7 @@ @dataclass(frozen=True) class CommandOutputRecord: - """One ANSI-free bounded text chunk ready for destination serialization.""" + """One ANSI-free bounded text chunk; schema version 2 adds its exact bytes in ``data``.""" invocation_id: str sequence: int @@ -34,6 +35,7 @@ class CommandOutputRecord: schema_version: int = CURRENT_COMMAND_OUTPUT_SCHEMA_VERSION producer: str = "sqlbuild" producer_version: str = field(default=SQLBUILD_COMMAND_OUTPUT_PRODUCER_VERSION) + data: bytes = b"" def __post_init__(self) -> None: """Validate and freeze the known command-output wire contract.""" @@ -46,6 +48,17 @@ def record_id(self) -> str: return f"{self.invocation_id}:command-output:{self.sequence}" + def replace( + self, *, data: bytes | None = None, message: str | None = None + ) -> CommandOutputRecord: + """Return a copy with new ``data`` or ``message`` that keeps the record ID and sequence.""" + + return dataclasses.replace( + self, + data=self.data if data is None else data, + message=self.message if message is None else message, + ) + @dataclass(frozen=True) class CommandOutputCaptureSummary: diff --git a/tests/integration/src/sqlbuild/cli/recall/_test_types.py b/tests/integration/src/sqlbuild/cli/recall/_test_types.py index c6d2807663..5b870c19d0 100644 --- a/tests/integration/src/sqlbuild/cli/recall/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/recall/_test_types.py @@ -155,3 +155,24 @@ class UndecodableArgumentTestCase: selector: str expected_error: bytes expected_recorded: list[list[object]] + + +@dataclass(frozen=True) +class ReplacementLifecycleTestCase: + description: str + dropped_prefix: str + expected_filter_active: int + + +@dataclass(frozen=True) +class ReplacementOutputTestCase: + description: str + replaced: tuple[bytes, bytes] + expected_complete: int + + +@dataclass(frozen=True) +class FailingReplacementTestCase: + description: str + expected_failed_streams: list[list[object]] + expected_notice: bytes diff --git a/tests/integration/src/sqlbuild/cli/recall/helpers.py b/tests/integration/src/sqlbuild/cli/recall/helpers.py index 423fdcdf46..a1ff1bc111 100644 --- a/tests/integration/src/sqlbuild/cli/recall/helpers.py +++ b/tests/integration/src/sqlbuild/cli/recall/helpers.py @@ -1,6 +1,8 @@ """Synthetic DuckDB projects and process helpers for local Recall tests.""" +import itertools import json +import operator import os import re import signal @@ -327,3 +329,69 @@ def simultaneous_builds(project_dir: Path, targets: tuple[str, ...]) -> list[int start(project_dir, "build", "--target", target) for target in targets ] return [process.wait(timeout=600) for process in started] + + +RECALL_SINK_IMPORTS: str = ( + "from pathlib import Path\n\n" + "from sqlbuild.recall import RecallWriter\n" + "from sqlbuild.sinks import (\n" + " CommandOutputRecord, LifecycleEvent, command_output_sink, lifecycle_event_sink,\n" + ")\n\n" +) +EVENT_LOG_SINK: str = ( + "@lifecycle_event_sink\n" + "def event_log(event: LifecycleEvent) -> None:\n" + " with (Path(__file__).parents[1] / 'events.log').open('a') as log:\n" + " log.write(event.event_type + '\\n')\n" +) +FAILING_SINKS: str = ( + "@lifecycle_event_sink\n" + "def recall_events(event: LifecycleEvent, writer: RecallWriter) -> None:\n" + " raise RuntimeError('lifecycle sink failed')\n\n" + "@command_output_sink\n" + "def recall_output(record: CommandOutputRecord, writer: RecallWriter) -> None:\n" + " raise RuntimeError('output sink failed')\n" +) + + +def filtering_lifecycle_sink(dropped_prefix: str) -> str: + """A Recall lifecycle replacement keeping every event whose type lacks `dropped_prefix`.""" + + return ( + "@lifecycle_event_sink\n" + "def recall_events(event: LifecycleEvent, writer: RecallWriter) -> None:\n" + f" if not event.event_type.startswith({dropped_prefix!r}):\n" + " writer.publish_lifecycle(event)\n" + ) + + +def rewriting_output_sink(replaced: tuple[bytes, bytes]) -> str: + """A Recall output replacement storing each chunk with `replaced[0]` as `replaced[1]`.""" + + old, new = replaced + return ( + "@command_output_sink\n" + "def recall_output(record: CommandOutputRecord, writer: RecallWriter) -> None:\n" + f" data = record.data.replace({old!r}, {new!r})\n" + " writer.publish_command_output(record.replace(data=data))\n" + ) + + +def write_sinks(project_dir: Path, sinks: dict[str, str]) -> Path: + """Add `sinks/.py` modules holding the given sink definitions.""" + + for name, source in sinks.items(): + path: Path = project_dir / "sinks" / f"{name}.py" + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_text(RECALL_SINK_IMPORTS + source, encoding="utf-8") + return project_dir + + +def logged_event_types(project_dir: Path, dropped_prefix: str) -> tuple[list[str], list[str]]: + """The sorted event types `event_log` wrote, then those not starting with `dropped_prefix`.""" + + logged: list[str] = sorted( + (project_dir / "events.log").read_text(encoding="utf-8").splitlines() + ) + dropped: operator.methodcaller = operator.methodcaller("startswith", dropped_prefix) + return logged, list(itertools.filterfalse(dropped, logged)) diff --git a/tests/integration/src/sqlbuild/cli/recall/test_recall_replacement_sinks.py b/tests/integration/src/sqlbuild/cli/recall/test_recall_replacement_sinks.py new file mode 100644 index 0000000000..d49e96ac30 --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/recall/test_recall_replacement_sinks.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.cli.recall._test_types import ( + FailingReplacementTestCase, + ReplacementLifecycleTestCase, + ReplacementOutputTestCase, +) +from tests.integration.src.sqlbuild.cli.recall.helpers import ( + EVENT_LOG_SINK, + FAILING_SINKS, + filtering_lifecycle_sink, + logged_event_types, + recall_rows, + rewriting_output_sink, + run, + write_project, + write_sinks, +) + + +@pytest.mark.parametrize( + "test_case", + [ + ReplacementLifecycleTestCase( + description="statement events are filtered from Recall but reach the other sink", + dropped_prefix="statement_", + expected_filter_active=1, + ), + ], + ids=lambda case: case.description, +) +def test_given_a_filtering_lifecycle_sink_when_building_then_recall_keeps_what_it_publishes( + test_case: ReplacementLifecycleTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_sinks( + write_project(tmp_path / "orders_shop"), + { + "recall_events": filtering_lifecycle_sink(test_case.dropped_prefix), + "event_log": EVENT_LOG_SINK, + }, + ) + + built: subprocess.CompletedProcess[bytes] = run(project_dir, "build") + + logged, kept = logged_event_types(project_dir, test_case.dropped_prefix) + assert built.returncode == 0, built.stderr + assert b"Recall" not in built.stderr + assert len(kept) < len(logged) + assert recall_rows(project_dir, "SELECT event_type FROM events ORDER BY event_type") == [ + [event] for event in kept + ] + assert recall_rows( + project_dir, + "SELECT accepted, delivered, filtered, failed FROM recall_delivery WHERE stream = 'events'", + ) == [[len(logged), len(kept), len(logged) - len(kept), 0]] + assert recall_rows(project_dir, "SELECT filter_active FROM output_invocations") == [ + [test_case.expected_filter_active] + ] + + +@pytest.mark.parametrize( + "test_case", + [ + ReplacementOutputTestCase( + description="a pass-through sink keeps every byte", + replaced=(b"", b""), + expected_complete=1, + ), + ReplacementOutputTestCase( + description="a rewriting sink stores the rewritten bytes", + replaced=(b"binary", b"BINARY"), + expected_complete=1, + ), + ], + ids=lambda case: case.description, +) +def test_given_an_output_replacement_sink_when_recalled_then_its_bytes_are_stored_exactly( + test_case: ReplacementOutputTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_sinks( + write_project(tmp_path / "orders_shop"), + {"recall_output": rewriting_output_sink(test_case.replaced)}, + ) + + built: subprocess.CompletedProcess[bytes] = run(project_dir, "build") + stdout: subprocess.CompletedProcess[bytes] = run( + project_dir, "recall", "output", "last", "--stream", "stdout" + ) + stderr: subprocess.CompletedProcess[bytes] = run( + project_dir, "recall", "output", "last", "--stream", "stderr" + ) + + old, new = test_case.replaced + assert built.returncode == 0, built.stderr + assert old in built.stdout + assert stdout.stdout == built.stdout.replace(old, new) + assert stderr.stdout == built.stderr.replace(old, new) + assert recall_rows(project_dir, "SELECT complete, filter_active FROM output_invocations") == [ + [test_case.expected_complete, 1] + ] + + +@pytest.mark.parametrize( + "test_case", + [ + FailingReplacementTestCase( + description="raising sinks count their records as failed and the build succeeds", + expected_failed_streams=[["events"], ["output"]], + expected_notice=b"sqb build completed successfully; only its Recall history is affected", + ), + ], + ids=lambda case: case.description, +) +def test_given_raising_replacement_sinks_when_building_then_records_fail_and_the_command_does_not( + test_case: FailingReplacementTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_sinks( + write_project(tmp_path / "orders_shop"), {"recall": FAILING_SINKS} + ) + + built: subprocess.CompletedProcess[bytes] = run(project_dir, "build") + + assert built.returncode == 0, built.stderr + assert b"sink failed" not in built.stdout + built.stderr + assert b"Recall" not in built.stdout + assert built.stderr.rstrip().endswith(test_case.expected_notice + b".") + assert ( + recall_rows( + project_dir, + "SELECT stream FROM recall_delivery WHERE failed > 0 AND failed = accepted " + "AND delivered = 0 ORDER BY stream", + ) + == test_case.expected_failed_streams + ) + assert recall_rows(project_dir, "SELECT command FROM recall_invocations") == [["build"]] + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_test_types.py b/tests/unit/src/sqlbuild/compiler/discovery/_test_types.py index cd50322209..64afc1d3fb 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_test_types.py @@ -105,3 +105,20 @@ class DiscoverUniqueTestNameTestCase: repo_files: dict[str, str] expected_test_block_names: tuple[str | None, ...] expected_case_names: tuple[str, ...] + + +@dataclass(frozen=True) +class RecallSinkDiscoveryTestCase: + description: str + sink_files: dict[str, str] + expected_replacements: tuple[tuple[tuple[str, str], ...], ...] + expected_ordinary: tuple[tuple[str, ...], tuple[str, ...]] + + +@dataclass(frozen=True) +class InvalidRecallSinkTestCase: + description: str + project_files: dict[str, str] + expected_error: type[Exception] + expected_code: str + expected_message: str diff --git a/tests/unit/src/sqlbuild/compiler/discovery/helpers.py b/tests/unit/src/sqlbuild/compiler/discovery/helpers.py index a35a152be8..8655b27d50 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/helpers.py @@ -2,6 +2,9 @@ import sqlbuild._native as _native from sqlbuild._native import ( + DiscoveredCommandOutputSink, + DiscoveredEventExporter, + DiscoveredRuntimeExtensions, DiscoveredSqlModelFile, DiscoveredSqlScenarioFile, DiscoveredSqlTestBlock, @@ -173,3 +176,45 @@ def parse_schema_yml( _native.load_yaml_document(str(file_path), contents, "schema"), str(file_path), ) + + +RECALL_SINK_IMPORTS: str = ( + "from sqlbuild.recall import RecallWriter\n" + "from sqlbuild.sinks import (\n" + " CommandOutputRecord, LifecycleEvent, command_output_sink, lifecycle_event_sink,\n" + ")\n\n" +) + + +def write_recall_sink_project(*, project_dir: Path, files: dict[str, str]) -> None: + """Write a project named `orders` with `files`, each relative to the project directory.""" + + _ = (project_dir / "sqlbuild_project.toml").write_text( + 'name = "orders"\nadapter = "duckdb"\n', encoding="utf-8" + ) + for relative, contents in files.items(): + path: Path = project_dir / relative + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_text(contents, encoding="utf-8") + + +def recall_sink_names( + extensions: DiscoveredRuntimeExtensions, +) -> tuple[tuple[tuple[tuple[str, str], ...], ...], tuple[tuple[str, ...], ...]]: + """Each stream's replacement as `(name, writer_parameter)`, then the ordinary sinks' names.""" + + lifecycle: tuple[tuple[DiscoveredEventExporter, str], ...] = tuple( + filter(None, (extensions.recall_lifecycle_sink,)) + ) + output: tuple[tuple[DiscoveredCommandOutputSink, str], ...] = tuple( + filter(None, (extensions.recall_output_sink,)) + ) + replacements: tuple[tuple[tuple[str, str], ...], ...] = ( + tuple((sink.name, writer) for sink, writer in lifecycle), + tuple((sink.name, writer) for sink, writer in output), + ) + ordinary: tuple[tuple[str, ...], ...] = ( + tuple(sink.name for sink in extensions.event_exporters), + tuple(sink.name for sink in extensions.command_output_sinks), + ) + return replacements, ordinary diff --git a/tests/unit/src/sqlbuild/compiler/discovery/test_recall_replacement_sinks.py b/tests/unit/src/sqlbuild/compiler/discovery/test_recall_replacement_sinks.py new file mode 100644 index 0000000000..cad384e6aa --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/discovery/test_recall_replacement_sinks.py @@ -0,0 +1,145 @@ +from pathlib import Path + +import pytest + +from sqlbuild._native import ( + DiscoveredRuntimeExtensions, + EventExporterDiscoveryError, + ProviderDiscoveryError, +) +from sqlbuild.compiler.discovery.main.runtime_extensions import discover_runtime_extensions +from tests.unit.src.sqlbuild.compiler.discovery._test_types import ( + InvalidRecallSinkTestCase, + RecallSinkDiscoveryTestCase, +) +from tests.unit.src.sqlbuild.compiler.discovery.helpers import ( + RECALL_SINK_IMPORTS, + recall_sink_names, + write_recall_sink_project, +) + +LIFECYCLE_REPLACEMENT: str = ( + "@lifecycle_event_sink\n" + "def keep_runs(event: LifecycleEvent, history: RecallWriter) -> None:\n" + " history.publish_lifecycle(event)\n\n" +) +OUTPUT_REPLACEMENT: str = ( + "@command_output_sink\n" + "def redact(record: CommandOutputRecord, writer: RecallWriter) -> None:\n" + " writer.publish_command_output(record)\n\n" +) +ORDINARY_SINKS: str = ( + "@lifecycle_event_sink\n" + "def audit_log(event: LifecycleEvent) -> None:\n" + " del event\n\n" + "@command_output_sink\n" + "def output_log(record: CommandOutputRecord) -> None:\n" + " del record\n" +) + + +@pytest.mark.parametrize( + "test_case", + ( + RecallSinkDiscoveryTestCase( + description="no replacement keeps Recall's default for both streams", + sink_files={"sinks/logs.py": RECALL_SINK_IMPORTS + ORDINARY_SINKS}, + expected_replacements=((), ()), + expected_ordinary=(("audit_log",), ("output_log",)), + ), + RecallSinkDiscoveryTestCase( + description="one replacement per stream, by writer type under any parameter name", + sink_files={ + "sinks/recall.py": RECALL_SINK_IMPORTS + LIFECYCLE_REPLACEMENT + OUTPUT_REPLACEMENT, + "sinks/logs.py": RECALL_SINK_IMPORTS + ORDINARY_SINKS, + }, + expected_replacements=((("keep_runs", "history"),), (("redact", "writer"),)), + expected_ordinary=(("audit_log",), ("output_log",)), + ), + ), + ids=lambda case: case.description, +) +def test_given_sinks_taking_a_recall_writer_when_discovering_then_they_replace_recall_only( + tmp_path: Path, test_case: RecallSinkDiscoveryTestCase +) -> None: + write_recall_sink_project(project_dir=tmp_path, files=test_case.sink_files) + + extensions: DiscoveredRuntimeExtensions = discover_runtime_extensions(project_dir=tmp_path) + + assert recall_sink_names(extensions) == ( + test_case.expected_replacements, + test_case.expected_ordinary, + ) + + +@pytest.mark.parametrize( + "test_case", + ( + InvalidRecallSinkTestCase( + description="two replacements for one stream are both named", + project_files={ + "sinks/first.py": RECALL_SINK_IMPORTS + OUTPUT_REPLACEMENT, + "sinks/second.py": RECALL_SINK_IMPORTS + + OUTPUT_REPLACEMENT.replace("def redact", "def mask"), + }, + expected_error=EventExporterDiscoveryError, + expected_code="D015", + expected_message=( + "Only one sink may replace Recall command-output history; found " + "sinks/first.py:redact, sinks/second.py:mask" + ), + ), + InvalidRecallSinkTestCase( + description="a sink with two writers", + project_files={ + "sinks/recall.py": RECALL_SINK_IMPORTS + "@lifecycle_event_sink\n" + "def both(event: LifecycleEvent, one: RecallWriter, two: RecallWriter) -> None:\n" + " del event, one, two\n", + }, + expected_error=EventExporterDiscoveryError, + expected_code="D015", + expected_message="sinks/recall.py:both declares more than one RecallWriter parameter", + ), + InvalidRecallSinkTestCase( + description="a replacement with a provider parameter", + project_files={ + "providers/orders_api.py": "from sqlbuild.providers import Provider\n\n\n" + "class OrdersApi(Provider):\n 'Order service settings.'\n\n" + " region: str = 'eu'\n", + "sinks/recall.py": RECALL_SINK_IMPORTS + "@command_output_sink\n" + "def redact(record: CommandOutputRecord, writer: RecallWriter, orders_api) -> None:\n" + " del record, writer, orders_api\n", + }, + expected_error=EventExporterDiscoveryError, + expected_code="D015", + expected_message="sinks/recall.py:redact must not declare provider parameters", + ), + InvalidRecallSinkTestCase( + description="a provider named recall_writer", + project_files={ + "providers/recall_writer.py": "from sqlbuild.providers import Provider\n\n\n" + "class RecallWriter(Provider):\n 'Shadows the writer name.'\n\n" + " region: str = 'eu'\n", + "sinks/logs.py": RECALL_SINK_IMPORTS + ORDINARY_SINKS, + }, + expected_error=ProviderDiscoveryError, + expected_code="D012", + expected_message="Provider name 'recall_writer' in providers/recall_writer.py is reserved", + ), + ), + ids=lambda case: case.description, +) +def test_given_invalid_recall_replacement_sinks_when_discovering_then_a_coded_error_names_them( + tmp_path: Path, test_case: InvalidRecallSinkTestCase +) -> None: + write_recall_sink_project(project_dir=tmp_path, files=test_case.project_files) + + with pytest.raises(test_case.expected_error) as raised: + discover_runtime_extensions(project_dir=tmp_path) + + assert test_case.expected_message in str(raised.value) + assert getattr(raised.value, "code", None) == test_case.expected_code + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/runtime/output_capture/main/_test_types.py b/tests/unit/src/sqlbuild/runtime/output_capture/main/_test_types.py index 2d3f17a78b..79d9df9a94 100644 --- a/tests/unit/src/sqlbuild/runtime/output_capture/main/_test_types.py +++ b/tests/unit/src/sqlbuild/runtime/output_capture/main/_test_types.py @@ -13,3 +13,18 @@ class InvocationContextTestCase: raw_value: str expected_value: object | None = None expected_error: str | None = None + + +@dataclass(frozen=True) +class ReplacementRecordTestCase: + description: str + data: bytes + message: str | None + expected_message: str + + +@dataclass(frozen=True) +class InvalidReplacementRecordTestCase: + description: str + changes: dict[str, object] + expected_error: str diff --git a/tests/unit/src/sqlbuild/runtime/output_capture/main/test_command_output_record.py b/tests/unit/src/sqlbuild/runtime/output_capture/main/test_command_output_record.py new file mode 100644 index 0000000000..d6efef2257 --- /dev/null +++ b/tests/unit/src/sqlbuild/runtime/output_capture/main/test_command_output_record.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +import dataclasses + +import pytest + +from sqlbuild.sinks import CommandOutputRecord, CommandOutputValidationError +from tests.unit.src.sqlbuild.runtime.output_capture.main._test_types import ( + InvalidReplacementRecordTestCase, + ReplacementRecordTestCase, +) +from tests.unit.src.sqlbuild.runtime.output_capture.main.helpers import command_output_record + + +@pytest.mark.parametrize( + "test_case", + ( + ReplacementRecordTestCase( + description="new bytes keep the message", + data=b"built \xffmodel\n", + message=None, + expected_message="built model\n", + ), + ReplacementRecordTestCase( + description="new bytes and message", + data=b"[redacted]\n", + message="[redacted]\n", + expected_message="[redacted]\n", + ), + ), + ids=lambda case: case.description, +) +def test_given_a_version_2_record_when_replaced_then_the_copy_keeps_its_id_and_sequence( + test_case: ReplacementRecordTestCase, +) -> None: + record: CommandOutputRecord = dataclasses.replace( + command_output_record(), data=b"built model\n", schema_version=2 + ) + + replaced: CommandOutputRecord = record.replace(data=test_case.data, message=test_case.message) + + assert (replaced.record_id, replaced.sequence, replaced.schema_version) == ( + record.record_id, + record.sequence, + 2, + ) + assert (replaced.data, replaced.message) == (test_case.data, test_case.expected_message) + + +@pytest.mark.parametrize( + "test_case", + ( + InvalidReplacementRecordTestCase( + description="an unknown schema version", + changes={"schema_version": 3}, + expected_error="schema_version must be 1 or 2", + ), + InvalidReplacementRecordTestCase( + description="text where bytes belong", + changes={"data": "built model\n"}, + expected_error="data must be bytes", + ), + ), + ids=lambda case: case.description, +) +def test_given_an_invalid_record_shape_when_created_then_validation_names_the_field( + test_case: InvalidReplacementRecordTestCase, +) -> None: + with pytest.raises(CommandOutputValidationError, match=test_case.expected_error): + dataclasses.replace(command_output_record(), **test_case.changes) # type: ignore[arg-type] + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/website/src/content/docs/docs/concepts/observability/recall.mdx b/website/src/content/docs/docs/concepts/observability/recall.mdx index 6334b6d269..c908f7efa2 100644 --- a/website/src/content/docs/docs/concepts/observability/recall.mdx +++ b/website/src/content/docs/docs/concepts/observability/recall.mdx @@ -146,6 +146,37 @@ retention = "30d" Use a single ClickHouse server or ClickHouse Cloud; `ON CLUSTER` stores are not supported yet. +## Filtering what Recall stores + +By default Recall stores every lifecycle event and every output byte. To decide yourself, write a +[sink](/docs/concepts/observability/sinks) with a parameter annotated `RecallWriter`: it replaces +Recall's default for its stream, and Recall stores only what it publishes. + +```python +from sqlbuild.recall import RecallWriter +from sqlbuild.sinks import CommandOutputRecord, command_output_sink + + +@command_output_sink +def recall_output(record: CommandOutputRecord, writer: RecallWriter) -> None: + redacted: bytes = record.data.replace(b"secret-token", b"[redacted]") + writer.publish_command_output(record.replace(data=redacted)) +``` + +- A `@command_output_sink` replacement receives each output chunk with its exact bytes in + `record.data` (record schema version 2). `record.replace(data=…, message=…)` returns a copy that + keeps the record ID and sequence; publish at most one record per call. +- A `@lifecycle_event_sink` replacement receives each event its `event_kinds` and `min_severity` + accept and keeps it with `writer.publish_lifecycle(event)`. +- A record the sink does not publish is counted as `filtered`; a sink that raises has its record + counted as `failed`, and Recall never falls back to storing the unfiltered record. The command's + exit code and output are unaffected. `output_invocations.filter_active` is `1` for invocations + that ran a replacement. +- There is at most one replacement per stream; discovery fails with `D015` naming each + `path:function` otherwise. A replacement takes only its record and the writer, not providers, and + the provider name `recall_writer` is reserved. +- Other sinks keep receiving every record. With `enabled = false`, no replacement runs. + ## Querying history Use [`sqb recall`](/docs/cli/recall) to read the store. Queries are read-only `SELECT` statements, diff --git a/website/src/content/docs/docs/concepts/observability/sinks.mdx b/website/src/content/docs/docs/concepts/observability/sinks.mdx index 667d788bc0..c5c19e20a8 100644 --- a/website/src/content/docs/docs/concepts/observability/sinks.mdx +++ b/website/src/content/docs/docs/concepts/observability/sinks.mdx @@ -58,6 +58,10 @@ Command output groups adjacent text into bounded byte chunks and flushes on elap change, size, or close. A loss record reports bounded-queue drops. This stream is useful for remote transcripts but is not lifecycle evidence. +A sink with a parameter annotated `sqlbuild.recall.RecallWriter` is not an ordinary sink: it +decides what [Recall](/docs/concepts/observability/recall#filtering-what-recall-stores) stores for +its stream. + ## Configuration Runtime configuration can narrow declaration filters: From b166562150a3ef42e9297887536506d6896877c3 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 12:16:13 +0100 Subject: [PATCH 06/19] fix: withhold the launch record when a Recall replacement sink is the only sink --- .../bindings/_helpers/cli/observability.rs | 12 ++++-- .../src/sqlbuild/cli/recall/_test_types.py | 7 +++ .../src/sqlbuild/cli/recall/helpers.py | 1 + .../recall/test_recall_replacement_sinks.py | 43 +++++++++++++++++++ 4 files changed, 59 insertions(+), 4 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs index f6c98f448d..e5946c1965 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs @@ -334,10 +334,14 @@ impl Observed { &host::entry(py, UserCodeHost::DiscoverRuntimeExtensions)?, &[("project_dir", project_dir.clone())], )?; - self.recall_sinks = Some(( - native_field(&extensions, "recall_lifecycle_sink")?.unbind(), - native_field(&extensions, "recall_output_sink")?.unbind(), - )); + let lifecycle_replacement: Bound<'_, PyAny> = + native_field(&extensions, "recall_lifecycle_sink")?; + let output_replacement: Bound<'_, PyAny> = + native_field(&extensions, "recall_output_sink")?; + if !lifecycle_replacement.is_none() || !output_replacement.is_none() { + withhold_launch_record(py)?; + } + self.recall_sinks = Some((lifecycle_replacement.unbind(), output_replacement.unbind())); let event_exporters: Bound<'_, PyAny> = native_field(&extensions, "event_exporters")?; let sinks: Bound<'_, PyAny> = native_field(&extensions, "command_output_sinks")?; command_output_sinks = sinks.is_truthy()?; diff --git a/tests/integration/src/sqlbuild/cli/recall/_test_types.py b/tests/integration/src/sqlbuild/cli/recall/_test_types.py index 5b870c19d0..683bf9e1c6 100644 --- a/tests/integration/src/sqlbuild/cli/recall/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/recall/_test_types.py @@ -176,3 +176,10 @@ class FailingReplacementTestCase: description: str expected_failed_streams: list[list[object]] expected_notice: bytes + + +@dataclass(frozen=True) +class ReplacementLaunchRecordTestCase: + description: str + sinks: dict[str, str] + expected_records: int diff --git a/tests/integration/src/sqlbuild/cli/recall/helpers.py b/tests/integration/src/sqlbuild/cli/recall/helpers.py index a1ff1bc111..007be8709c 100644 --- a/tests/integration/src/sqlbuild/cli/recall/helpers.py +++ b/tests/integration/src/sqlbuild/cli/recall/helpers.py @@ -15,6 +15,7 @@ from pathlib import Path PYTHON_CLI: Path = Path(sys.executable).with_name("sqlbuild-python") +NATIVE_SQB: Path = Path(sys.executable).with_name("sqb") EMITTED_STDOUT: bytes = b"binary \x00\xff\xfe chunk\n" EMITTED_STDERR: bytes = "warning \u2713 from the task\n".encode() EMITTED_TAIL: bytes = b"after the warning\n" diff --git a/tests/integration/src/sqlbuild/cli/recall/test_recall_replacement_sinks.py b/tests/integration/src/sqlbuild/cli/recall/test_recall_replacement_sinks.py index d49e96ac30..ae0e522f87 100644 --- a/tests/integration/src/sqlbuild/cli/recall/test_recall_replacement_sinks.py +++ b/tests/integration/src/sqlbuild/cli/recall/test_recall_replacement_sinks.py @@ -5,14 +5,18 @@ import pytest +from sqlbuild._native import COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR from tests.integration.src.sqlbuild.cli.recall._test_types import ( FailingReplacementTestCase, + ReplacementLaunchRecordTestCase, ReplacementLifecycleTestCase, ReplacementOutputTestCase, ) from tests.integration.src.sqlbuild.cli.recall.helpers import ( EVENT_LOG_SINK, FAILING_SINKS, + NATIVE_SQB, + environment, filtering_lifecycle_sink, logged_event_types, recall_rows, @@ -140,5 +144,44 @@ def test_given_raising_replacement_sinks_when_building_then_records_fail_and_the assert recall_rows(project_dir, "SELECT command FROM recall_invocations") == [["build"]] +@pytest.mark.parametrize( + "test_case", + [ + ReplacementLaunchRecordTestCase( + description="without sinks sqb may replay the compile", + sinks={}, + expected_records=1, + ), + ReplacementLaunchRecordTestCase( + description="a Recall output replacement alone withholds the launch record", + sinks={"recall_output": rewriting_output_sink((b"", b""))}, + expected_records=0, + ), + ReplacementLaunchRecordTestCase( + description="a Recall lifecycle replacement alone withholds the launch record", + sinks={"recall_events": filtering_lifecycle_sink("statement_")}, + expected_records=0, + ), + ], + ids=lambda case: case.description, +) +def test_given_only_replacement_sinks_when_compiling_through_sqb_then_no_launch_record_is_left( + test_case: ReplacementLaunchRecordTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_sinks(write_project(tmp_path / "orders_shop"), test_case.sinks) + + compiled: subprocess.CompletedProcess[bytes] = subprocess.run( + [str(NATIVE_SQB), "compile"], + cwd=project_dir, + capture_output=True, + check=False, + env=environment(**{REUSE_DISABLE_ENV_VAR: "0"}), + ) + + assert compiled.returncode == 0, compiled.stderr + records: list[Path] = list((project_dir / "target" / "cache" / "launcher-v1").glob("*.json")) + assert len(records) == test_case.expected_records + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) From b0fcc0fc42aac36f5aac08c808832db2b388c98d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 12:26:06 +0100 Subject: [PATCH 07/19] fix: let a SELECT and INSERT user write a Recall store created from the printed DDL --- .../_helpers/recall/clickhouse_store.rs | 70 ++++++++++++------- .../sqlbuild-recall/src/schema/constants.rs | 2 + .../src/schema/main/ddl_script.rs | 25 ++++++- .../src/schema/tests/test_ddl.rs | 14 ++++ .../commands/main/clickhouse/_test_types.py | 7 ++ .../cli/commands/main/clickhouse/conftest.py | 4 +- .../cli/commands/main/clickhouse/helpers.py | 36 ++++++++++ .../main/clickhouse/test_clickhouse_recall.py | 60 ++++++++++++++++ .../docs/concepts/observability/recall.mdx | 3 +- 9 files changed, 192 insertions(+), 29 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs index 399f0c8f98..7939a6bceb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs @@ -30,7 +30,7 @@ use sqlbuild_recall::schema::main::table_columns::table_columns; use sqlbuild_recall::schema::main::table_spec::table_spec; use sqlbuild_recall::schema::main::table_statements::table_statements; use sqlbuild_recall::schema::main::view_statements::view_statements; -use sqlbuild_recall::schema::models::{DdlOptions, Dialect, Table}; +use sqlbuild_recall::schema::models::{ColumnSpec, DdlOptions, Dialect, Table}; use sqlbuild_recall::store::constants::LAST_INVOCATION_SELECTOR; use sqlbuild_recall::store::errors::{StoreError, StoreErrorKind}; use sqlbuild_recall::store::models::{ @@ -234,24 +234,25 @@ impl ClickHouseStore { self.read_columns(py)?; return Ok(tables); } + if complete && version == 0 { + self.read_columns(py)?; + if self.missing_columns().is_empty() { + self.record_version(py, envelope)?; + return Ok(tables); + } + } self.require_database(py)?; for statement in table_statements(Dialect::ClickHouse, &self.location.options) { self.command(py, &statement, "create the Recall tables")?; } self.read_columns(py)?; - for table in ALL_TABLES { - let spec = table_spec(table); - let present: &[StoredColumn] = self.columns.get(&table).map_or(&[], Vec::as_slice); - for column in table_columns(table) { - if !present.iter().any(|stored| stored.name == column.name) { - let statement = add_column_statement( - Dialect::ClickHouse, - &self.location.options, - (spec, &column), - ); - self.command(py, &statement, "add a Recall column")?; - } - } + for (table, column) in self.missing_columns() { + let statement = add_column_statement( + Dialect::ClickHouse, + &self.location.options, + (table_spec(table), &column), + ); + self.command(py, &statement, "add a Recall column")?; } self.read_columns(py)?; let upgrade: bool = version < RECALL_SCHEMA_VERSION; @@ -265,21 +266,40 @@ impl ClickHouseStore { } } if upgrade { - let now = format_timestamp(SystemTime::now()); - let row = meta_row( - envelope, - ( - META_SCHEMA_COMPONENT, - META_SCHEMA_VERSION_KEY, - &RECALL_SCHEMA_VERSION.to_string(), - ), - &now, - ); - self.write(py, Table::Meta, &[row], SystemTime::now())?; + self.record_version(py, envelope)?; } Ok(self.state(py)?.tables) } + /// Every column this build writes that the stored tables lack. + fn missing_columns(&self) -> Vec<(Table, ColumnSpec)> { + let mut missing: Vec<(Table, ColumnSpec)> = Vec::new(); + for table in ALL_TABLES { + let present: &[StoredColumn] = self.columns.get(&table).map_or(&[], Vec::as_slice); + for column in table_columns(table) { + if !present.iter().any(|stored| stored.name == column.name) { + missing.push((table, column)); + } + } + } + missing + } + + /// Record this build's schema version with an insert, which a writer allowed only `SELECT` and `INSERT` may run. + fn record_version(&self, py: Python<'_>, envelope: &RowEnvelope) -> Result<(), StoreError> { + let now = format_timestamp(SystemTime::now()); + let row = meta_row( + envelope, + ( + META_SCHEMA_COMPONENT, + META_SCHEMA_VERSION_KEY, + &RECALL_SCHEMA_VERSION.to_string(), + ), + &now, + ); + self.write(py, Table::Meta, &[row], SystemTime::now()) + } + fn write( &self, py: Python<'_>, diff --git a/crates/sqlbuild-recall/src/schema/constants.rs b/crates/sqlbuild-recall/src/schema/constants.rs index e3bf8ed3c9..1a09878b15 100644 --- a/crates/sqlbuild-recall/src/schema/constants.rs +++ b/crates/sqlbuild-recall/src/schema/constants.rs @@ -9,6 +9,8 @@ pub const MINIMUM_SQLITE_VERSION: (u32, u32) = (3, 44); /// The `recall_meta` component of schema rows. pub const META_SCHEMA_COMPONENT: &str = "schema"; pub const META_SCHEMA_VERSION_KEY: &str = "schema_version"; +/// The invocation ID of the schema version row an administrator's DDL script inserts. +pub const DDL_SCRIPT_INVOCATION: &str = "ddl-script"; /// The `recall_meta` component of maintenance claims. pub const META_MAINTENANCE_COMPONENT: &str = "maintenance"; pub const META_MAINTENANCE_CLAIM_KEY: &str = "claim"; diff --git a/crates/sqlbuild-recall/src/schema/main/ddl_script.rs b/crates/sqlbuild-recall/src/schema/main/ddl_script.rs index 42ea6a014c..671d2d8449 100644 --- a/crates/sqlbuild-recall/src/schema/main/ddl_script.rs +++ b/crates/sqlbuild-recall/src/schema/main/ddl_script.rs @@ -1,10 +1,28 @@ //! `ddl_script`: the complete ordered script an administrator runs to create a Recall store. +use crate::schema::_helpers::rendering::clickhouse_table; +use crate::schema::constants::{ + DDL_SCRIPT_INVOCATION, META_SCHEMA_COMPONENT, META_SCHEMA_VERSION_KEY, RECALL_SCHEMA_VERSION, +}; +use crate::schema::main::table_spec::table_spec; use crate::schema::main::table_statements::table_statements; use crate::schema::main::view_statements::view_statements; -use crate::schema::models::{DdlOptions, Dialect}; +use crate::schema::models::{DdlOptions, Dialect, Table}; -/// Every table, index and view statement, each terminated by `;`. +/// The `recall_meta` row recording the schema version, so writers allowed only `SELECT` and `INSERT` find the store complete. +fn clickhouse_version_row(options: &DdlOptions) -> String { + format!( + "INSERT INTO {meta} (component, key, value, recorded_at, invocation_id, \ + invocation_started_at, schema_version, producer_version, ingested_at, content_hash)\n\ + SELECT '{META_SCHEMA_COMPONENT}', '{META_SCHEMA_VERSION_KEY}', \ + '{RECALL_SCHEMA_VERSION}', now64(6), '{DDL_SCRIPT_INVOCATION}', now64(6), \ + {RECALL_SCHEMA_VERSION}, '{version}', now64(6), ''", + meta = clickhouse_table(table_spec(Table::Meta).name, options), + version = env!("CARGO_PKG_VERSION"), + ) +} + +/// Every table, index and view statement, then for ClickHouse the schema version row, each terminated by `;`. #[must_use] pub fn ddl_script(dialect: Dialect, options: &DdlOptions) -> String { let mut statements: Vec = table_statements(dialect, options); @@ -13,6 +31,9 @@ pub fn ddl_script(dialect: Dialect, options: &DdlOptions) -> String { .into_iter() .map(|(_, statement)| statement), ); + if dialect == Dialect::ClickHouse { + statements.push(clickhouse_version_row(options)); + } let mut script: String = statements.join(";\n\n"); script.push_str(";\n"); script diff --git a/crates/sqlbuild-recall/src/schema/tests/test_ddl.rs b/crates/sqlbuild-recall/src/schema/tests/test_ddl.rs index 1010e1fe76..4efd681ea6 100644 --- a/crates/sqlbuild-recall/src/schema/tests/test_ddl.rs +++ b/crates/sqlbuild-recall/src/schema/tests/test_ddl.rs @@ -1,5 +1,6 @@ use crate::schema::constants::{ALL_TABLES, VIEW_NAMES}; use crate::schema::main::add_column_statement::add_column_statement; +use crate::schema::main::ddl_script::ddl_script; use crate::schema::main::table_spec::table_spec; use crate::schema::main::table_statements::table_statements; use crate::schema::main::view_statements::view_statements; @@ -120,6 +121,19 @@ fn given_clickhouse_statement_sets_when_rendered_then_every_statement_has_the_sh required_fragments: &["CREATE OR REPLACE VIEW", "ON CLUSTER `analytics`"], expected_count: VIEW_NAMES.len(), }, + StatementSetTestCase { + description: "the administrator's script ends by recording the schema version", + statements: ddl_script(Dialect::ClickHouse, &options(None)) + .rsplit(";\n\n") + .take(1) + .map(str::to_owned) + .collect(), + required_fragments: &[ + "INSERT INTO `observability`.`recall_meta`", + "SELECT 'schema', 'schema_version', '1'", + ], + expected_count: 1, + }, StatementSetTestCase { description: "views deduplicate rows and filter by retention", statements: views(None), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py index 166d882b71..4c2ac1d00e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py @@ -154,3 +154,10 @@ class ClickHouseRecallRetentionConflictTestCase: created_with: str built_with: str expected_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class ClickHouseRecallLimitedWriterTestCase: + description: str + between_builds: tuple[str, ...] + expected_recorded: list[list[object]] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/conftest.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/conftest.py index 359deff916..62ac2d203a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/conftest.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/conftest.py @@ -36,7 +36,9 @@ def clickhouse_e2e_config() -> Iterator[dict[str, object]]: from testcontainers.clickhouse import ClickHouseContainer try: - with ClickHouseContainer(CLICKHOUSE_IMAGE) as container: + with ClickHouseContainer(CLICKHOUSE_IMAGE).with_env( + "CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT", "1" + ) as container: yield { "host": container.get_container_host_ip(), "port": int(container.get_exposed_port(8123)), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py index 91d1a61d15..3df2ee70f9 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py @@ -4,6 +4,8 @@ import re import uuid +from collections.abc import Iterator +from contextlib import contextmanager from pathlib import Path from textwrap import dedent from typing import Any, cast @@ -441,6 +443,40 @@ def apply_printed_retention_fix(*, config: dict[str, object], stderr: str) -> in return len(statements) +def apply_printed_ddl(*, config: dict[str, object], stderr: str) -> None: + """Run, as an administrator, the script a permission notice printed for creating the store.""" + + script: str = stderr.split(" with:\n\n", 1)[1].split("\nsqb ", 1)[0].strip() + execute_clickhouse_sql( + config=config, + statements=tuple( + statement.strip().removesuffix(";") for statement in script.split(";\n\n") + ), + ) + + +@contextmanager +def limited_user( + *, config: dict[str, object], database: str, revoked_inserts: tuple[str, ...] = () +) -> Iterator[dict[str, object]]: + """A user with only `SELECT` and `INSERT` on `database`, less `revoked_inserts`.""" + + name: str = build_unique_schema_name(prefix="recall_writer") + password: str = uuid.uuid4().hex + execute_clickhouse_sql( + config=config, + statements=( + f"CREATE USER {name} IDENTIFIED BY '{password}'", + f"GRANT SELECT, INSERT ON {database}.* TO {name}", + *(f"REVOKE INSERT ON {database}.{table} FROM {name}" for table in revoked_inserts), + ), + ) + try: + yield {**config, "user": name, "password": password} + finally: + execute_clickhouse_sql(config=config, statements=(f"DROP USER IF EXISTS {name}",)) + + def duplicate_recall_rows(*, config: dict[str, object], database: str) -> None: """Deliver every stored Recall row a second time, as a retried insert would.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py index ea468cc3d8..c9a9664750 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py @@ -12,6 +12,7 @@ ClickHouseRecallConcurrencyTestCase, ClickHouseRecallDuplicateTestCase, ClickHouseRecallExpiryTestCase, + ClickHouseRecallLimitedWriterTestCase, ClickHouseRecallOutputTestCase, ClickHouseRecallPermissionTestCase, ClickHouseRecallRetentionConflictTestCase, @@ -19,10 +20,13 @@ ) from tests.e2e.src.sqlbuild.cli.commands.main.clickhouse.helpers import ( age_recorded_invocations, + apply_printed_ddl, apply_printed_retention_fix, duplicate_recall_rows, + execute_clickhouse_sql, expire_recall_parts, fetch_clickhouse_rows, + limited_user, physical_rows, stored_ttl_days, write_recall_project, @@ -421,5 +425,61 @@ def test_given_a_store_with_another_retention_when_building_then_the_conflict_is ) +@pytest.mark.parametrize( + "test_case", + [ + ClickHouseRecallLimitedWriterTestCase( + description="a store created from the printed script", + between_builds=(), + expected_recorded=[["build", "completed"], ["build", "completed"]], + ), + ClickHouseRecallLimitedWriterTestCase( + description="a printed-script store whose version row expired", + between_builds=( + "ALTER TABLE {database}.recall_meta DELETE WHERE 1 SETTINGS mutations_sync = 2", + ), + expected_recorded=[["build", "completed"], ["build", "completed"]], + ), + ], + ids=lambda case: case.description, +) +def test_given_a_store_from_the_printed_ddl_when_a_select_insert_user_builds_then_history_is_written( + test_case: ClickHouseRecallLimitedWriterTestCase, + tmp_path: Path, + clickhouse_e2e_config: dict[str, object], + recall_database: str, +) -> None: + readonly: Path = write_recall_project( + parent=tmp_path / "readonly", + config=clickhouse_e2e_config, + database=recall_database, + settings=("", "settings = { readonly = 2 }\n"), + ) + apply_printed_ddl( + config=clickhouse_e2e_config, + stderr=run(readonly, "build").stderr.decode(errors="replace"), + ) + + with limited_user(config=clickhouse_e2e_config, database=recall_database) as writer: + project_dir: Path = write_recall_project( + parent=tmp_path / "writer", config=writer, database=recall_database + ) + first: subprocess.CompletedProcess[bytes] = run(project_dir, "build") + execute_clickhouse_sql( + config=clickhouse_e2e_config, + statements=tuple( + statement.format(database=recall_database) for statement in test_case.between_builds + ), + ) + builds: list[subprocess.CompletedProcess[bytes]] = [first, run(project_dir, "build")] + recorded: list[list[object]] = recall_rows( + project_dir, "SELECT command, status FROM invocations ORDER BY invocation_started_at" + ) + + assert [built.returncode for built in builds] == [0 for _ in builds] + assert [b"Recall" in built.stderr for built in builds] == [False for _ in builds] + assert recorded == test_case.expected_recorded + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/website/src/content/docs/docs/concepts/observability/recall.mdx b/website/src/content/docs/docs/concepts/observability/recall.mdx index c908f7efa2..1ea8b40045 100644 --- a/website/src/content/docs/docs/concepts/observability/recall.mdx +++ b/website/src/content/docs/docs/concepts/observability/recall.mdx @@ -127,7 +127,8 @@ retention = "30d" - **Setup.** On first use Recall creates its `recall_*` MergeTree tables and the views above in that database, and adds columns that a newer SQLBuild introduces. After that, writing needs only `INSERT` and `SELECT` on the tables. If the user may not create them, the command still succeeds - and prints the complete DDL for an administrator to run. + and prints the complete DDL for an administrator to run; a store created from it accepts writers + limited to `SELECT` and `INSERT`. - **Writes.** History is sent in batches of up to 5,000 rows about once a second, one insert per table, with the same queue, shutdown budget and exit-code guarantees as the local store. Many concurrent commands can write one store. From b548f56d3b546f8c49535d40905bfb4cec313ece Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 12:32:02 +0100 Subject: [PATCH 08/19] fix: count only the records of a failed ClickHouse table insert as failed --- .../_helpers/recall/clickhouse_store.rs | 16 +++++- crates/sqlbuild-recall/src/store/models.rs | 3 + .../src/writer/_helpers/delivery.rs | 8 +++ .../src/writer/_helpers/queue.rs | 18 +++++- .../src/writer/tests/helpers.rs | 24 +++++++- .../src/writer/tests/test_session.rs | 31 +++++++++- .../src/writer/tests/test_types.rs | 56 ++++++++++++++++++- .../commands/main/clickhouse/_test_types.py | 7 +++ .../main/clickhouse/test_clickhouse_recall.py | 48 ++++++++++++++++ 9 files changed, 202 insertions(+), 9 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs index 7939a6bceb..406f944458 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs @@ -386,10 +386,22 @@ impl RecallStore for ClickHouseStore { &stored, (&format_timestamp(now), &format_timestamp(refresh_before)), ); + let mut outcome: InsertOutcome = plan.outcome; for (table, table_rows) in &plan.inserts { - self.write(py, *table, table_rows, now)?; + match self.write(py, *table, table_rows, now) { + Ok(()) => {} + Err(error) if error.kind == StoreErrorKind::Busy => return Err(error), + Err(error) => { + outcome.failed.push(*table); + outcome.failure.get_or_insert(error.message); + } + } } - Ok(plan.outcome) + let failed: &[Table] = &outcome.failed; + outcome + .delivered + .retain(|(table, _)| !failed.contains(table)); + Ok(outcome) }) } diff --git a/crates/sqlbuild-recall/src/store/models.rs b/crates/sqlbuild-recall/src/store/models.rs index 8440f7aee7..3e3d6ec05d 100644 --- a/crates/sqlbuild-recall/src/store/models.rs +++ b/crates/sqlbuild-recall/src/store/models.rs @@ -61,6 +61,9 @@ pub struct InsertOutcome { /// Rows written or already present with equal content, by table. pub delivered: Vec<(Table, u64)>, pub conflicts: Vec, + /// Tables whose insert failed, so their rows were not written, and the first failure. + pub failed: Vec, + pub failure: Option, } /// What a maintenance run is allowed to do. diff --git a/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs b/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs index b2ecc16eb3..b9dfbf3259 100644 --- a/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs +++ b/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs @@ -193,6 +193,14 @@ fn heartbeat(config: &SessionConfig, now: SystemTime) -> RecallRow { ) } +/// The stream whose records are stored in `table`; `None` for the store's own bookkeeping. +pub(crate) fn record_stream(table: Table) -> Option { + match table { + Table::Heartbeats | Table::Delivery | Table::Meta | Table::Conflicts => None, + other => Some(stream_of(other)), + } +} + pub(crate) fn stream_of(table: Table) -> DeliveryStream { match table { Table::Output => DeliveryStream::Output, diff --git a/crates/sqlbuild-recall/src/writer/_helpers/queue.rs b/crates/sqlbuild-recall/src/writer/_helpers/queue.rs index c9d9648d02..034560654a 100644 --- a/crates/sqlbuild-recall/src/writer/_helpers/queue.rs +++ b/crates/sqlbuild-recall/src/writer/_helpers/queue.rs @@ -14,7 +14,7 @@ use crate::records::models::{ RowEnvelope, }; use crate::store::models::InsertOutcome; -use crate::writer::_helpers::delivery::stream_of; +use crate::writer::_helpers::delivery::{record_stream, stream_of}; use crate::writer::_helpers::tail::{TailRing, TailSegment}; use crate::writer::constants::{QUEUE_BYTE_CAPACITY, QUEUE_RECORD_CAPACITY}; use crate::writer::models::{DeliveryStream, Priority}; @@ -449,7 +449,7 @@ impl WriterState { } } - /// Count a written batch as delivered, less conflicts, or a failed one as failed. + /// Count a written batch as delivered, less conflicts and records of tables whose insert failed, or a failed one as failed. pub(crate) fn account( &mut self, batch: &[DeliveryStream], @@ -457,8 +457,20 @@ impl WriterState { ) { match outcome { Ok(outcome) => { + let failed: Vec = outcome + .failed + .iter() + .filter_map(|table| record_stream(*table)) + .collect(); for stream in batch { - self.counts_mut(*stream).delivered += 1; + if failed.contains(stream) { + self.counts_mut(*stream).failed += 1; + } else { + self.counts_mut(*stream).delivered += 1; + } + } + if let Some(reason) = &outcome.failure { + self.degraded.get_or_insert_with(|| reason.clone()); } for conflict in &outcome.conflicts { let stream = stream_of(conflict.table); diff --git a/crates/sqlbuild-recall/src/writer/tests/helpers.rs b/crates/sqlbuild-recall/src/writer/tests/helpers.rs index 9eda3f65f4..bf1a9ef004 100644 --- a/crates/sqlbuild-recall/src/writer/tests/helpers.rs +++ b/crates/sqlbuild-recall/src/writer/tests/helpers.rs @@ -6,6 +6,7 @@ use crate::records::main::invocation_row::invocation_row; use crate::records::models::{ DeliveryCounts, InvocationContext, OutputStream, RecallValue, RowEnvelope, }; +use crate::schema::models::Table; use crate::store::main::open_sqlite_store::open_sqlite_store; use crate::store::main::query_store::query_store; use crate::store::main::read_output_transcript::read_output_transcript; @@ -15,7 +16,7 @@ use crate::writer::main::start_recall_session::start_recall_session; use crate::writer::models::{ DeliveryStream, DeliverySummary, Priority, SessionConfig, SessionStore, SetupOutcome, }; -use crate::writer::tests::test_types::{PrefixReplacement, RecordedSession}; +use crate::writer::tests::test_types::{PartialStore, PrefixReplacement, RecordedSession}; /// The envelope of invocation `invocation_id`. pub(crate) fn envelope(invocation_id: &str) -> RowEnvelope { @@ -372,3 +373,24 @@ pub(crate) fn replaced_session( .expect("the query is valid"); (transcript.data, facts.rows) } + +/// `(accepted, delivered, failed)` of events and output after one batch whose insert into `failing` fails. +pub(crate) fn partially_inserted(failing: Table) -> [(u64, u64, u64); 2] { + let session = start_recall_session(SessionConfig { + store: SessionStore::Hosted(Arc::new(PartialStore { failing })), + ..config(PathBuf::from("unused"), "partial") + }); + session.append_output(OutputStream::Stdout, b"printed\n"); + session + .publish_event( + r#"{"event_id":"e1","event_type":"invocation_completed","schema_version":2, + "producer":"sqlbuild","producer_version":"1","occurred_at":"2026-10-19T00:00:01.000000Z", + "invocation_id":"partial","invocation_sequence":1,"payload":{"command":"build"}}"#, + ) + .expect("the event is valid"); + let summary = session.close(); + [DeliveryStream::Events, DeliveryStream::Output].map(|stream| { + let counts = stream_counts(&summary, stream); + (counts.accepted, counts.delivered, counts.failed) + }) +} diff --git a/crates/sqlbuild-recall/src/writer/tests/test_session.rs b/crates/sqlbuild-recall/src/writer/tests/test_session.rs index f215cd128d..894c1e75e2 100644 --- a/crates/sqlbuild-recall/src/writer/tests/test_session.rs +++ b/crates/sqlbuild-recall/src/writer/tests/test_session.rs @@ -1,19 +1,21 @@ use std::time::Duration; use crate::records::models::{OutputStream, RecallValue}; +use crate::schema::models::Table; use crate::writer::models::SetupOutcome; #[cfg(unix)] use crate::writer::tests::helpers::unwritable_session; use crate::writer::tests::helpers::{ artifact_counts, close_under_held_lock, event_counts, output_stored_while_running, - recorded_session, replaced_session, session_on_corrupt_store, + partially_inserted, recorded_session, replaced_session, session_on_corrupt_store, }; #[cfg(unix)] use crate::writer::tests::test_types::UnwritableTestCase; use crate::writer::tests::test_types::{ ArtifactSizeTestCase, CorruptStoreTestCase, EventBuildTestCase, HeldLockTestCase, - RecordedSession, ReplacementTestCase, SessionTestCase, StoredWhileRunningTestCase, + PartialInsertTestCase, RecordedSession, ReplacementTestCase, SessionTestCase, + StoredWhileRunningTestCase, }; const CLOSE_SLACK_SECONDS: f64 = 0.3; @@ -238,3 +240,28 @@ fn given_an_output_replacement_when_closed_then_published_filtered_and_failed_ch ); } } + +#[test] +fn given_one_table_of_a_batch_failing_when_written_then_only_its_records_count_as_failed() { + let test_cases = [ + PartialInsertTestCase { + description: "a refused output insert fails the output, not the events", + failing: Table::Output, + expected_counts: [(1, 1, 0), (1, 0, 1)], + }, + PartialInsertTestCase { + description: "a refused heartbeat insert fails no record", + failing: Table::Heartbeats, + expected_counts: [(1, 1, 0), (1, 1, 0)], + }, + ]; + + for test_case in test_cases { + assert_eq!( + partially_inserted(test_case.failing), + test_case.expected_counts, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-recall/src/writer/tests/test_types.rs b/crates/sqlbuild-recall/src/writer/tests/test_types.rs index bd727343b8..b4826e5ff1 100644 --- a/crates/sqlbuild-recall/src/writer/tests/test_types.rs +++ b/crates/sqlbuild-recall/src/writer/tests/test_types.rs @@ -1,4 +1,10 @@ -use crate::records::models::{OutputChunk, OutputStream, RecallValue}; +use std::time::{Duration, Instant}; + +use crate::records::models::{OutputChunk, OutputStream, RecallRow, RecallValue, RowEnvelope}; +use crate::schema::models::Table; +use crate::store::errors::StoreError; +use crate::store::models::InsertOutcome; +use crate::store::types::{RecallStore, RecallStoreOpener}; use crate::writer::models::{DeliveryStream, Priority, ReplacedOutput, SetupOutcome}; use crate::writer::types::OutputReplacement; @@ -107,3 +113,51 @@ impl OutputReplacement for PrefixReplacement { } } } + +pub(crate) struct PartialInsertTestCase { + pub(crate) description: &'static str, + pub(crate) failing: Table, + /// `(accepted, delivered, failed)` of the events and output streams. + pub(crate) expected_counts: [(u64, u64, u64); 2], +} + +/// A hosted store whose insert into `failing` fails while the batch's other tables are written. +#[derive(Debug)] +pub(crate) struct PartialStore { + pub(crate) failing: Table, +} + +impl RecallStore for PartialStore { + fn insert(&self, rows: &[RecallRow], _give_up: Instant) -> Result { + let failing: bool = rows.iter().any(|row| row.table == self.failing); + Ok(InsertOutcome { + failed: if failing { + vec![self.failing] + } else { + Vec::new() + }, + failure: failing.then(|| "the insert was refused".to_owned()), + ..InsertOutcome::default() + }) + } +} + +impl RecallStoreOpener for PartialStore { + fn open( + &self, + _envelope: &RowEnvelope, + _give_up: Instant, + ) -> Result, StoreError> { + Ok(Box::new(Self { + failing: self.failing, + })) + } + + fn batch_rows(&self) -> usize { + 1_000 + } + + fn batch_window(&self) -> Duration { + Duration::from_millis(10) + } +} diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py index 4c2ac1d00e..77545d8262 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py @@ -161,3 +161,10 @@ class ClickHouseRecallLimitedWriterTestCase: description: str between_builds: tuple[str, ...] expected_recorded: list[list[object]] + + +@dataclass(frozen=True) +class ClickHouseRecallPartialInsertTestCase: + description: str + revoked_inserts: tuple[str, ...] + expected_delivery: list[list[object]] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py index c9a9664750..014458cfff 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py @@ -14,6 +14,7 @@ ClickHouseRecallExpiryTestCase, ClickHouseRecallLimitedWriterTestCase, ClickHouseRecallOutputTestCase, + ClickHouseRecallPartialInsertTestCase, ClickHouseRecallPermissionTestCase, ClickHouseRecallRetentionConflictTestCase, ClickHouseRecallRetentionTestCase, @@ -481,5 +482,52 @@ def test_given_a_store_from_the_printed_ddl_when_a_select_insert_user_builds_the assert recorded == test_case.expected_recorded +@pytest.mark.parametrize( + "test_case", + [ + ClickHouseRecallPartialInsertTestCase( + description="refused output inserts fail the output and leave the events delivered", + revoked_inserts=("recall_output",), + expected_delivery=[["events", 1, 0], ["output", 0, 1]], + ), + ], + ids=lambda case: case.description, +) +def test_given_one_refused_table_when_building_then_only_its_records_count_as_failed( + test_case: ClickHouseRecallPartialInsertTestCase, + tmp_path: Path, + clickhouse_e2e_config: dict[str, object], + recall_database: str, +) -> None: + creator: Path = write_recall_project( + parent=tmp_path / "creator", config=clickhouse_e2e_config, database=recall_database + ) + assert run(creator, "build").returncode == 0 + + with limited_user( + config=clickhouse_e2e_config, + database=recall_database, + revoked_inserts=test_case.revoked_inserts, + ) as writer: + built: subprocess.CompletedProcess[bytes] = run( + write_recall_project( + parent=tmp_path / "writer", config=writer, database=recall_database + ), + "build", + ) + + assert built.returncode == 0, built.stderr + assert ( + recall_rows( + creator, + "SELECT stream, toUInt8(delivered = accepted), toUInt8(failed = accepted) " + "FROM recall_delivery FINAL WHERE stream IN ('events', 'output') AND invocation_id = " + "(SELECT invocation_id FROM invocations ORDER BY invocation_started_at DESC LIMIT 1) " + "ORDER BY stream", + ) + == test_case.expected_delivery + ) + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) From 39fb75fd4962b53f853735816e78ab5afab69cc5 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 12:35:30 +0100 Subject: [PATCH 09/19] fix: keep referenced SQL text available in the ClickHouse artifacts view until its TTL --- .../src/schema/_helpers/clickhouse_views.rs | 5 +-- .../src/schema/tests/test_ddl.rs | 11 +++++ .../commands/main/clickhouse/_test_types.py | 8 ++++ .../cli/commands/main/clickhouse/helpers.py | 15 +++++++ .../main/clickhouse/test_clickhouse_recall.py | 45 +++++++++++++++++++ .../docs/concepts/observability/recall.mdx | 2 +- 6 files changed, 82 insertions(+), 4 deletions(-) diff --git a/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs b/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs index ba54bed59e..5b35558015 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs @@ -5,15 +5,15 @@ use crate::schema::constants::{OUTPUT_TAIL_RECORDS, REPLACING_ENGINE}; use crate::schema::main::table_spec::table_spec; use crate::schema::models::{DdlOptions, Table}; -/// `table`'s rows within retention, one copy per key: `FINAL` for replacing tables, so filters on the sorting key reach the read, else the first ingested copy. +/// `table`'s rows within its TTL, one copy per key: `FINAL` for replacing tables, so filters on the sorting key reach the read, else the first ingested copy. SQL text outlives retention like its TTL, so every reference still within retention finds it. fn deduplicated(table: Table, options: &DdlOptions) -> String { let spec = table_spec(table); + let days: u32 = options.retention_days.saturating_add(spec.ttl_extra_days); if spec.clickhouse_engine == REPLACING_ENGINE { return format!( "(SELECT * FROM {name} FINAL WHERE {expiry} >= now64(6) - INTERVAL {days} DAY)", name = clickhouse_table(spec.name, options), expiry = spec.expiry_column, - days = options.retention_days, ); } format!( @@ -21,7 +21,6 @@ fn deduplicated(table: Table, options: &DdlOptions) -> String { ORDER BY ingested_at ASC LIMIT 1 BY {key})", name = clickhouse_table(spec.name, options), expiry = spec.expiry_column, - days = options.retention_days, key = spec.key.join(", "), ) } diff --git a/crates/sqlbuild-recall/src/schema/tests/test_ddl.rs b/crates/sqlbuild-recall/src/schema/tests/test_ddl.rs index 4efd681ea6..a924eb5c78 100644 --- a/crates/sqlbuild-recall/src/schema/tests/test_ddl.rs +++ b/crates/sqlbuild-recall/src/schema/tests/test_ddl.rs @@ -134,6 +134,17 @@ fn given_clickhouse_statement_sets_when_rendered_then_every_statement_has_the_sh ], expected_count: 1, }, + StatementSetTestCase { + description: "SQL text stays visible as long as its TTL keeps it", + statements: views(None) + .into_iter() + .filter(|statement| statement.contains("`recall_artifacts` FINAL")) + .collect(), + required_fragments: &[ + "`recall_artifacts` FINAL WHERE seen_at >= now64(6) - INTERVAL 21 DAY", + ], + expected_count: 1, + }, StatementSetTestCase { description: "views deduplicate rows and filter by retention", statements: views(None), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py index 77545d8262..25bf993f57 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py @@ -168,3 +168,11 @@ class ClickHouseRecallPartialInsertTestCase: description: str revoked_inserts: tuple[str, ...] expected_delivery: list[list[object]] + + +@dataclass(frozen=True) +class ClickHouseRecallArtifactAgeTestCase: + description: str + artifact_age_days: int + reference_age_days: int + expected_availability: list[list[object]] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py index 3df2ee70f9..4e83dc5d7b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py @@ -408,6 +408,21 @@ def age_recorded_invocations(*, config: dict[str, object], database: str, days: execute_clickhouse_sql(config=config, statements=tuple(statements)) +def age_stored_artifacts(*, config: dict[str, object], database: str, days: int) -> None: + """Move every stored SQL text's `seen_at` `days` earlier, as if not refreshed since.""" + + table: str = f"{database}.recall_artifacts" + execute_clickhouse_sql( + config=config, + statements=( + f"INSERT INTO {table} SELECT * REPLACE (seen_at - INTERVAL {days} DAY AS seen_at) " + f"FROM {table}", + f"ALTER TABLE {table} DELETE WHERE seen_at > now64(6) - INTERVAL {days - 1} DAY " + "SETTINGS mutations_sync = 2", + ), + ) + + def expire_recall_parts(*, config: dict[str, object], database: str) -> None: """Apply every Recall table's TTL now, as background TTL merges eventually would.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py index 014458cfff..1ffd971d08 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py @@ -9,6 +9,7 @@ from tests.e2e.src.sqlbuild.cli.commands.main.clickhouse._test_types import ( ClickHouseRecallAbandonedTestCase, + ClickHouseRecallArtifactAgeTestCase, ClickHouseRecallConcurrencyTestCase, ClickHouseRecallDuplicateTestCase, ClickHouseRecallExpiryTestCase, @@ -21,6 +22,7 @@ ) from tests.e2e.src.sqlbuild.cli.commands.main.clickhouse.helpers import ( age_recorded_invocations, + age_stored_artifacts, apply_printed_ddl, apply_printed_retention_fix, duplicate_recall_rows, @@ -529,5 +531,48 @@ def test_given_one_refused_table_when_building_then_only_its_records_count_as_fa ) +@pytest.mark.parametrize( + "test_case", + [ + ClickHouseRecallArtifactAgeTestCase( + description="SQL seen past retention stays available to a reference within it", + artifact_age_days=33, + reference_age_days=27, + expected_availability=[[1, 1]], + ), + ], + ids=lambda case: case.description, +) +def test_given_sql_older_than_retention_when_still_referenced_then_artifacts_show_it_available( + test_case: ClickHouseRecallArtifactAgeTestCase, + tmp_path: Path, + clickhouse_e2e_config: dict[str, object], + recall_database: str, +) -> None: + project_dir: Path = write_recall_project( + parent=tmp_path, config=clickhouse_e2e_config, database=recall_database + ) + assert run(project_dir, "build").returncode == 0 + age_recorded_invocations( + config=clickhouse_e2e_config, + database=recall_database, + days=test_case.reference_age_days, + ) + age_stored_artifacts( + config=clickhouse_e2e_config, + database=recall_database, + days=test_case.artifact_age_days, + ) + + assert ( + recall_rows( + project_dir, + "SELECT toUInt8(count() > 0), min(available) FROM artifacts " + "WHERE startsWith(invocation_id, 'aged-')", + ) + == test_case.expected_availability + ) + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/website/src/content/docs/docs/concepts/observability/recall.mdx b/website/src/content/docs/docs/concepts/observability/recall.mdx index 1ea8b40045..7d2d249551 100644 --- a/website/src/content/docs/docs/concepts/observability/recall.mdx +++ b/website/src/content/docs/docs/concepts/observability/recall.mdx @@ -137,7 +137,7 @@ retention = "30d" is counted once. Before writing, Recall looks up the batch's keys: a key already stored with different content keeps the stored row, and the rejected copy is listed in `conflicts`. - **Retention.** Every table carries a TTL of `retention` (SQL text a week longer, and refreshed - while it is still referenced), and the views hide rows past `retention` at once. ClickHouse drops + while it is still referenced), and the views hide rows past their TTL at once. ClickHouse drops expired daily parts in the background, so rows may stay on disk for a while after the views stop showing them. Retention belongs to the store: when a project's `retention` differs from the tables' TTL, Recall warns once a day with the `ALTER TABLE … MODIFY TTL` and view statements that From 6743d05e13d1c959d3fa967f2875f51e0c462d53 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 12:39:59 +0100 Subject: [PATCH 10/19] fix: suppress output capture only on the Recall delivery thread, not for sqb recall readers --- .../_helpers/recall/clickhouse_calls.rs | 10 ++++-- .../_helpers/recall/clickhouse_store.rs | 4 +-- .../clickhouse/_helpers/recall_store.py | 7 ++-- .../commands/main/clickhouse/_test_types.py | 7 ++++ .../main/clickhouse/test_clickhouse_recall.py | 33 +++++++++++++++++++ .../src/sqlbuild/cli/recall/helpers.py | 6 ++++ 6 files changed, 59 insertions(+), 8 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_calls.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_calls.rs index d0f7651cd1..4918ddf991 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_calls.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_calls.rs @@ -85,10 +85,10 @@ fn reply<'py>( } } -/// Connect with the store's connection options; the calling thread's output is never captured. +/// Connect with the store's connection options; a `quiet` caller, Recall's delivery thread, never has its output captured. pub(crate) fn connect( py: Python<'_>, - config: &[(String, ConfigValue)], + (config, quiet): (&[(String, ConfigValue)], bool), action: &str, ) -> Result { let options = PyDict::new(py); @@ -104,7 +104,11 @@ pub(crate) fn connect( ("config", options.into_any()), ( "quiet_thread_attribute", - PyString::new(py, OUTPUT_SUPPRESSED_ATTRIBUTE).into_any(), + if quiet { + PyString::new(py, OUTPUT_SUPPRESSED_ATTRIBUTE).into_any() + } else { + py.None().into_bound(py) + }, ), ], action, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs index 406f944458..499126f5ef 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs @@ -427,7 +427,7 @@ impl RecallStoreOpener for ClickHouseStoreOpener { location: self.location.clone(), connection: Some(calls::connect( py, - &self.location.config, + (&self.location.config, true), &self.location.connect_action(), )?), columns: HashMap::new(), @@ -473,7 +473,7 @@ impl ClickHouseReader { location: location.clone(), connection: Some(calls::connect( py, - &location.config, + (&location.config, false), &location.connect_action(), )?), columns: HashMap::new(), diff --git a/src/sqlbuild/adapters/clickhouse/_helpers/recall_store.py b/src/sqlbuild/adapters/clickhouse/_helpers/recall_store.py index e7182cbfc6..818ed30e31 100644 --- a/src/sqlbuild/adapters/clickhouse/_helpers/recall_store.py +++ b/src/sqlbuild/adapters/clickhouse/_helpers/recall_store.py @@ -59,11 +59,12 @@ def _number(*, summary: dict[str, Any], key: str) -> int | None: def connect_recall_store( - *, config: dict[str, object], quiet_thread_attribute: str + *, config: dict[str, object], quiet_thread_attribute: str | None ) -> tuple[RecallFailure, int]: - """Connect for Recall with a short connect timeout and return the connection's handle.""" + """Connect for Recall and return the handle; only the delivery thread asks to stay quiet.""" - setattr(threading.current_thread(), quiet_thread_attribute, True) + if quiet_thread_attribute is not None: + setattr(threading.current_thread(), quiet_thread_attribute, True) options: dict[str, object] = {_CONNECT_TIMEOUT_KEY: _DEFAULT_CONNECT_TIMEOUT_SECONDS, **config} try: adapter: ClickHouseAdapter = ClickHouseAdapter() diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py index 25bf993f57..48d9ce4a7d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py @@ -176,3 +176,10 @@ class ClickHouseRecallArtifactAgeTestCase: artifact_age_days: int reference_age_days: int expected_availability: list[list[object]] + + +@dataclass(frozen=True) +class ClickHouseRecallQueryOutputSinkTestCase: + description: str + sql: str + expected_fragment: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py index 1ffd971d08..aa05d2c78a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py @@ -17,6 +17,7 @@ ClickHouseRecallOutputTestCase, ClickHouseRecallPartialInsertTestCase, ClickHouseRecallPermissionTestCase, + ClickHouseRecallQueryOutputSinkTestCase, ClickHouseRecallRetentionConflictTestCase, ClickHouseRecallRetentionTestCase, ) @@ -35,10 +36,12 @@ write_recall_project, ) from tests.integration.src.sqlbuild.cli.recall.helpers import ( + OUTPUT_LOG_SINK, concurrent_builds, killed_build, recall_rows, run, + write_sinks, ) @@ -574,5 +577,35 @@ def test_given_sql_older_than_retention_when_still_referenced_then_artifacts_sho ) +@pytest.mark.parametrize( + "test_case", + [ + ClickHouseRecallQueryOutputSinkTestCase( + description="query results reach a command-output sink", + sql="SELECT 'shipped_orders' AS answer", + expected_fragment="shipped_orders", + ), + ], + ids=lambda case: case.description, +) +def test_given_an_output_sink_when_querying_clickhouse_recall_then_the_sink_sees_the_results( + test_case: ClickHouseRecallQueryOutputSinkTestCase, + tmp_path: Path, + clickhouse_e2e_config: dict[str, object], + recall_database: str, +) -> None: + project_dir: Path = write_recall_project( + parent=tmp_path, config=clickhouse_e2e_config, database=recall_database + ) + assert run(project_dir, "build").returncode == 0 + _ = write_sinks(project_dir, {"output_log": OUTPUT_LOG_SINK}) + + queried: subprocess.CompletedProcess[bytes] = run(project_dir, "recall", "query", test_case.sql) + + assert queried.returncode == 0, queried.stderr + assert test_case.expected_fragment.encode() in queried.stdout + assert test_case.expected_fragment in (project_dir / "output.log").read_text(encoding="utf-8") + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/cli/recall/helpers.py b/tests/integration/src/sqlbuild/cli/recall/helpers.py index 007be8709c..92df43f010 100644 --- a/tests/integration/src/sqlbuild/cli/recall/helpers.py +++ b/tests/integration/src/sqlbuild/cli/recall/helpers.py @@ -345,6 +345,12 @@ def simultaneous_builds(project_dir: Path, targets: tuple[str, ...]) -> list[int " with (Path(__file__).parents[1] / 'events.log').open('a') as log:\n" " log.write(event.event_type + '\\n')\n" ) +OUTPUT_LOG_SINK: str = ( + "@command_output_sink\n" + "def output_log(record: CommandOutputRecord) -> None:\n" + " with (Path(__file__).parents[1] / 'output.log').open('a') as log:\n" + " log.write(record.message)\n" +) FAILING_SINKS: str = ( "@lifecycle_event_sink\n" "def recall_events(event: LifecycleEvent, writer: RecallWriter) -> None:\n" From f7b840f49a3b2e783e793032029404818f58d381 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 12:45:59 +0100 Subject: [PATCH 11/19] fix: hold Recall output until discovery decides whether a replacement sink filters it --- .../src/writer/_helpers/delivery.rs | 5 +- .../src/writer/_helpers/queue.rs | 22 ++++++++- .../src/writer/tests/helpers.rs | 8 ++-- .../src/writer/tests/test_session.rs | 46 ++++++++++++------- .../src/writer/tests/test_types.rs | 2 + 5 files changed, 58 insertions(+), 25 deletions(-) diff --git a/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs b/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs index b9dfbf3259..fe02d8be9c 100644 --- a/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs +++ b/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs @@ -248,13 +248,12 @@ fn next_batch( { state.flush_output(); } - if state.queued() > 0 && first_seen.is_none() { + if state.ready() > 0 && first_seen.is_none() { first_seen = Some(now); } let window_over = first_seen.is_some_and(|seen| now.duration_since(seen) >= batching.window); - if state.queued() >= batching.rows || (state.queued() > 0 && (window_over || state.closing)) - { + if state.ready() >= batching.rows || (state.ready() > 0 && (window_over || state.closing)) { return (state.take_batch(batching.rows), state.closing); } if state.closing && state.queued() == 0 && !state.has_pending_output() { diff --git a/crates/sqlbuild-recall/src/writer/_helpers/queue.rs b/crates/sqlbuild-recall/src/writer/_helpers/queue.rs index 034560654a..7a778bdece 100644 --- a/crates/sqlbuild-recall/src/writer/_helpers/queue.rs +++ b/crates/sqlbuild-recall/src/writer/_helpers/queue.rs @@ -236,11 +236,29 @@ impl WriterState { self.records } - /// Up to `limit` rows, highest priority first. + /// Whether output waits in the queue until discovery decides whether a replacement sink filters it. + fn holding_output(&self) -> bool { + self.held_context.is_some() + } + + /// Records a batch may take now: output is held while the context row is. + pub(crate) fn ready(&self) -> usize { + if self.holding_output() { + self.records - self.buckets[Priority::Output as usize].len() + } else { + self.records + } + } + + /// Up to `limit` rows, highest priority first; held output stays queued. pub(crate) fn take_batch(&mut self, limit: usize) -> Vec { let mut batch: Vec = Vec::new(); let mut rows = 0; - for bucket in self.buckets.iter_mut().rev() { + let held: Option = self.holding_output().then_some(Priority::Output as usize); + for (index, bucket) in self.buckets.iter_mut().enumerate().rev() { + if held == Some(index) { + continue; + } while rows < limit { let Some(record) = bucket.pop_front() else { break; diff --git a/crates/sqlbuild-recall/src/writer/tests/helpers.rs b/crates/sqlbuild-recall/src/writer/tests/helpers.rs index bf1a9ef004..69743baca7 100644 --- a/crates/sqlbuild-recall/src/writer/tests/helpers.rs +++ b/crates/sqlbuild-recall/src/writer/tests/helpers.rs @@ -345,9 +345,10 @@ pub(crate) fn output_stored_while_running( stored.rows } -/// The transcript and output accounting a session with a deferred context and `PrefixReplacement` stored for `output`. +/// The transcript and output accounting a session with a deferred context and `PrefixReplacement` stored for `output`, written `discovery` before the replacement is set. pub(crate) fn replaced_session( output: &[(OutputStream, &[u8])], + discovery: Duration, ) -> (Vec, Vec>) { let directory = tempfile::tempdir().expect("a temporary directory can be created"); let path = directory.path().join("logs/recall.sqlite3"); @@ -355,11 +356,12 @@ pub(crate) fn replaced_session( defer_context: true, ..config(path.clone(), "replaced") }); - session.set_output_replacement(Arc::new(PrefixReplacement)); - session.publish_context(true); for (stream, data) in output { session.append_output(*stream, data); } + std::thread::sleep(discovery); + session.set_output_replacement(Arc::new(PrefixReplacement)); + session.publish_context(true); drop(session.close()); let store = open_sqlite_store(&path, OpenMode::ReadOnly).expect("the store opens"); let transcript = diff --git a/crates/sqlbuild-recall/src/writer/tests/test_session.rs b/crates/sqlbuild-recall/src/writer/tests/test_session.rs index 894c1e75e2..a9c98d4938 100644 --- a/crates/sqlbuild-recall/src/writer/tests/test_session.rs +++ b/crates/sqlbuild-recall/src/writer/tests/test_session.rs @@ -211,26 +211,38 @@ fn given_heartbeats_due_faster_than_batches_fill_when_running_then_queued_record #[test] fn given_an_output_replacement_when_closed_then_published_filtered_and_failed_chunks_are_accounted() { - let test_cases = [ReplacementTestCase { - description: "one chunk published changed, one filtered, one failed", - output: &[ - (OutputStream::Stderr, b"drop that\n"), - (OutputStream::Stdout, b"boom\n"), - (OutputStream::Stderr, b"keep this\n"), - ], - expected_transcript: b"KEEP THIS\n", - expected_accounting: vec![vec![ - RecallValue::Integer(1), - RecallValue::Integer(1), - RecallValue::Integer(2), - RecallValue::Integer(1), - RecallValue::Text("dropped".to_owned()), - ]], - }]; + let accounting = vec![vec![ + RecallValue::Integer(1), + RecallValue::Integer(1), + RecallValue::Integer(2), + RecallValue::Integer(1), + RecallValue::Text("dropped".to_owned()), + ]]; + let output: &[(OutputStream, &[u8])] = &[ + (OutputStream::Stderr, b"drop that\n"), + (OutputStream::Stdout, b"boom\n"), + (OutputStream::Stderr, b"keep this\n"), + ]; + let test_cases = [ + ReplacementTestCase { + description: "one chunk published changed, one filtered, one failed", + output, + discovery: Duration::ZERO, + expected_transcript: b"KEEP THIS\n", + expected_accounting: accounting.clone(), + }, + ReplacementTestCase { + description: "output written during a slow discovery waits for the replacement", + output, + discovery: Duration::from_millis(900), + expected_transcript: b"KEEP THIS\n", + expected_accounting: accounting, + }, + ]; for test_case in test_cases { assert_eq!( - replaced_session(test_case.output), + replaced_session(test_case.output, test_case.discovery), ( test_case.expected_transcript.to_vec(), test_case.expected_accounting diff --git a/crates/sqlbuild-recall/src/writer/tests/test_types.rs b/crates/sqlbuild-recall/src/writer/tests/test_types.rs index b4826e5ff1..cee81d31ca 100644 --- a/crates/sqlbuild-recall/src/writer/tests/test_types.rs +++ b/crates/sqlbuild-recall/src/writer/tests/test_types.rs @@ -94,6 +94,8 @@ pub(crate) struct StoredWhileRunningTestCase { pub(crate) struct ReplacementTestCase { pub(crate) description: &'static str, pub(crate) output: &'static [(OutputStream, &'static [u8])], + /// How long discovery takes after the output is written, before the replacement is set. + pub(crate) discovery: Duration, pub(crate) expected_transcript: &'static [u8], pub(crate) expected_accounting: Vec>, } From 0acda4441e2131ce199e99580e8df39d2117c6b6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 13:05:10 +0100 Subject: [PATCH 12/19] fix: name the command in the first Recall heartbeat so a run killed during discovery is listed --- .../_helpers/recall/clickhouse_store.rs | 35 +++++++++++-------- .../src/clickhouse/tests/helpers.rs | 1 + .../src/records/main/heartbeat_row.rs | 5 ++- .../src/schema/_helpers/clickhouse_views.rs | 7 ++-- .../src/schema/_helpers/sqlite_views.rs | 7 ++-- .../sqlbuild-recall/src/schema/constants.rs | 4 ++- .../src/store/tests/test_maintenance.rs | 1 + .../src/store/tests/test_views.rs | 3 ++ .../src/writer/_helpers/delivery.rs | 15 ++++++-- .../src/writer/tests/helpers.rs | 21 +++++++++++ .../src/writer/tests/test_session.rs | 32 ++++++++++++++--- .../src/writer/tests/test_types.rs | 6 ++++ .../main/clickhouse/test_clickhouse_recall.py | 31 ++++++++++++++++ .../src/sqlbuild/cli/recall/helpers.py | 32 +++++++++++++++++ 14 files changed, 170 insertions(+), 30 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs index 499126f5ef..12bbe6ff39 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs @@ -218,7 +218,7 @@ impl ClickHouseStore { Ok(()) } - /// Create missing tables, add missing columns and replace the views when this build's schema is newer; a newer store keeps its views. + /// Create missing tables, add missing columns and replace the views when this build's schema is newer or columns were added; a newer store keeps its views. fn ensure_schema( &mut self, py: Python<'_>, @@ -230,13 +230,13 @@ impl ClickHouseStore { version, } = self.state(py)?; let complete: bool = tables.len() == ALL_TABLES.len() && views.len() == VIEW_NAMES.len(); - if complete && version >= RECALL_SCHEMA_VERSION { + if complete { self.read_columns(py)?; - return Ok(tables); - } - if complete && version == 0 { - self.read_columns(py)?; - if self.missing_columns().is_empty() { + let columns_complete: bool = self.missing_columns().is_empty(); + if columns_complete && version >= RECALL_SCHEMA_VERSION { + return Ok(tables); + } + if columns_complete && version == 0 { self.record_version(py, envelope)?; return Ok(tables); } @@ -246,7 +246,9 @@ impl ClickHouseStore { self.command(py, &statement, "create the Recall tables")?; } self.read_columns(py)?; - for (table, column) in self.missing_columns() { + let missing: Vec<(Table, ColumnSpec)> = self.missing_columns(); + let added: bool = !missing.is_empty(); + for (table, column) in missing { let statement = add_column_statement( Dialect::ClickHouse, &self.location.options, @@ -257,7 +259,7 @@ impl ClickHouseStore { self.read_columns(py)?; let upgrade: bool = version < RECALL_SCHEMA_VERSION; for (name, statement) in view_statements(Dialect::ClickHouse, &self.location.options) { - if upgrade { + if upgrade || added { self.command(py, &statement, "create the Recall views")?; } else if !views.contains(name) { let statement = @@ -585,18 +587,21 @@ impl ClickHouseReader { py: Python<'_>, selector: &str, ) -> Result { - let table = self - .store - .location - .table(table_spec(Table::Invocations).name); + let location = &self.store.location; + let started = format!( + "(SELECT invocation_id, invocation_started_at FROM {} \ + UNION ALL SELECT invocation_id, invocation_started_at FROM {})", + location.table(table_spec(Table::Invocations).name), + location.table(table_spec(Table::Heartbeats).name), + ); let sql = if selector == LAST_INVOCATION_SELECTOR { format!( - "SELECT invocation_id FROM {table} \ + "SELECT invocation_id FROM {started} \ ORDER BY invocation_started_at DESC, invocation_id DESC LIMIT 1" ) } else { format!( - "SELECT DISTINCT invocation_id FROM {table} \ + "SELECT DISTINCT invocation_id FROM {started} \ WHERE startsWith(invocation_id, {}) ORDER BY invocation_id LIMIT 3", literal(selector) ) diff --git a/crates/sqlbuild-recall/src/clickhouse/tests/helpers.rs b/crates/sqlbuild-recall/src/clickhouse/tests/helpers.rs index 698944697b..0288784e98 100644 --- a/crates/sqlbuild-recall/src/clickhouse/tests/helpers.rs +++ b/crates/sqlbuild-recall/src/clickhouse/tests/helpers.rs @@ -26,6 +26,7 @@ pub(crate) fn batch() -> Vec { &envelope(), ("2026-10-19T00:00:01.000000Z", "2026-10-19T00:01:31.000000Z"), (Some("host"), Some(7)), + (None, None), )]; rows.extend(artifact_rows( &envelope(), diff --git a/crates/sqlbuild-recall/src/records/main/heartbeat_row.rs b/crates/sqlbuild-recall/src/records/main/heartbeat_row.rs index 340c09365f..f9ecc0e136 100644 --- a/crates/sqlbuild-recall/src/records/main/heartbeat_row.rs +++ b/crates/sqlbuild-recall/src/records/main/heartbeat_row.rs @@ -4,12 +4,13 @@ use crate::records::_helpers::rows::{finish_row, integer, optional_text, text}; use crate::records::models::{RecallRow, RecallValue, RowEnvelope}; use crate::schema::models::Table; -/// A heartbeat at `beat_at` that declares the invocation abandoned after `stale_at`. +/// A heartbeat at `beat_at` that declares the invocation abandoned after `stale_at`; the first also records the command and subcommand, so an invocation that never writes its context row still names them. #[must_use] pub fn heartbeat_row( envelope: &RowEnvelope, (beat_at, stale_at): (&str, &str), (host_token, pid): (Option<&str>, Option), + (command, subcommand): (Option<&str>, Option<&str>), ) -> RecallRow { finish_row( Table::Heartbeats, @@ -22,6 +23,8 @@ pub fn heartbeat_row( "pid", pid.map_or(RecallValue::Null, |pid| integer(u64::from(pid))), ), + ("command", optional_text(command)), + ("subcommand", optional_text(subcommand)), ], ) } diff --git a/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs b/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs index 5b35558015..fa774d6aeb 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs @@ -58,7 +58,8 @@ terminals AS ( GROUP BY invocation_id ), beats AS ( - SELECT invocation_id, max(beat_at) AS last_heartbeat_at, max(stale_at) AS stale_at + SELECT invocation_id, max(beat_at) AS last_heartbeat_at, max(stale_at) AS stale_at, + min(command) AS command, min(subcommand) AS subcommand FROM {beats} GROUP BY invocation_id ) SELECT @@ -81,8 +82,8 @@ SELECT terminals.terminal_exit_code AS exit_code, terminals.terminal_error_type AS error_type, terminals.terminal_error_code AS error_code, - coalesce(context.command, started.command) AS command, - context.subcommand AS subcommand, + coalesce(context.command, started.command, beats.command) AS command, + coalesce(context.subcommand, beats.subcommand) AS subcommand, context.target AS target, context.adapter AS adapter, context.project_name AS project_name, diff --git a/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs b/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs index bc7a1922d2..91d09d179f 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs @@ -259,7 +259,8 @@ terminals AS ( WHERE event_type IN ('invocation_completed', 'invocation_failed') ), beats AS ( - SELECT invocation_id, MAX(beat_at) AS last_heartbeat_at, MAX(stale_at) AS stale_at + SELECT invocation_id, MAX(beat_at) AS last_heartbeat_at, MAX(stale_at) AS stale_at, + MIN(command) AS command, MIN(subcommand) AS subcommand FROM recall_heartbeats GROUP BY invocation_id ), @@ -281,8 +282,8 @@ projected AS ( terminals.exit_code, terminals.error_type, terminals.error_code, - COALESCE(context.command, started.command) AS command, - context.subcommand, + COALESCE(context.command, started.command, beats.command) AS command, + COALESCE(context.subcommand, beats.subcommand) AS subcommand, context.target, context.adapter, context.project_name, diff --git a/crates/sqlbuild-recall/src/schema/constants.rs b/crates/sqlbuild-recall/src/schema/constants.rs index 1a09878b15..d6306c1e9b 100644 --- a/crates/sqlbuild-recall/src/schema/constants.rs +++ b/crates/sqlbuild-recall/src/schema/constants.rs @@ -140,11 +140,13 @@ const ARTIFACT_REF_COLUMNS: [ColumnSpec; 6] = [ ColumnSpec::required("referenced_at", ColumnType::Timestamp), ]; -const HEARTBEAT_COLUMNS: [ColumnSpec; 4] = [ +const HEARTBEAT_COLUMNS: [ColumnSpec; 6] = [ ColumnSpec::required("beat_at", ColumnType::Timestamp), ColumnSpec::required("stale_at", ColumnType::Timestamp), ColumnSpec::optional("host_token", ColumnType::Text), ColumnSpec::optional("pid", ColumnType::Integer), + ColumnSpec::optional("command", ColumnType::Text), + ColumnSpec::optional("subcommand", ColumnType::Text), ]; const DELIVERY_COLUMNS: [ColumnSpec; 9] = [ diff --git a/crates/sqlbuild-recall/src/store/tests/test_maintenance.rs b/crates/sqlbuild-recall/src/store/tests/test_maintenance.rs index c94249656a..83d1899c82 100644 --- a/crates/sqlbuild-recall/src/store/tests/test_maintenance.rs +++ b/crates/sqlbuild-recall/src/store/tests/test_maintenance.rs @@ -26,6 +26,7 @@ fn given_expired_recent_and_running_invocations_when_maintained_then_only_expire &old_running, (&shifted(-10), &shifted(80)), (None, None), + (None, None), )); mixed.extend(artifact_rows( &old, diff --git a/crates/sqlbuild-recall/src/store/tests/test_views.rs b/crates/sqlbuild-recall/src/store/tests/test_views.rs index 6875dd5ab2..742d993e6d 100644 --- a/crates/sqlbuild-recall/src/store/tests/test_views.rs +++ b/crates/sqlbuild-recall/src/store/tests/test_views.rs @@ -219,16 +219,19 @@ fn given_terminals_and_heartbeats_when_projected_then_status_never_claims_false_ &running, (&wall_clock(-1), &wall_clock(90)), (None, Some(1)), + (None, None), ), heartbeat_row( &abandoned, ("2026-10-01T12:00:30.000000Z", "2026-10-01T12:02:00.000000Z"), (None, None), + (None, None), ), heartbeat_row( &abandoned, ("2026-10-01T12:00:00.000000Z", "2026-10-01T12:01:30.000000Z"), (None, None), + (None, None), ), ]; let test_cases = [ProjectionTestCase { diff --git a/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs b/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs index fe02d8be9c..07ae4cf049 100644 --- a/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs +++ b/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs @@ -184,12 +184,19 @@ fn write_batch( } } -fn heartbeat(config: &SessionConfig, now: SystemTime) -> RecallRow { +/// A heartbeat at `now`; the `first` names the command before the context row may be written. +fn heartbeat(config: &SessionConfig, now: SystemTime, first: bool) -> RecallRow { let stale = now + config.heartbeat_interval * HEARTBEAT_STALE_INTERVALS; + let context = &config.context; heartbeat_row( &config.envelope, (&format_timestamp(now), &format_timestamp(stale)), - (config.context.host_token.as_deref(), config.context.pid), + (context.host_token.as_deref(), context.pid), + if first { + (context.command.as_deref(), context.subcommand.as_deref()) + } else { + (None, None) + }, ) } @@ -328,6 +335,7 @@ pub(crate) fn deliver(shared: &Arc, config: &SessionConfig) { } }; let mut next_heartbeat: SystemTime = SystemTime::now(); + let mut first_heartbeat: bool = true; let mut materializer = Materializer::new(config.envelope.clone()); let batching: Batching = batching(config); loop { @@ -344,7 +352,8 @@ pub(crate) fn deliver(shared: &Arc, config: &SessionConfig) { let mut rows: Vec = built.rows; let now: SystemTime = SystemTime::now(); if now >= next_heartbeat && !closing { - rows.push(heartbeat(config, now)); + rows.push(heartbeat(config, now, first_heartbeat)); + first_heartbeat = false; next_heartbeat = now + config.heartbeat_interval; } if !rows.is_empty() { diff --git a/crates/sqlbuild-recall/src/writer/tests/helpers.rs b/crates/sqlbuild-recall/src/writer/tests/helpers.rs index 69743baca7..fc2f8835f7 100644 --- a/crates/sqlbuild-recall/src/writer/tests/helpers.rs +++ b/crates/sqlbuild-recall/src/writer/tests/helpers.rs @@ -396,3 +396,24 @@ pub(crate) fn partially_inserted(failing: Table) -> [(u64, u64, u64); 2] { (counts.accepted, counts.delivered, counts.failed) }) } + +/// What `invocations` shows for a session still waiting on discovery, its context row held, after `wait`. +pub(crate) fn invocation_during_discovery(wait: Duration) -> Vec> { + let directory = tempfile::tempdir().expect("a temporary directory can be created"); + let path = directory.path().join("logs/recall.sqlite3"); + let session = start_recall_session(SessionConfig { + defer_context: true, + ..config(path.clone(), "discovering") + }); + std::thread::sleep(wait); + let store = open_sqlite_store(&path, OpenMode::ReadOnly).expect("the store opens"); + let shown = query_store( + &store, + "SELECT command, status, started_at IS NOT NULL, \ + (SELECT COUNT(*) FROM recall_invocations) FROM invocations", + None, + ) + .expect("the query is valid"); + drop(session.close()); + shown.rows +} diff --git a/crates/sqlbuild-recall/src/writer/tests/test_session.rs b/crates/sqlbuild-recall/src/writer/tests/test_session.rs index a9c98d4938..ad150377fa 100644 --- a/crates/sqlbuild-recall/src/writer/tests/test_session.rs +++ b/crates/sqlbuild-recall/src/writer/tests/test_session.rs @@ -7,14 +7,15 @@ use crate::writer::models::SetupOutcome; #[cfg(unix)] use crate::writer::tests::helpers::unwritable_session; use crate::writer::tests::helpers::{ - artifact_counts, close_under_held_lock, event_counts, output_stored_while_running, - partially_inserted, recorded_session, replaced_session, session_on_corrupt_store, + artifact_counts, close_under_held_lock, event_counts, invocation_during_discovery, + output_stored_while_running, partially_inserted, recorded_session, replaced_session, + session_on_corrupt_store, }; #[cfg(unix)] use crate::writer::tests::test_types::UnwritableTestCase; use crate::writer::tests::test_types::{ - ArtifactSizeTestCase, CorruptStoreTestCase, EventBuildTestCase, HeldLockTestCase, - PartialInsertTestCase, RecordedSession, ReplacementTestCase, SessionTestCase, + ArtifactSizeTestCase, CorruptStoreTestCase, DiscoveryTestCase, EventBuildTestCase, + HeldLockTestCase, PartialInsertTestCase, RecordedSession, ReplacementTestCase, SessionTestCase, StoredWhileRunningTestCase, }; @@ -277,3 +278,26 @@ fn given_one_table_of_a_batch_failing_when_written_then_only_its_records_count_a ); } } + +#[test] +fn given_a_held_context_row_when_discovery_runs_then_the_first_heartbeat_names_the_command() { + let test_cases = [DiscoveryTestCase { + description: "a run killed during discovery is still listed with its command", + wait: Duration::from_millis(600), + expected_rows: vec![vec![ + RecallValue::Text("build".to_owned()), + RecallValue::Text("running".to_owned()), + RecallValue::Integer(1), + RecallValue::Integer(0), + ]], + }]; + + for test_case in test_cases { + assert_eq!( + invocation_during_discovery(test_case.wait), + test_case.expected_rows, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-recall/src/writer/tests/test_types.rs b/crates/sqlbuild-recall/src/writer/tests/test_types.rs index cee81d31ca..9b5c81a76b 100644 --- a/crates/sqlbuild-recall/src/writer/tests/test_types.rs +++ b/crates/sqlbuild-recall/src/writer/tests/test_types.rs @@ -163,3 +163,9 @@ impl RecallStoreOpener for PartialStore { Duration::from_millis(10) } } + +pub(crate) struct DiscoveryTestCase { + pub(crate) description: &'static str, + pub(crate) wait: Duration, + pub(crate) expected_rows: Vec>, +} diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py index aa05d2c78a..3a41ad74c3 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py @@ -39,6 +39,7 @@ OUTPUT_LOG_SINK, concurrent_builds, killed_build, + killed_during_discovery, recall_rows, run, write_sinks, @@ -330,6 +331,36 @@ def test_given_a_killed_build_when_heartbeats_stop_then_the_invocation_is_abando assert recorded == [[running_id, test_case.expected_status, 1, 0]] +@pytest.mark.parametrize( + "test_case", + [ + ClickHouseRecallAbandonedTestCase( + description="a build SIGKILLed during discovery is listed and recalled", + heartbeat_seconds="0.5", + expected_status="abandoned", + ), + ], + ids=lambda case: case.description, +) +def test_given_a_build_killed_during_discovery_when_recalled_then_its_command_and_start_remain( + test_case: ClickHouseRecallAbandonedTestCase, + tmp_path: Path, + clickhouse_e2e_config: dict[str, object], + recall_database: str, +) -> None: + project_dir: Path = write_recall_project( + parent=tmp_path, config=clickhouse_e2e_config, database=recall_database + ) + + recorded, recalled = killed_during_discovery( + project_dir, heartbeat_seconds=test_case.heartbeat_seconds + ) + + assert recorded == [["build", test_case.expected_status, 1]] + assert recalled.returncode == 0, recalled.stderr + assert f"the invocation is {test_case.expected_status}".encode() in recalled.stderr + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/integration/src/sqlbuild/cli/recall/helpers.py b/tests/integration/src/sqlbuild/cli/recall/helpers.py index 92df43f010..ebf4633adc 100644 --- a/tests/integration/src/sqlbuild/cli/recall/helpers.py +++ b/tests/integration/src/sqlbuild/cli/recall/helpers.py @@ -199,6 +199,38 @@ def killed_build(project_dir: Path, *, heartbeat_seconds: str) -> tuple[str, lis return str(running[0][0]), recorded +def killed_during_discovery( + project_dir: Path, *, heartbeat_seconds: str +) -> tuple[list[list[object]], subprocess.CompletedProcess[bytes]]: + """Rows left by a build SIGKILLed while importing a slow sink, and `recall output last`.""" + + sink: Path = project_dir / "sinks" / "slow_import.py" + sink.parent.mkdir(parents=True, exist_ok=True) + _ = sink.write_text( + "import os\nimport time\n\n" + "time.sleep(float(os.environ.get('ORDERS_IMPORT_SECONDS', '0')))\n", + encoding="utf-8", + ) + process: subprocess.Popen[bytes] = start( + project_dir, + "build", + SQLBUILD_RECALL_HEARTBEAT_SECONDS=heartbeat_seconds, + ORDERS_IMPORT_SECONDS="60", + ) + _ = wait_for( + "SELECT invocation_id FROM invocations WHERE status = 'running'", + project_dir, + timeout=120, + ) + os.killpg(process.pid, signal.SIGKILL) + _ = process.wait(timeout=30) + time.sleep(5 * float(heartbeat_seconds)) + recorded: list[list[object]] = recall_rows( + project_dir, "SELECT command, status, started_at IS NOT NULL FROM invocations" + ) + return recorded, run(project_dir, "recall", "output", "last") + + def concurrent_builds(project_dir: Path, targets: tuple[str, ...]) -> tuple[list[int], int, int]: """Exit codes of concurrent builds, reads made meanwhile and unexpected read failures.""" From 18a7a36b8febe34208c81c6d303836af27563c40 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 13:18:38 +0100 Subject: [PATCH 13/19] fix: run the Recall lifecycle replacement sink on the delivery thread within the shutdown budget --- .../bindings/_helpers/recall/replacements.rs | 104 ++++++++++++------ .../src/bindings/_helpers/recall/session.rs | 4 +- .../src/bindings/_helpers/recall/sinks.rs | 26 ++++- .../src/bindings/_helpers/recall/writer.rs | 53 +++++---- .../src/writer/_helpers/delivery.rs | 6 +- .../src/writer/_helpers/materialize.rs | 22 +++- .../src/writer/_helpers/queue.rs | 7 +- .../src/writer/_helpers/session.rs | 25 ++++- .../sqlbuild-recall/src/writer/constants.rs | 2 + crates/sqlbuild-recall/src/writer/models.rs | 9 ++ .../src/writer/tests/helpers.rs | 26 ++++- .../src/writer/tests/test_session.rs | 26 ++++- .../src/writer/tests/test_types.rs | 43 +++++++- crates/sqlbuild-recall/src/writer/types.rs | 8 +- fensu.toml | 4 +- .../main/_invoke_recall_sink.py | 4 +- .../src/sqlbuild/cli/recall/_test_types.py | 8 ++ .../src/sqlbuild/cli/recall/helpers.py | 32 ++++++ .../recall/test_recall_replacement_sinks.py | 69 ++++++++++++ .../docs/concepts/observability/recall.mdx | 9 +- 20 files changed, 403 insertions(+), 84 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/replacements.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/replacements.rs index f2f5752f2d..275143aa8a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/replacements.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/replacements.rs @@ -11,8 +11,10 @@ use sqlbuild_core::command_output::main::ansi_free_text::ansi_free_text; use sqlbuild_core::event_export::main::route_lifecycle_event::route_lifecycle_event; use sqlbuild_core::event_export::models::{ExporterCounts, ExporterFilter}; use sqlbuild_recall::records::models::{OutputChunk, OutputStream}; -use sqlbuild_recall::writer::models::{DeliveryStream, RecallSession, ReplacedOutput}; -use sqlbuild_recall::writer::types::OutputReplacement; +use sqlbuild_recall::writer::models::{ + DeliveryStream, RecallSession, ReplacedEvents, ReplacedOutput, +}; +use sqlbuild_recall::writer::types::{OutputReplacement, PendingLifecycleEvent}; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::event_exporting::models::BoundEventExporter; @@ -29,16 +31,33 @@ use crate::bindings::_helpers::recall::writer::RecallWriter; use crate::bindings::_helpers::records::fields::field as native_field; thread_local! { - /// Set while this thread runs the output replacement sink, whose own writes Recall never records. - static IN_OUTPUT_REPLACEMENT: Cell = const { Cell::new(false) }; + /// Set while this thread runs a replacement sink, whose own writes Recall never records. + static IN_REPLACEMENT_SINK: Cell = const { Cell::new(false) }; +} + +/// Whether the current thread is running a replacement sink. +pub(crate) fn in_replacement_sink() -> bool { + IN_REPLACEMENT_SINK.with(Cell::get) } -/// Whether the current thread is running the output replacement sink. -pub(crate) fn in_output_replacement() -> bool { - IN_OUTPUT_REPLACEMENT.with(Cell::get) +/// Run `call` as a replacement sink on the delivery thread: its writes are neither recorded nor handed to command-output sinks. +fn as_replacement_sink(call: impl FnOnce(Python<'_>) -> PyResult, failed: T) -> T { + IN_REPLACEMENT_SINK.with(|flag| flag.set(true)); + let result = Python::attach(|py| { + if let Ok(thread) = + imported(py, "threading", "current_thread").and_then(|current| current.call0()) + { + drop(thread.setattr(OUTPUT_SUPPRESSED_ATTRIBUTE, true)); + } + call(py) + }) + .unwrap_or(failed); + IN_REPLACEMENT_SINK.with(|flag| flag.set(false)); + result } /// The lifecycle replacement: the sink function, the parameter its writer goes in, and its event filter. +#[derive(Debug)] struct LifecycleReplacement { function: Py, writer_parameter: String, @@ -48,7 +67,7 @@ struct LifecycleReplacement { /// The replacement sinks of one invocation, configured once discovery has run. #[derive(Default)] pub(crate) struct RecallReplacements { - lifecycle: OnceLock, + lifecycle: OnceLock>, } impl std::fmt::Debug for RecallReplacements { @@ -109,11 +128,11 @@ impl RecallReplacements { Some(&native_field(&sink, "min_severity")?), ), )?; - drop(self.lifecycle.set(LifecycleReplacement { + drop(self.lifecycle.set(Arc::new(LifecycleReplacement { function: bound.function, writer_parameter, filter: bound.filter, - })); + }))); active = true; } if let Some((sink, writer_parameter)) = discovered(output)? { @@ -135,7 +154,7 @@ impl RecallReplacements { Ok(active) } - /// Hand `event` to the lifecycle replacement and account for what it kept; `false` when there is none. + /// Queue `event` for the lifecycle replacement, which runs on the delivery thread; `false` when there is none. pub(crate) fn deliver_lifecycle( &self, py: Python<'_>, @@ -158,23 +177,52 @@ impl RecallReplacements { session.record_filtered(DeliveryStream::Events, 1); return Ok(true); } - let writer = Bound::new(py, RecallWriter::lifecycle(Arc::clone(session)))?; - let raised: PyResult = invoke( - py, - &replacement.function, - (EVENT_PARAMETER, event), - (&replacement.writer_parameter, writer.clone().into_any()), + let schema_version: i64 = native_field(event, "schema_version")?.extract()?; + session.publish_pending_event( + Box::new(PendingEvent { + replacement: Arc::clone(replacement), + event: event.clone().unbind(), + }), + (&event_type, schema_version), ); - let (published, _) = writer.get().finish(); - match raised { - Ok(false) if published == 0 => session.record_filtered(DeliveryStream::Events, 1), - Ok(false) => {} - Ok(true) | Err(_) => session.record_failed(DeliveryStream::Events, 1), - } Ok(true) } } +/// One lifecycle event queued for the lifecycle replacement. +#[derive(Debug)] +struct PendingEvent { + replacement: Arc, + event: Py, +} + +impl PendingEvent { + fn call(&self, py: Python<'_>) -> PyResult { + let writer = Bound::new(py, RecallWriter::lifecycle())?; + let raised: bool = invoke( + py, + &self.replacement.function, + (EVENT_PARAMETER, self.event.bind(py)), + ( + &self.replacement.writer_parameter, + writer.clone().into_any(), + ), + )?; + let (events, _) = writer.get().finish(); + Ok(if raised { + ReplacedEvents::Failed + } else { + ReplacedEvents::Published(events) + }) + } +} + +impl PendingLifecycleEvent for PendingEvent { + fn replace(self: Box) -> ReplacedEvents { + as_replacement_sink(|py| self.call(py), ReplacedEvents::Failed) + } +} + /// The output replacement, called on the Recall delivery thread for each chunk. struct OutputSink { function: Py, @@ -232,11 +280,6 @@ impl OutputSink { } fn call(&self, py: Python<'_>, chunk: &OutputChunk) -> PyResult { - if let Ok(thread) = - imported(py, "threading", "current_thread").and_then(|current| current.call0()) - { - thread.setattr(OUTPUT_SUPPRESSED_ATTRIBUTE, true)?; - } let record = self.record(py, chunk)?; let writer = Bound::new(py, RecallWriter::output(chunk.sequence))?; let raised: bool = invoke( @@ -259,9 +302,6 @@ impl OutputReplacement for OutputSink { if !self.streams[usize::from(chunk.stream == OutputStream::Stderr)] { return ReplacedOutput::Filtered; } - IN_OUTPUT_REPLACEMENT.with(|flag| flag.set(true)); - let replaced = Python::attach(|py| self.call(py, chunk)).unwrap_or(ReplacedOutput::Failed); - IN_OUTPUT_REPLACEMENT.with(|flag| flag.set(false)); - replaced + as_replacement_sink(|py| self.call(py, chunk), ReplacedOutput::Failed) } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs index e5cb911f75..36b96eb21f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs @@ -37,7 +37,7 @@ use crate::bindings::_helpers::recall::constants::{ STDOUT_STREAM_INDEX, }; use crate::bindings::_helpers::recall::events::escaped_text; -use crate::bindings::_helpers::recall::replacements::{RecallReplacements, in_output_replacement}; +use crate::bindings::_helpers::recall::replacements::{RecallReplacements, in_replacement_sink}; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::rules::config_json::dumped_json; @@ -462,7 +462,7 @@ pub(crate) fn forward_output( let Some(session) = session.filter(|session| session.invocation_id() == invocation_id) else { return; }; - if in_output_replacement() { + if in_replacement_sink() { return; } let stream = if stream == STDOUT_STREAM_INDEX { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs index 5511815af6..84e62531ba 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs @@ -5,6 +5,8 @@ use std::sync::Arc; use pyo3::exceptions::PyException; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyclass, pymethods}; use pyo3::types::PyString; +use serde_json::{Map, Value}; +use sqlbuild_recall::records::main::parse_event::parse_event; use sqlbuild_recall::writer::models::RecallSession; use crate::bindings::_helpers::native_host::executor::ExecutorHost; @@ -12,23 +14,35 @@ use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::recall::events::{escaped_text, event_object}; use crate::bindings::_helpers::recall::replacements::RecallReplacements; +/// `event` as the JSON object `lifecycle_event_to_json` encodes; `None` when it cannot be encoded. +pub(crate) fn lifecycle_event_object( + py: Python<'_>, + event: &Bound<'_, PyAny>, +) -> PyResult>> { + if let Some(object) = event_object(event)? { + return Ok(Some(object)); + } + let encoded = host::entry(py, ExecutorHost::LifecycleEventToJson)?.call1((event,))?; + let text = escaped_text(encoded.downcast::()?)?; + match parse_event(&text) { + Ok(object) => Ok(Some(object)), + Err(_unencodable) => Ok(None), + } +} + /// Queue `event` in `session`; an event that cannot be encoded is skipped. pub(crate) fn publish_lifecycle_event( py: Python<'_>, session: &RecallSession, event: &Bound<'_, PyAny>, ) -> PyResult<()> { - if let Some(object) = event_object(event)? { + if let Some(object) = lifecycle_event_object(py, event)? { session.publish_event_object(object); - return Ok(()); } - let encoded = host::entry(py, ExecutorHost::LifecycleEventToJson)?.call1((event,))?; - let text = escaped_text(encoded.downcast::()?)?; - drop(session.publish_event(&text)); Ok(()) } -/// Publishes lifecycle events, through the Recall replacement sink when one is configured, and SQL artifacts of the active session; delivery is best effort. +/// Publishes lifecycle events, through the Recall replacement sink on the delivery thread when one is configured, and SQL artifacts of the active session; delivery is best effort. #[pyclass(module = "sqlbuild._native", frozen)] pub(crate) struct RecallLifecycleSink { pub(crate) session: Arc, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/writer.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/writer.rs index db228b2e63..484f24cd82 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/writer.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/writer.rs @@ -1,22 +1,23 @@ //! `sqlbuild.recall.RecallWriter`: what a Recall replacement sink publishes through. -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use std::sync::{Arc, Mutex, PoisonError}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Mutex, PoisonError}; use pyo3::exceptions::{PyTypeError, PyValueError}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyclass, pymethods}; use pyo3::types::{PyBytes, PyBytesMethods}; -use sqlbuild_recall::writer::models::RecallSession; +use serde_json::{Map, Value}; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; -use crate::bindings::_helpers::recall::sinks::publish_lifecycle_event; +use crate::bindings::_helpers::recall::sinks::lifecycle_event_object; use crate::bindings::_helpers::records::fields::field as native_field; /// The one stream a writer publishes to. #[derive(Debug)] enum Target { - Lifecycle(Arc), + /// The events the sink published for the one it was handed. + Lifecycle(Mutex>>), /// The chunk the sink was handed, by sequence, and what it published for it. Output { sequence: u64, @@ -29,13 +30,12 @@ enum Target { pub(crate) struct RecallWriter { target: Target, open: AtomicBool, - published: AtomicU64, } impl RecallWriter { /// A writer for one lifecycle event handed to the lifecycle replacement. - pub(crate) fn lifecycle(session: Arc) -> Self { - Self::new(Target::Lifecycle(session)) + pub(crate) fn lifecycle() -> Self { + Self::new(Target::Lifecycle(Mutex::new(Vec::new()))) } /// A writer for the output chunk with `sequence` handed to the output replacement. @@ -50,21 +50,25 @@ impl RecallWriter { Self { target, open: AtomicBool::new(true), - published: AtomicU64::new(0), } } - /// End the sink call: later publishing raises, and the records published are returned. - pub(crate) fn finish(&self) -> (u64, Option>) { + /// End the sink call: later publishing raises, and the events or output bytes published are returned. + pub(crate) fn finish(&self) -> (Vec>, Option>) { self.open.store(false, Ordering::SeqCst); - let data = match &self.target { - Target::Lifecycle(_) => None, - Target::Output { published, .. } => published - .lock() - .unwrap_or_else(PoisonError::into_inner) - .take(), - }; - (self.published.load(Ordering::SeqCst), data) + match &self.target { + Target::Lifecycle(published) => ( + std::mem::take(&mut *published.lock().unwrap_or_else(PoisonError::into_inner)), + None, + ), + Target::Output { published, .. } => ( + Vec::new(), + published + .lock() + .unwrap_or_else(PoisonError::into_inner) + .take(), + ), + } } fn ensure_open(&self) -> PyResult<()> { @@ -82,7 +86,7 @@ impl RecallWriter { /// Record `event` in Recall; for the lifecycle replacement sink. fn publish_lifecycle(&self, py: Python<'_>, event: &Bound<'_, PyAny>) -> PyResult<()> { self.ensure_open()?; - let Target::Lifecycle(session) = &self.target else { + let Target::Lifecycle(published) = &self.target else { return Err(PyValueError::new_err( "this RecallWriter belongs to a command-output sink; use publish_command_output", )); @@ -92,8 +96,12 @@ impl RecallWriter { "publish_lifecycle expects a LifecycleEvent", )); } - publish_lifecycle_event(py, session, event)?; - self.published.fetch_add(1, Ordering::SeqCst); + if let Some(object) = lifecycle_event_object(py, event)? { + published + .lock() + .unwrap_or_else(PoisonError::into_inner) + .push(object); + } Ok(()) } @@ -133,7 +141,6 @@ impl RecallWriter { )); } *slot = Some(data); - self.published.fetch_add(1, Ordering::SeqCst); Ok(()) } } diff --git a/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs b/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs index 07ae4cf049..e5ea0d1823 100644 --- a/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs +++ b/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs @@ -342,9 +342,13 @@ pub(crate) fn deliver(shared: &Arc, config: &SessionConfig) { let (batch, closing) = next_batch(shared, next_heartbeat, batching); let built: Built = materializer.build(batch, shared.output_replacement.get().map(Arc::as_ref)); - if !built.settled.is_empty() || !built.unbuildable.is_empty() || !built.filtered.is_empty() + if !built.settled.is_empty() + || !built.unbuildable.is_empty() + || !built.filtered.is_empty() + || built.extra_events > 0 { let mut state = locked(&shared.state); + state.counts_mut(DeliveryStream::Events).accepted += built.extra_events; state.account(&built.settled, Ok(&InsertOutcome::default())); state.account_unbuildable(&built.unbuildable); state.account_filtered(&built.filtered); diff --git a/crates/sqlbuild-recall/src/writer/_helpers/materialize.rs b/crates/sqlbuild-recall/src/writer/_helpers/materialize.rs index f03c812a3e..4ee2a77855 100644 --- a/crates/sqlbuild-recall/src/writer/_helpers/materialize.rs +++ b/crates/sqlbuild-recall/src/writer/_helpers/materialize.rs @@ -13,7 +13,7 @@ use crate::records::main::sha256_hex::sha256_hex; use crate::records::models::{ArtifactInput, OutputChunk, OutputStream, RecallRow, RowEnvelope}; use crate::writer::_helpers::output_chunk::output_chunk; use crate::writer::_helpers::queue::{Payload, Queued, SqlInput}; -use crate::writer::models::{DeliveryStream, ReplacedOutput}; +use crate::writer::models::{DeliveryStream, ReplacedEvents, ReplacedOutput}; use crate::writer::types::OutputReplacement; /// A batch ready to write: its rows, the streams of the records they came from, and records already settled. @@ -28,6 +28,8 @@ pub(crate) struct Built { pub(crate) unbuildable: Vec, /// Records a replacement sink chose not to store. pub(crate) filtered: Vec, + /// Events a lifecycle replacement published beyond the one it was handed, accepted as they are built. + pub(crate) extra_events: u64, } /// Builds rows for one session; remembers which SQL it has stored. @@ -143,6 +145,24 @@ impl Materializer { } Err(_) => built.unbuildable.push(stream), }, + Payload::Pending(pending) => match pending.replace() { + ReplacedEvents::Published(events) if events.is_empty() => { + built.filtered.push(stream); + } + ReplacedEvents::Published(events) => { + built.extra_events += events.len() as u64 - 1; + for event in events { + match event_object_row(&self.envelope, &event) { + Ok(row) => { + built.rows.push(row); + built.written.push(stream); + } + Err(_) => built.unbuildable.push(stream), + } + } + } + ReplacedEvents::Failed => built.unbuildable.push(stream), + }, Payload::Sql(input) => match self.sql_rows(input) { Some(rows) => { built.rows.extend(rows); diff --git a/crates/sqlbuild-recall/src/writer/_helpers/queue.rs b/crates/sqlbuild-recall/src/writer/_helpers/queue.rs index 7a778bdece..837939338d 100644 --- a/crates/sqlbuild-recall/src/writer/_helpers/queue.rs +++ b/crates/sqlbuild-recall/src/writer/_helpers/queue.rs @@ -18,6 +18,7 @@ use crate::writer::_helpers::delivery::{record_stream, stream_of}; use crate::writer::_helpers::tail::{TailRing, TailSegment}; use crate::writer::constants::{QUEUE_BYTE_CAPACITY, QUEUE_RECORD_CAPACITY}; use crate::writer::models::{DeliveryStream, Priority}; +use crate::writer::types::PendingLifecycleEvent; /// SQL a statement executed, hashed and turned into rows on the delivery thread. #[derive(Debug)] @@ -33,6 +34,8 @@ pub(crate) struct SqlInput { pub(crate) enum Payload { Rows(Vec), Event(Map), + /// An event the lifecycle replacement sink decides on when its batch is built. + Pending(Box), Sql(SqlInput), } @@ -49,14 +52,14 @@ impl Queued { pub(crate) fn rows(&self) -> &[RecallRow] { match &self.payload { Payload::Rows(rows) => rows, - Payload::Event(_) | Payload::Sql(_) => &[], + Payload::Event(_) | Payload::Pending(_) | Payload::Sql(_) => &[], } } fn row_count(&self) -> usize { match &self.payload { Payload::Rows(rows) => rows.len(), - Payload::Event(_) => 1, + Payload::Event(_) | Payload::Pending(_) => 1, Payload::Sql(_) => 2, } } diff --git a/crates/sqlbuild-recall/src/writer/_helpers/session.rs b/crates/sqlbuild-recall/src/writer/_helpers/session.rs index 8a4b470c87..bdf58bd515 100644 --- a/crates/sqlbuild-recall/src/writer/_helpers/session.rs +++ b/crates/sqlbuild-recall/src/writer/_helpers/session.rs @@ -14,11 +14,11 @@ use crate::records::main::parse_event::parse_event; use crate::records::models::OutputStream; use crate::writer::_helpers::delivery::locked; use crate::writer::_helpers::queue::{Payload, Queued, SqlInput}; -use crate::writer::constants::{JSON_SCALAR_BYTES, MAX_ARTIFACT_BYTES}; +use crate::writer::constants::{JSON_SCALAR_BYTES, MAX_ARTIFACT_BYTES, PENDING_EVENT_BYTES}; use crate::writer::models::{ DeliveryStream, DeliverySummary, Priority, RecallSession, SetupOutcome, }; -use crate::writer::types::OutputReplacement; +use crate::writer::types::{OutputReplacement, PendingLifecycleEvent}; pub(crate) fn severity_rank(severity: &str) -> usize { SEVERITY_ORDER @@ -109,6 +109,27 @@ impl RecallSession { self.shared.wake.notify_one(); } + /// Queue one lifecycle event of `event_type` for the replacement sink, which runs on the delivery thread within the shutdown budget. + pub fn publish_pending_event( + &self, + event: Box, + (event_type, schema_version): (&str, i64), + ) { + let mut identity: Map = Map::new(); + identity.insert("event_type".to_owned(), Value::from(event_type)); + identity.insert("schema_version".to_owned(), Value::from(schema_version)); + let priority: Priority = + event_severity(&identity).map_or(Priority::Lifecycle, event_priority); + let record = Queued { + payload: Payload::Pending(event), + stream: DeliveryStream::Events, + bytes: PENDING_EVENT_BYTES, + }; + let mut state = locked(&self.shared.state); + state.push_record(record, priority); + self.shared.wake.notify_one(); + } + /// Queue one diagnostic from its canonical JSON, when at or above the minimum severity. pub fn publish_diagnostic(&self, diagnostic_json: &str) -> Result<(), RecordError> { let mut state = locked(&self.shared.state); diff --git a/crates/sqlbuild-recall/src/writer/constants.rs b/crates/sqlbuild-recall/src/writer/constants.rs index f38530229d..653bab6c91 100644 --- a/crates/sqlbuild-recall/src/writer/constants.rs +++ b/crates/sqlbuild-recall/src/writer/constants.rs @@ -27,3 +27,5 @@ pub const MINIMUM_MAINTENANCE_MS: u64 = 200; pub const MAX_ARTIFACT_BYTES: usize = 4 * 1024 * 1024; /// What a JSON scalar counts toward the queue's byte capacity. pub const JSON_SCALAR_BYTES: usize = 8; +/// What a lifecycle event awaiting its replacement sink counts toward the queue's byte capacity. +pub const PENDING_EVENT_BYTES: usize = 1024; diff --git a/crates/sqlbuild-recall/src/writer/models.rs b/crates/sqlbuild-recall/src/writer/models.rs index 3ba49c8f4e..bd59ae3060 100644 --- a/crates/sqlbuild-recall/src/writer/models.rs +++ b/crates/sqlbuild-recall/src/writer/models.rs @@ -79,6 +79,15 @@ pub enum ReplacedOutput { Failed, } +/// What a Recall replacement made of one lifecycle event. +#[derive(Clone, Debug, PartialEq)] +pub enum ReplacedEvents { + /// Store these events, each as `lifecycle_event_to_json` encodes it; none is counted as filtered. + Published(Vec>), + /// Store nothing; counted as failed. + Failed, +} + /// Where a session writes. #[derive(Clone, Debug)] pub enum SessionStore { diff --git a/crates/sqlbuild-recall/src/writer/tests/helpers.rs b/crates/sqlbuild-recall/src/writer/tests/helpers.rs index fc2f8835f7..69d7124038 100644 --- a/crates/sqlbuild-recall/src/writer/tests/helpers.rs +++ b/crates/sqlbuild-recall/src/writer/tests/helpers.rs @@ -16,7 +16,9 @@ use crate::writer::main::start_recall_session::start_recall_session; use crate::writer::models::{ DeliveryStream, DeliverySummary, Priority, SessionConfig, SessionStore, SetupOutcome, }; -use crate::writer::tests::test_types::{PartialStore, PrefixReplacement, RecordedSession}; +use crate::writer::tests::test_types::{ + CopyingReplacement, PartialStore, PrefixReplacement, RecordedSession, +}; /// The envelope of invocation `invocation_id`. pub(crate) fn envelope(invocation_id: &str) -> RowEnvelope { @@ -417,3 +419,25 @@ pub(crate) fn invocation_during_discovery(wait: Duration) -> Vec]) -> (u64, u64, u64, u64) { + let directory = tempfile::tempdir().expect("a temporary directory can be created"); + let session = start_recall_session(config(directory.path().join("recall.sqlite3"), "pending")); + for (index, copies) in copies.iter().enumerate() { + session.publish_pending_event( + Box::new(CopyingReplacement { + index, + copies: *copies, + }), + ("run_started", 2), + ); + } + let counts = stream_counts(&session.close(), DeliveryStream::Events); + ( + counts.accepted, + counts.delivered, + counts.filtered, + counts.failed, + ) +} diff --git a/crates/sqlbuild-recall/src/writer/tests/test_session.rs b/crates/sqlbuild-recall/src/writer/tests/test_session.rs index ad150377fa..09f9875b81 100644 --- a/crates/sqlbuild-recall/src/writer/tests/test_session.rs +++ b/crates/sqlbuild-recall/src/writer/tests/test_session.rs @@ -8,15 +8,15 @@ use crate::writer::models::SetupOutcome; use crate::writer::tests::helpers::unwritable_session; use crate::writer::tests::helpers::{ artifact_counts, close_under_held_lock, event_counts, invocation_during_discovery, - output_stored_while_running, partially_inserted, recorded_session, replaced_session, - session_on_corrupt_store, + output_stored_while_running, partially_inserted, pending_event_counts, recorded_session, + replaced_session, session_on_corrupt_store, }; #[cfg(unix)] use crate::writer::tests::test_types::UnwritableTestCase; use crate::writer::tests::test_types::{ ArtifactSizeTestCase, CorruptStoreTestCase, DiscoveryTestCase, EventBuildTestCase, - HeldLockTestCase, PartialInsertTestCase, RecordedSession, ReplacementTestCase, SessionTestCase, - StoredWhileRunningTestCase, + HeldLockTestCase, PartialInsertTestCase, PendingEventTestCase, RecordedSession, + ReplacementTestCase, SessionTestCase, StoredWhileRunningTestCase, }; const CLOSE_SLACK_SECONDS: f64 = 0.3; @@ -301,3 +301,21 @@ fn given_a_held_context_row_when_discovery_runs_then_the_first_heartbeat_names_t ); } } + +#[test] +fn given_pending_events_when_replaced_on_the_delivery_thread_then_each_outcome_is_accounted() { + let test_cases = [PendingEventTestCase { + description: "one kept, one split in two, one filtered, one failed", + copies: vec![Some(1), Some(2), Some(0), None], + expected_counts: (5, 3, 1, 1), + }]; + + for test_case in test_cases { + assert_eq!( + pending_event_counts(&test_case.copies), + test_case.expected_counts, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-recall/src/writer/tests/test_types.rs b/crates/sqlbuild-recall/src/writer/tests/test_types.rs index 9b5c81a76b..35a80731a7 100644 --- a/crates/sqlbuild-recall/src/writer/tests/test_types.rs +++ b/crates/sqlbuild-recall/src/writer/tests/test_types.rs @@ -5,8 +5,10 @@ use crate::schema::models::Table; use crate::store::errors::StoreError; use crate::store::models::InsertOutcome; use crate::store::types::{RecallStore, RecallStoreOpener}; -use crate::writer::models::{DeliveryStream, Priority, ReplacedOutput, SetupOutcome}; -use crate::writer::types::OutputReplacement; +use crate::writer::models::{ + DeliveryStream, Priority, ReplacedEvents, ReplacedOutput, SetupOutcome, +}; +use crate::writer::types::{OutputReplacement, PendingLifecycleEvent}; /// What one recorded session left in the store. #[derive(Debug, PartialEq)] @@ -169,3 +171,40 @@ pub(crate) struct DiscoveryTestCase { pub(crate) wait: Duration, pub(crate) expected_rows: Vec>, } + +pub(crate) struct PendingEventTestCase { + pub(crate) description: &'static str, + /// How many copies of the event each replacement publishes; `None` when it fails. + pub(crate) copies: Vec>, + /// `(accepted, delivered, filtered, failed)` of the events stream. + pub(crate) expected_counts: (u64, u64, u64, u64), +} + +/// A lifecycle event whose replacement publishes `copies` distinct copies of it, or fails. +#[derive(Debug)] +pub(crate) struct CopyingReplacement { + pub(crate) index: usize, + pub(crate) copies: Option, +} + +impl PendingLifecycleEvent for CopyingReplacement { + fn replace(self: Box) -> ReplacedEvents { + let Some(copies) = self.copies else { + return ReplacedEvents::Failed; + }; + ReplacedEvents::Published( + (0..copies) + .map(|copy| { + serde_json::from_str(&format!( + r#"{{"event_id":"e{}-{copy}","event_type":"run_started","schema_version":2, + "producer":"sqlbuild","producer_version":"1", + "occurred_at":"2026-10-19T00:00:01.000000Z","invocation_id":"pending", + "payload":{{}}}}"#, + self.index + )) + .expect("the fixture is a JSON object") + }) + .collect(), + ) + } +} diff --git a/crates/sqlbuild-recall/src/writer/types.rs b/crates/sqlbuild-recall/src/writer/types.rs index b1b7defa5d..537e3b3348 100644 --- a/crates/sqlbuild-recall/src/writer/types.rs +++ b/crates/sqlbuild-recall/src/writer/types.rs @@ -1,10 +1,16 @@ //! The hook a Recall replacement sink plugs into delivery. use crate::records::models::OutputChunk; -use crate::writer::models::ReplacedOutput; +use crate::writer::models::{ReplacedEvents, ReplacedOutput}; /// Decides, on the delivery thread, what Recall stores for each output chunk. pub trait OutputReplacement: Send + Sync + std::fmt::Debug { /// What to store for `chunk`: its replacement bytes, nothing, or nothing because the replacement failed. fn replace(&self, chunk: &OutputChunk) -> ReplacedOutput; } + +/// A lifecycle event waiting for the replacement sink, which decides on the delivery thread what Recall stores for it. +pub trait PendingLifecycleEvent: Send + std::fmt::Debug { + /// The events to store in place of this one, nothing, or nothing because the replacement failed. + fn replace(self: Box) -> ReplacedEvents; +} diff --git a/fensu.toml b/fensu.toml index 103bb85c38..154893a523 100644 --- a/fensu.toml +++ b/fensu.toml @@ -961,7 +961,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/query_command/query_values.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/recall/console.rs=10", - "crates/sqlbuild-python/src/bindings/_helpers/recall/replacements.rs=7", + "crates/sqlbuild-python/src/bindings/_helpers/recall/replacements.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/recall/writer.rs=1", @@ -1294,7 +1294,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/models.rs=15", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/recall/replacements.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/recall/replacements.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/records/error_classes.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/records/models.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_compare.rs=2", diff --git a/src/sqlbuild/runtime/event_exporting/main/_invoke_recall_sink.py b/src/sqlbuild/runtime/event_exporting/main/_invoke_recall_sink.py index 4a973c62af..d5203ca684 100644 --- a/src/sqlbuild/runtime/event_exporting/main/_invoke_recall_sink.py +++ b/src/sqlbuild/runtime/event_exporting/main/_invoke_recall_sink.py @@ -4,10 +4,10 @@ def invoke_recall_sink(*, function: Callable[..., object], arguments: Mapping[str, object]) -> bool: - """Call a user replacement sink and report whether it raised.""" + """Call a user replacement sink and report whether it raised; interrupts and exits propagate.""" try: _ = function(**arguments) - except BaseException: + except Exception: return True return False diff --git a/tests/integration/src/sqlbuild/cli/recall/_test_types.py b/tests/integration/src/sqlbuild/cli/recall/_test_types.py index 683bf9e1c6..ac54d72345 100644 --- a/tests/integration/src/sqlbuild/cli/recall/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/recall/_test_types.py @@ -183,3 +183,11 @@ class ReplacementLaunchRecordTestCase: description: str sinks: dict[str, str] expected_records: int + + +@dataclass(frozen=True) +class SlowReplacementTestCase: + description: str + shutdown_seconds: float + sink_sleep_seconds: str + expected_max_seconds: float diff --git a/tests/integration/src/sqlbuild/cli/recall/helpers.py b/tests/integration/src/sqlbuild/cli/recall/helpers.py index ebf4633adc..c7028d75fa 100644 --- a/tests/integration/src/sqlbuild/cli/recall/helpers.py +++ b/tests/integration/src/sqlbuild/cli/recall/helpers.py @@ -1,5 +1,6 @@ """Synthetic DuckDB projects and process helpers for local Recall tests.""" +import contextlib import itertools import json import operator @@ -231,6 +232,29 @@ def killed_during_discovery( return recorded, run(project_dir, "recall", "output", "last") +def interrupted_build(project_dir: Path, **env: str) -> tuple[int | None, float]: + """Exit code of a build sent SIGINT after its first logged event, and seconds to stop.""" + + log: Path = project_dir / "events.log" + process: subprocess.Popen[bytes] = start(project_dir, "build", **env) + try: + deadline: float = time.monotonic() + 120 + while not log.exists() and time.monotonic() < deadline: + time.sleep(0.1) + time.sleep(1) + interrupted: float = time.monotonic() + os.killpg(process.pid, signal.SIGINT) + try: + _ = process.communicate(timeout=30) + except subprocess.TimeoutExpired: + return None, time.monotonic() - interrupted + return process.returncode, time.monotonic() - interrupted + finally: + with contextlib.suppress(ProcessLookupError): + os.killpg(process.pid, signal.SIGKILL) + _ = process.wait(timeout=30) + + def concurrent_builds(project_dir: Path, targets: tuple[str, ...]) -> tuple[list[int], int, int]: """Exit codes of concurrent builds, reads made meanwhile and unexpected read failures.""" @@ -383,6 +407,14 @@ def simultaneous_builds(project_dir: Path, targets: tuple[str, ...]) -> list[int " with (Path(__file__).parents[1] / 'output.log').open('a') as log:\n" " log.write(record.message)\n" ) +SLOW_LIFECYCLE_SINK: str = ( + "@lifecycle_event_sink\n" + "def recall_events(event: LifecycleEvent, writer: RecallWriter) -> None:\n" + " import os\n" + " import time\n\n" + " time.sleep(float(os.environ.get('ORDERS_SINK_SLEEP_SECONDS', '0')))\n" + " writer.publish_lifecycle(event)\n" +) FAILING_SINKS: str = ( "@lifecycle_event_sink\n" "def recall_events(event: LifecycleEvent, writer: RecallWriter) -> None:\n" diff --git a/tests/integration/src/sqlbuild/cli/recall/test_recall_replacement_sinks.py b/tests/integration/src/sqlbuild/cli/recall/test_recall_replacement_sinks.py index ae0e522f87..f993680526 100644 --- a/tests/integration/src/sqlbuild/cli/recall/test_recall_replacement_sinks.py +++ b/tests/integration/src/sqlbuild/cli/recall/test_recall_replacement_sinks.py @@ -1,6 +1,7 @@ from __future__ import annotations import subprocess +import time from pathlib import Path import pytest @@ -11,13 +12,16 @@ ReplacementLaunchRecordTestCase, ReplacementLifecycleTestCase, ReplacementOutputTestCase, + SlowReplacementTestCase, ) from tests.integration.src.sqlbuild.cli.recall.helpers import ( EVENT_LOG_SINK, FAILING_SINKS, NATIVE_SQB, + SLOW_LIFECYCLE_SINK, environment, filtering_lifecycle_sink, + interrupted_build, logged_event_types, recall_rows, rewriting_output_sink, @@ -183,5 +187,70 @@ def test_given_only_replacement_sinks_when_compiling_through_sqb_then_no_launch_ assert len(records) == test_case.expected_records +@pytest.mark.parametrize( + "test_case", + [ + SlowReplacementTestCase( + description="a sink sleeping past the budget costs the command at most the budget", + shutdown_seconds=1.0, + sink_sleep_seconds="30", + expected_max_seconds=12.0, + ), + ], + ids=lambda case: case.description, +) +def test_given_a_lifecycle_sink_slower_than_the_budget_when_building_then_nothing_else_waits( + test_case: SlowReplacementTestCase, tmp_path: Path +) -> None: + budget: str = f'\n[recall]\nshutdown_timeout = "{int(test_case.shutdown_seconds * 1000)}ms"\n' + project_dir: Path = write_sinks( + write_project(tmp_path / "orders_shop", extra_toml=budget), + {"recall_events": SLOW_LIFECYCLE_SINK, "event_log": EVENT_LOG_SINK}, + ) + + started: float = time.monotonic() + built: subprocess.CompletedProcess[bytes] = run( + project_dir, "build", ORDERS_SINK_SLEEP_SECONDS=test_case.sink_sleep_seconds + ) + elapsed: float = time.monotonic() - started + + logged: list[str] = (project_dir / "events.log").read_text(encoding="utf-8").splitlines() + assert built.returncode == 0, built.stderr + assert elapsed < test_case.expected_max_seconds + assert logged[-1] == "invocation_completed" + assert b"Recall history for this run is incomplete" in built.stderr + + +@pytest.mark.parametrize( + "test_case", + [ + SlowReplacementTestCase( + description="Ctrl-C stops the build while the sink sleeps", + shutdown_seconds=1.0, + sink_sleep_seconds="30", + expected_max_seconds=10.0, + ), + ], + ids=lambda case: case.description, +) +def test_given_a_slow_lifecycle_sink_when_interrupted_then_the_build_stops( + test_case: SlowReplacementTestCase, tmp_path: Path +) -> None: + budget: str = f'\n[recall]\nshutdown_timeout = "{int(test_case.shutdown_seconds * 1000)}ms"\n' + project_dir: Path = write_sinks( + write_project(tmp_path / "orders_shop", extra_toml=budget), + {"recall_events": SLOW_LIFECYCLE_SINK, "event_log": EVENT_LOG_SINK}, + ) + + exit_code, stopped_after = interrupted_build( + project_dir, + ORDERS_SINK_SLEEP_SECONDS=test_case.sink_sleep_seconds, + ORDERS_TASK_SLEEP_SECONDS="60", + ) + + assert exit_code not in (None, 0) + assert stopped_after < test_case.expected_max_seconds + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/website/src/content/docs/docs/concepts/observability/recall.mdx b/website/src/content/docs/docs/concepts/observability/recall.mdx index 7d2d249551..69a1328e64 100644 --- a/website/src/content/docs/docs/concepts/observability/recall.mdx +++ b/website/src/content/docs/docs/concepts/observability/recall.mdx @@ -169,9 +169,12 @@ def recall_output(record: CommandOutputRecord, writer: RecallWriter) -> None: keeps the record ID and sequence; publish at most one record per call. - A `@lifecycle_event_sink` replacement receives each event its `event_kinds` and `min_severity` accept and keeps it with `writer.publish_lifecycle(event)`. -- A record the sink does not publish is counted as `filtered`; a sink that raises has its record - counted as `failed`, and Recall never falls back to storing the unfiltered record. The command's - exit code and output are unaffected. `output_invocations.filter_active` is `1` for invocations +- Both replacements run on Recall's background thread, so a slow one never delays the command or + other sinks by more than `shutdown_timeout`; what it has not handled by then is counted as + `dropped`. Output written before discovery finishes is held until the replacement can see it. +- A record the sink does not publish is counted as `filtered`; a sink that raises an exception has + its record counted as `failed`, and Recall never falls back to storing the unfiltered record. The + command's exit code and output are unaffected. `output_invocations.filter_active` is `1` for invocations that ran a replacement. - There is at most one replacement per stream; discovery fails with `D015` naming each `path:function` otherwise. A replacement takes only its record and the writer, not providers, and From b755ce28a98f0e295bca5dc753457da45e6eb05a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 13:58:09 +0100 Subject: [PATCH 14/19] fix: add missing Recall columns to SQLite stores at the same schema version and write only stored columns --- .../src/store/_helpers/rows.rs | 32 ++++-- .../src/store/_helpers/schema.rs | 76 ++++++++----- .../src/writer/tests/helpers.rs | 101 ++++++++++++++++++ .../src/writer/tests/test_session.rs | 46 ++++++-- .../src/writer/tests/test_types.rs | 6 ++ 5 files changed, 220 insertions(+), 41 deletions(-) diff --git a/crates/sqlbuild-recall/src/store/_helpers/rows.rs b/crates/sqlbuild-recall/src/store/_helpers/rows.rs index 91ee58ea45..5fed0c3900 100644 --- a/crates/sqlbuild-recall/src/store/_helpers/rows.rs +++ b/crates/sqlbuild-recall/src/store/_helpers/rows.rs @@ -1,11 +1,15 @@ //! Idempotent row writes: insert once, compare content on a duplicate key, record conflicts. +use std::collections::hash_map::Entry; +use std::collections::{HashMap, HashSet}; + use rusqlite::{Connection, OptionalExtension}; use crate::records::main::conflict_row::conflict_row; use crate::records::models::{RecallRow, RecallValue, RowEnvelope}; use crate::schema::main::table_spec::table_spec; use crate::schema::models::Table; +use crate::store::_helpers::schema::stored_columns; use crate::store::_helpers::sqlite::{SqlValue, checked}; use crate::store::errors::StoreError; use crate::store::models::{ConflictEvidence, InsertOutcome}; @@ -24,15 +28,20 @@ fn text_of(row: &RecallRow, column: &str) -> String { key_text(row.value(column)) } -/// Insert one row; `Some(evidence)` when its key holds different content. +/// Insert one row's values for the columns its table has; `Some(evidence)` when its key holds different content. fn write_row( connection: &Connection, - row: &RecallRow, + (row, columns): (&RecallRow, &HashSet), ingested_at: &str, ) -> Result, StoreError> { let spec = table_spec(row.table); let ingested: RecallValue = RecallValue::Text(ingested_at.to_owned()); - let mut names: Vec<&str> = row.values.iter().map(|(name, _)| *name).collect(); + let stored: Vec<&(&str, RecallValue)> = row + .values + .iter() + .filter(|(name, _)| columns.contains(*name)) + .collect(); + let mut names: Vec<&str> = stored.iter().map(|(name, _)| *name).collect(); names.push("ingested_at"); let placeholders: Vec = (1..=names.len()).map(|index| format!("?{index}")).collect(); let sql = format!( @@ -41,8 +50,7 @@ fn write_row( names.join(", "), placeholders.join(", ") ); - let parameters: Vec> = row - .values + let parameters: Vec> = stored .iter() .map(|(_, value)| SqlValue(value)) .chain(std::iter::once(SqlValue(&ingested))) @@ -105,8 +113,14 @@ pub(crate) fn write_rows( ingested_at: &str, ) -> Result { let mut outcome = InsertOutcome::default(); + let mut columns: HashMap> = HashMap::new(); + for row in rows.iter().map(|row| row.table).chain([Table::Conflicts]) { + if let Entry::Vacant(entry) = columns.entry(row) { + entry.insert(stored_columns(connection, row)?); + } + } for row in rows { - match write_row(connection, row, ingested_at)? { + match write_row(connection, (row, &columns[&row.table]), ingested_at)? { None => match outcome .delivered .iter_mut() @@ -128,7 +142,11 @@ pub(crate) fn write_rows( ingested_at, ); if row.table != Table::Conflicts { - write_row(connection, &conflict, ingested_at)?; + write_row( + connection, + (&conflict, &columns[&Table::Conflicts]), + ingested_at, + )?; } outcome.conflicts.push(evidence); } diff --git a/crates/sqlbuild-recall/src/store/_helpers/schema.rs b/crates/sqlbuild-recall/src/store/_helpers/schema.rs index 0a95844fcb..6b4c35bfd7 100644 --- a/crates/sqlbuild-recall/src/store/_helpers/schema.rs +++ b/crates/sqlbuild-recall/src/store/_helpers/schema.rs @@ -8,7 +8,7 @@ use crate::schema::constants::{ALL_TABLES, RECALL_SCHEMA_VERSION, VIEW_NAMES}; use crate::schema::main::add_column_statement::add_column_statement; use crate::schema::main::table_columns::table_columns; use crate::schema::main::table_spec::table_spec; -use crate::schema::models::{DdlOptions, Dialect}; +use crate::schema::models::{ColumnSpec, DdlOptions, Dialect, Table}; use crate::store::_helpers::sqlite::{checked, stored_schema_version}; use crate::store::errors::StoreError; @@ -20,7 +20,7 @@ pub(crate) fn sqlite_options() -> DdlOptions { } } -/// Whether every table and view exists at this build's schema version, read without a lock. +/// Whether every table, column and view exists at this build's schema version, read without a lock. pub(crate) fn current(connection: &Connection) -> Result { if stored_schema_version(connection)?.is_none_or(|version| version < RECALL_SCHEMA_VERSION) { return Ok(false); @@ -34,7 +34,10 @@ pub(crate) fn current(connection: &Connection) -> Result { ), "inspect Recall store", )?; - Ok(present == i64::try_from(ALL_TABLES.len() + VIEW_NAMES.len()).unwrap_or(i64::MAX)) + if present != i64::try_from(ALL_TABLES.len() + VIEW_NAMES.len()).unwrap_or(i64::MAX) { + return Ok(false); + } + Ok(missing_columns(connection)?.is_empty()) } fn expected_names() -> String { @@ -46,35 +49,52 @@ fn expected_names() -> String { serde_json::Value::from(names).to_string() } +/// The names of the columns `table` has in the store. +pub(crate) fn stored_columns( + connection: &Connection, + table: Table, +) -> Result, StoreError> { + let mut statement = checked( + connection.prepare_cached(&format!( + "SELECT name FROM pragma_table_xinfo('{}')", + table_spec(table).name + )), + "inspect Recall table", + )?; + checked( + statement + .query_map([], |row| row.get::<_, String>(0)) + .and_then(Iterator::collect), + "inspect Recall table", + ) +} + +/// Every column this build writes that the stored tables lack. +fn missing_columns(connection: &Connection) -> Result, StoreError> { + let mut missing: Vec<(Table, ColumnSpec)> = Vec::new(); + for table in ALL_TABLES { + let existing: HashSet = stored_columns(connection, table)?; + missing.extend( + table_columns(table) + .into_iter() + .filter(|column| !existing.contains(column.name)) + .map(|column| (table, column)), + ); + } + Ok(missing) +} + pub(crate) fn add_missing_columns(connection: &Connection) -> Result<(), StoreError> { let options = sqlite_options(); - for table in ALL_TABLES { - let spec = table_spec(table); - let mut statement = checked( - connection.prepare(&format!( - "SELECT name FROM pragma_table_xinfo('{}')", - spec.name + for (table, column) in missing_columns(connection)? { + checked( + connection.execute_batch(&add_column_statement( + Dialect::Sqlite, + &options, + (table_spec(table), &column), )), - "inspect Recall table", - )?; - let existing: HashSet = checked( - statement - .query_map([], |row| row.get::<_, String>(0)) - .and_then(Iterator::collect), - "inspect Recall table", + "add Recall column", )?; - for column in table_columns(table) { - if !existing.contains(column.name) { - checked( - connection.execute_batch(&add_column_statement( - Dialect::Sqlite, - &options, - (spec, &column), - )), - "add Recall column", - )?; - } - } } Ok(()) } diff --git a/crates/sqlbuild-recall/src/writer/tests/helpers.rs b/crates/sqlbuild-recall/src/writer/tests/helpers.rs index 69d7124038..97924d3932 100644 --- a/crates/sqlbuild-recall/src/writer/tests/helpers.rs +++ b/crates/sqlbuild-recall/src/writer/tests/helpers.rs @@ -441,3 +441,104 @@ pub(crate) fn pending_event_counts(copies: &[Option]) -> (u64, u64, u64, counts.failed, ) } + +/// The heartbeat columns the first Recall release lacks, and the view text that reads them. +const CORE_RELEASE_EDITS: [(&str, &str); 3] = [ + ( + ",\n MIN(command) AS command, MIN(subcommand) AS subcommand", + "", + ), + ( + "COALESCE(context.command, started.command, beats.command)", + "COALESCE(context.command, started.command)", + ), + ( + "COALESCE(context.subcommand, beats.subcommand) AS subcommand", + "context.subcommand", + ), +]; + +/// A store at schema version 1 as the first Recall release creates it: heartbeats without `command` or `subcommand`. +pub(crate) fn core_release_store(path: &std::path::Path) { + use crate::schema::main::ddl_script::ddl_script; + use crate::schema::main::view_statements::view_statements; + use crate::schema::models::{DdlOptions, Dialect}; + + let options = DdlOptions { + database: None, + retention_days: 30, + cluster: None, + }; + std::fs::create_dir_all(path.parent().expect("the store has a directory")) + .expect("the store directory can be created"); + let connection = rusqlite::Connection::open(path).expect("the store opens"); + connection + .execute_batch(&ddl_script(Dialect::Sqlite, &options)) + .expect("the script runs"); + let views = view_statements(Dialect::Sqlite, &options); + for (name, _) in &views { + connection + .execute_batch(&format!("DROP VIEW \"{name}\"")) + .expect("the view drops"); + } + connection + .execute_batch( + "ALTER TABLE recall_heartbeats DROP COLUMN command; \ + ALTER TABLE recall_heartbeats DROP COLUMN subcommand; \ + INSERT INTO recall_meta (component, key, value, recorded_at, invocation_id, \ + invocation_started_at, schema_version, producer_version, ingested_at, content_hash) \ + VALUES ('schema', 'schema_version', '1', '2026-10-01T00:00:00.000000Z', 'core', \ + '2026-10-01T00:00:00.000000Z', 1, 'core', '2026-10-01T00:00:00.000000Z', '')", + ) + .expect("the release schema is restored"); + for (_, statement) in views { + let released: String = CORE_RELEASE_EDITS + .iter() + .fold(statement, |text, (new, old)| text.replace(new, old)); + connection + .execute_batch(&released) + .expect("the release view is created"); + } +} + +/// Write a heartbeat naming the command straight into the store, before any upgrade. +pub(crate) fn insert_heartbeat(path: &std::path::Path) { + use crate::records::main::heartbeat_row::heartbeat_row; + use crate::store::main::insert_rows::insert_rows; + + let store = open_sqlite_store(path, OpenMode::ReadWrite).expect("the store opens"); + let beat = heartbeat_row( + &envelope("core"), + ("2026-10-19T00:00:01.000000Z", "2026-10-19T00:01:31.000000Z"), + (None, Some(1)), + (Some("build"), None), + ); + insert_rows(&store, &[beat], "2026-10-19T00:00:01.000000Z").expect("the row is written"); +} + +/// Record a short run through a session. +pub(crate) fn record_run(path: &std::path::Path) { + let session = start_recall_session(config(path.to_path_buf(), "core")); + session.append_output(OutputStream::Stdout, b"built\n"); + drop(session.wait_for_setup(Duration::from_secs(10))); + std::thread::sleep(Duration::from_millis(300)); + drop(session.close()); +} + +/// What `write` leaves in a store the first Recall release created. +pub(crate) fn written_to_core_release_store(write: fn(&std::path::Path)) -> Vec> { + let directory = tempfile::tempdir().expect("a temporary directory can be created"); + let path = directory.path().join("logs/recall.sqlite3"); + core_release_store(&path); + write(&path); + let store = open_sqlite_store(&path, OpenMode::ReadOnly).expect("the store opens"); + query_store( + &store, + "SELECT (SELECT COUNT(*) FROM recall_heartbeats), (SELECT command FROM invocations), \ + (SELECT COUNT(*) FROM pragma_table_xinfo('recall_heartbeats') \ + WHERE name IN ('command', 'subcommand'))", + None, + ) + .expect("the query is valid") + .rows +} diff --git a/crates/sqlbuild-recall/src/writer/tests/test_session.rs b/crates/sqlbuild-recall/src/writer/tests/test_session.rs index 09f9875b81..a78f741b66 100644 --- a/crates/sqlbuild-recall/src/writer/tests/test_session.rs +++ b/crates/sqlbuild-recall/src/writer/tests/test_session.rs @@ -7,16 +7,17 @@ use crate::writer::models::SetupOutcome; #[cfg(unix)] use crate::writer::tests::helpers::unwritable_session; use crate::writer::tests::helpers::{ - artifact_counts, close_under_held_lock, event_counts, invocation_during_discovery, - output_stored_while_running, partially_inserted, pending_event_counts, recorded_session, - replaced_session, session_on_corrupt_store, + artifact_counts, close_under_held_lock, event_counts, insert_heartbeat, + invocation_during_discovery, output_stored_while_running, partially_inserted, + pending_event_counts, record_run, recorded_session, replaced_session, session_on_corrupt_store, + written_to_core_release_store, }; #[cfg(unix)] use crate::writer::tests::test_types::UnwritableTestCase; use crate::writer::tests::test_types::{ - ArtifactSizeTestCase, CorruptStoreTestCase, DiscoveryTestCase, EventBuildTestCase, - HeldLockTestCase, PartialInsertTestCase, PendingEventTestCase, RecordedSession, - ReplacementTestCase, SessionTestCase, StoredWhileRunningTestCase, + ArtifactSizeTestCase, CoreReleaseStoreTestCase, CorruptStoreTestCase, DiscoveryTestCase, + EventBuildTestCase, HeldLockTestCase, PartialInsertTestCase, PendingEventTestCase, + RecordedSession, ReplacementTestCase, SessionTestCase, StoredWhileRunningTestCase, }; const CLOSE_SLACK_SECONDS: f64 = 0.3; @@ -319,3 +320,36 @@ fn given_pending_events_when_replaced_on_the_delivery_thread_then_each_outcome_i ); } } + +#[test] +fn given_a_store_from_the_first_release_when_written_then_rows_land_and_columns_are_added() { + let test_cases = [ + CoreReleaseStoreTestCase { + description: "a session adds the heartbeat columns and records the run", + write: record_run, + expected_rows: vec![vec![ + RecallValue::Integer(1), + RecallValue::Text("build".to_owned()), + RecallValue::Integer(2), + ]], + }, + CoreReleaseStoreTestCase { + description: "a write before any upgrade stores only the columns the table has", + write: insert_heartbeat, + expected_rows: vec![vec![ + RecallValue::Integer(1), + RecallValue::Null, + RecallValue::Integer(0), + ]], + }, + ]; + + for test_case in test_cases { + assert_eq!( + written_to_core_release_store(test_case.write), + test_case.expected_rows, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-recall/src/writer/tests/test_types.rs b/crates/sqlbuild-recall/src/writer/tests/test_types.rs index 35a80731a7..ac3229898a 100644 --- a/crates/sqlbuild-recall/src/writer/tests/test_types.rs +++ b/crates/sqlbuild-recall/src/writer/tests/test_types.rs @@ -208,3 +208,9 @@ impl PendingLifecycleEvent for CopyingReplacement { ) } } + +pub(crate) struct CoreReleaseStoreTestCase { + pub(crate) description: &'static str, + pub(crate) write: fn(&std::path::Path), + pub(crate) expected_rows: Vec>, +} From 4224583685033d6f6134c2e141923ed556319aac Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 14:03:48 +0100 Subject: [PATCH 15/19] fix: keep writing a ClickHouse Recall store whose user may not add new optional columns --- .../_helpers/recall/clickhouse_store.rs | 138 ++++++++++++++---- .../commands/main/clickhouse/_test_types.py | 8 + .../cli/commands/main/clickhouse/helpers.py | 28 +++- .../main/clickhouse/test_clickhouse_recall.py | 55 +++++++ 4 files changed, 194 insertions(+), 35 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs index 12bbe6ff39..0ed9bb2e5c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs @@ -90,6 +90,12 @@ struct StoreState { version: i64, } +/// A store ready to write: its tables with their `CREATE` queries, and the column additions an administrator must run. +struct ReadyStore { + tables: Vec<(String, String)>, + refused_columns: Vec, +} + /// The store the delivery thread writes. struct ClickHouseStore { location: ClickHouseLocation, @@ -218,48 +224,36 @@ impl ClickHouseStore { Ok(()) } - /// Create missing tables, add missing columns and replace the views when this build's schema is newer or columns were added; a newer store keeps its views. + /// Create a store whose tables are missing; on existing tables add missing columns where allowed, create missing views, and replace views only on a version upgrade. fn ensure_schema( &mut self, py: Python<'_>, envelope: &RowEnvelope, - ) -> Result, StoreError> { + ) -> Result { let StoreState { tables, views, version, } = self.state(py)?; - let complete: bool = tables.len() == ALL_TABLES.len() && views.len() == VIEW_NAMES.len(); - if complete { - self.read_columns(py)?; - let columns_complete: bool = self.missing_columns().is_empty(); - if columns_complete && version >= RECALL_SCHEMA_VERSION { - return Ok(tables); - } - if columns_complete && version == 0 { - self.record_version(py, envelope)?; - return Ok(tables); - } - } - self.require_database(py)?; - for statement in table_statements(Dialect::ClickHouse, &self.location.options) { - self.command(py, &statement, "create the Recall tables")?; + if tables.len() < ALL_TABLES.len() { + self.create(py, envelope, &views)?; + return Ok(ReadyStore { + tables: self.state(py)?.tables, + refused_columns: Vec::new(), + }); } self.read_columns(py)?; - let missing: Vec<(Table, ColumnSpec)> = self.missing_columns(); - let added: bool = !missing.is_empty(); - for (table, column) in missing { - let statement = add_column_statement( - Dialect::ClickHouse, - &self.location.options, - (table_spec(table), &column), - ); - self.command(py, &statement, "add a Recall column")?; + let refused: Vec = self.add_columns(py)?; + let upgrade: bool = version > 0 && version < RECALL_SCHEMA_VERSION; + let views_due: bool = upgrade || views.len() < VIEW_NAMES.len(); + if views_due && !refused.is_empty() { + return Err(StoreError { + kind: StoreErrorKind::Permission, + message: "the Recall views need columns this user may not add".to_owned(), + }); } - self.read_columns(py)?; - let upgrade: bool = version < RECALL_SCHEMA_VERSION; for (name, statement) in view_statements(Dialect::ClickHouse, &self.location.options) { - if upgrade || added { + if upgrade { self.command(py, &statement, "create the Recall views")?; } else if !views.contains(name) { let statement = @@ -267,10 +261,68 @@ impl ClickHouseStore { self.command(py, &statement, "create the Recall views")?; } } - if upgrade { + if version < RECALL_SCHEMA_VERSION { self.record_version(py, envelope)?; } - Ok(self.state(py)?.tables) + Ok(ReadyStore { + tables, + refused_columns: refused, + }) + } + + /// Create every table, column and view of a store whose tables are missing. + fn create( + &mut self, + py: Python<'_>, + envelope: &RowEnvelope, + views: &HashSet, + ) -> Result<(), StoreError> { + self.require_database(py)?; + for statement in table_statements(Dialect::ClickHouse, &self.location.options) { + self.command(py, &statement, "create the Recall tables")?; + } + self.read_columns(py)?; + let refused: Vec = self.add_columns(py)?; + if !refused.is_empty() { + return Err(StoreError { + kind: StoreErrorKind::Permission, + message: "the Recall tables need columns this user may not add".to_owned(), + }); + } + for (name, statement) in view_statements(Dialect::ClickHouse, &self.location.options) { + if !views.contains(name) { + self.command(py, &statement, "create the Recall views")?; + } + } + self.record_version(py, envelope) + } + + /// Add the additive columns the stored tables lack; when the user may not, the statements an administrator must run. + fn add_columns(&mut self, py: Python<'_>) -> Result, StoreError> { + let missing: Vec<(Table, ColumnSpec)> = self.missing_columns(); + let statements: Vec = missing + .iter() + .map(|(table, column)| { + add_column_statement( + Dialect::ClickHouse, + &self.location.options, + (table_spec(*table), column), + ) + }) + .collect(); + for (index, statement) in statements.iter().enumerate() { + match self.command(py, statement, "add a Recall column") { + Ok(()) => {} + Err(error) if error.kind == StoreErrorKind::Permission => { + return Ok(statements[index..].to_vec()); + } + Err(error) => return Err(error), + } + } + if !missing.is_empty() { + self.read_columns(py)?; + } + Ok(Vec::new()) } /// Every column this build writes that the stored tables lack. @@ -349,6 +401,17 @@ impl ClickHouseStore { .collect()) } + /// The notice for columns this user may not add: history is written without them until an administrator adds them. + fn columns_notice(&self, statements: &[String]) -> String { + format!( + "the ClickHouse store in `{}` lacks columns this version of SQLBuild writes, and this \ + user may not add them; history is still written without them. An administrator can \ + add them with:\n\n{};\n", + self.location.database(), + statements.join(";\n\n") + ) + } + fn retention_conflict(&self, tables: &[(String, String)]) -> Option { let mismatches: Vec = ttl_mismatches(tables, self.location.options.retention_days); @@ -436,7 +499,18 @@ impl RecallStoreOpener for ClickHouseStoreOpener { notices: Vec::new(), }; match store.ensure_schema(py, envelope) { - Ok(tables) => { + Ok(ReadyStore { + tables, + refused_columns: refused, + }) => { + if !refused.is_empty() + && (self.retention_notice)( + &self.project_dir, + &format!("{}:columns", self.location.database()), + ) + { + store.notices.push(store.columns_notice(&refused)); + } if let Some(notice) = store.retention_conflict(&tables) && (self.retention_notice)( &self.project_dir, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py index 48d9ce4a7d..7312b6365a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py @@ -183,3 +183,11 @@ class ClickHouseRecallQueryOutputSinkTestCase: description: str sql: str expected_fragment: str + + +@dataclass(frozen=True) +class ClickHouseRecallFirstReleaseStoreTestCase: + description: str + expected_recorded: list[list[object]] + expected_notice: str + expected_heartbeat_columns: tuple[tuple[object, ...], ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py index 4e83dc5d7b..76d02e003c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/helpers.py @@ -2,6 +2,7 @@ from __future__ import annotations +import functools import re import uuid from collections.abc import Iterator @@ -458,10 +459,31 @@ def apply_printed_retention_fix(*, config: dict[str, object], stderr: str) -> in return len(statements) -def apply_printed_ddl(*, config: dict[str, object], stderr: str) -> None: - """Run, as an administrator, the script a permission notice printed for creating the store.""" +FIRST_RELEASE_EDITS: tuple[tuple[str, str], ...] = ( + ( + " `pid` Nullable(Int64),\n `command` Nullable(String),\n" + " `subcommand` Nullable(String),", + " `pid` Nullable(Int64),", + ), + (",\n min(command) AS command, min(subcommand) AS subcommand", ""), + ( + "coalesce(context.command, started.command, beats.command)", + "coalesce(context.command, started.command)", + ), + ( + "coalesce(context.subcommand, beats.subcommand) AS subcommand", + "context.subcommand AS subcommand", + ), +) + + +def apply_printed_ddl( + *, config: dict[str, object], stderr: str, edits: tuple[tuple[str, str], ...] = () +) -> None: + """Run, as an administrator, the script a permission notice printed, with `edits` applied.""" - script: str = stderr.split(" with:\n\n", 1)[1].split("\nsqb ", 1)[0].strip() + printed: str = stderr.split(" with:\n\n", 1)[1].split("\nsqb ", 1)[0].strip() + script: str = functools.reduce(lambda text, edit: text.replace(*edit), edits, printed) execute_clickhouse_sql( config=config, statements=tuple( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py index 3a41ad74c3..5d0e475ce5 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py @@ -13,6 +13,7 @@ ClickHouseRecallConcurrencyTestCase, ClickHouseRecallDuplicateTestCase, ClickHouseRecallExpiryTestCase, + ClickHouseRecallFirstReleaseStoreTestCase, ClickHouseRecallLimitedWriterTestCase, ClickHouseRecallOutputTestCase, ClickHouseRecallPartialInsertTestCase, @@ -22,6 +23,7 @@ ClickHouseRecallRetentionTestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.clickhouse.helpers import ( + FIRST_RELEASE_EDITS, age_recorded_invocations, age_stored_artifacts, apply_printed_ddl, @@ -638,5 +640,58 @@ def test_given_an_output_sink_when_querying_clickhouse_recall_then_the_sink_sees assert test_case.expected_fragment in (project_dir / "output.log").read_text(encoding="utf-8") +@pytest.mark.parametrize( + "test_case", + [ + ClickHouseRecallFirstReleaseStoreTestCase( + description="heartbeats without the newer columns are still written", + expected_recorded=[["build", "completed", 1]], + expected_notice="ADD COLUMN IF NOT EXISTS `command` Nullable(String)", + expected_heartbeat_columns=((0,),), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_first_release_store_when_a_select_insert_user_builds_then_history_is_written( + test_case: ClickHouseRecallFirstReleaseStoreTestCase, + tmp_path: Path, + clickhouse_e2e_config: dict[str, object], + recall_database: str, +) -> None: + readonly: Path = write_recall_project( + parent=tmp_path / "readonly", + config=clickhouse_e2e_config, + database=recall_database, + settings=("", "settings = { readonly = 2 }\n"), + ) + apply_printed_ddl( + config=clickhouse_e2e_config, + stderr=run(readonly, "build").stderr.decode(errors="replace"), + edits=FIRST_RELEASE_EDITS, + ) + + with limited_user(config=clickhouse_e2e_config, database=recall_database) as writer: + project_dir: Path = write_recall_project( + parent=tmp_path / "writer", config=writer, database=recall_database + ) + built: subprocess.CompletedProcess[bytes] = run(project_dir, "build") + recorded: list[list[object]] = recall_rows( + project_dir, + "SELECT command, status, toUInt8(last_heartbeat_at IS NOT NULL) FROM invocations", + ) + + assert built.returncode == 0, built.stderr + assert test_case.expected_notice in built.stderr.decode(errors="replace") + assert recorded == test_case.expected_recorded + assert ( + fetch_clickhouse_rows( + config=clickhouse_e2e_config, + sql=f"SELECT count() FROM system.columns WHERE database = '{recall_database}' " + "AND table = 'recall_heartbeats' AND name IN ('command', 'subcommand')", + ) + == test_case.expected_heartbeat_columns + ) + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) From 99bec8031b7ef97fc9d0983aed132695e13ff293 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 14:03:48 +0100 Subject: [PATCH 16/19] fix: hand lifecycle events to the Recall replacement sink in publication order --- .../src/writer/_helpers/materialize.rs | 46 +++++++++++-------- .../src/writer/_helpers/queue.rs | 31 +++++++++++-- .../src/writer/_helpers/session.rs | 9 +--- .../src/writer/tests/helpers.rs | 22 ++++++++- .../src/writer/tests/test_session.rs | 33 +++++++++++-- .../src/writer/tests/test_types.rs | 23 ++++++++++ 6 files changed, 127 insertions(+), 37 deletions(-) diff --git a/crates/sqlbuild-recall/src/writer/_helpers/materialize.rs b/crates/sqlbuild-recall/src/writer/_helpers/materialize.rs index 4ee2a77855..dd17ee103a 100644 --- a/crates/sqlbuild-recall/src/writer/_helpers/materialize.rs +++ b/crates/sqlbuild-recall/src/writer/_helpers/materialize.rs @@ -14,7 +14,7 @@ use crate::records::models::{ArtifactInput, OutputChunk, OutputStream, RecallRow use crate::writer::_helpers::output_chunk::output_chunk; use crate::writer::_helpers::queue::{Payload, Queued, SqlInput}; use crate::writer::models::{DeliveryStream, ReplacedEvents, ReplacedOutput}; -use crate::writer::types::OutputReplacement; +use crate::writer::types::{OutputReplacement, PendingLifecycleEvent}; /// A batch ready to write: its rows, the streams of the records they came from, and records already settled. #[derive(Debug, Default)] @@ -114,13 +114,14 @@ impl Materializer { Some(artifact_rows(&self.envelope, &artifact).to_vec()) } - /// The rows of `batch`, with each record's stream sorted by how it is accounted. + /// The rows of `batch`, with each record's stream sorted by how it is accounted; lifecycle replacements see their events in publication order. pub(crate) fn build( &mut self, batch: Vec, replacement: Option<&dyn OutputReplacement>, ) -> Built { let mut built = Built::default(); + let mut pending_events: Vec<(u64, Box)> = Vec::new(); for record in batch { let stream: DeliveryStream = record.stream; match record.payload { @@ -145,24 +146,7 @@ impl Materializer { } Err(_) => built.unbuildable.push(stream), }, - Payload::Pending(pending) => match pending.replace() { - ReplacedEvents::Published(events) if events.is_empty() => { - built.filtered.push(stream); - } - ReplacedEvents::Published(events) => { - built.extra_events += events.len() as u64 - 1; - for event in events { - match event_object_row(&self.envelope, &event) { - Ok(row) => { - built.rows.push(row); - built.written.push(stream); - } - Err(_) => built.unbuildable.push(stream), - } - } - } - ReplacedEvents::Failed => built.unbuildable.push(stream), - }, + Payload::Pending(sequence, pending) => pending_events.push((sequence, pending)), Payload::Sql(input) => match self.sql_rows(input) { Some(rows) => { built.rows.extend(rows); @@ -172,6 +156,28 @@ impl Materializer { }, } } + pending_events.sort_by_key(|(sequence, _)| *sequence); + for (_, pending) in pending_events { + let stream = DeliveryStream::Events; + match pending.replace() { + ReplacedEvents::Published(events) if events.is_empty() => { + built.filtered.push(stream); + } + ReplacedEvents::Published(events) => { + built.extra_events += events.len() as u64 - 1; + for event in events { + match event_object_row(&self.envelope, &event) { + Ok(row) => { + built.rows.push(row); + built.written.push(stream); + } + Err(_) => built.unbuildable.push(stream), + } + } + } + ReplacedEvents::Failed => built.unbuildable.push(stream), + } + } built } } diff --git a/crates/sqlbuild-recall/src/writer/_helpers/queue.rs b/crates/sqlbuild-recall/src/writer/_helpers/queue.rs index 837939338d..50016bdc7b 100644 --- a/crates/sqlbuild-recall/src/writer/_helpers/queue.rs +++ b/crates/sqlbuild-recall/src/writer/_helpers/queue.rs @@ -16,7 +16,7 @@ use crate::records::models::{ use crate::store::models::InsertOutcome; use crate::writer::_helpers::delivery::{record_stream, stream_of}; use crate::writer::_helpers::tail::{TailRing, TailSegment}; -use crate::writer::constants::{QUEUE_BYTE_CAPACITY, QUEUE_RECORD_CAPACITY}; +use crate::writer::constants::{PENDING_EVENT_BYTES, QUEUE_BYTE_CAPACITY, QUEUE_RECORD_CAPACITY}; use crate::writer::models::{DeliveryStream, Priority}; use crate::writer::types::PendingLifecycleEvent; @@ -34,8 +34,8 @@ pub(crate) struct SqlInput { pub(crate) enum Payload { Rows(Vec), Event(Map), - /// An event the lifecycle replacement sink decides on when its batch is built. - Pending(Box), + /// An event the lifecycle replacement sink decides on when its batch is built, by publication sequence. + Pending(u64, Box), Sql(SqlInput), } @@ -52,14 +52,14 @@ impl Queued { pub(crate) fn rows(&self) -> &[RecallRow] { match &self.payload { Payload::Rows(rows) => rows, - Payload::Event(_) | Payload::Pending(_) | Payload::Sql(_) => &[], + Payload::Event(_) | Payload::Pending(..) | Payload::Sql(_) => &[], } } fn row_count(&self) -> usize { match &self.payload { Payload::Rows(rows) => rows.len(), - Payload::Event(_) | Payload::Pending(_) => 1, + Payload::Event(_) | Payload::Pending(..) => 1, Payload::Sql(_) => 2, } } @@ -95,6 +95,8 @@ pub(crate) struct WriterState { tail_open: Option, unreported_output_drops: u64, pub(crate) diagnostic_sequence: u64, + /// The publication sequence of the next event for the lifecycle replacement. + pending_sequence: u64, pub(crate) closing: bool, pub(crate) deadline: Option, pub(crate) finished: bool, @@ -142,6 +144,7 @@ impl WriterState { tail_open: None, unreported_output_drops: 0, diagnostic_sequence: 0, + pending_sequence: 0, closing: false, deadline: None, finished: false, @@ -235,6 +238,24 @@ impl WriterState { self.buckets[priority as usize].push_back(record); } + /// Queue an event for the lifecycle replacement, numbered in publication order. + pub(crate) fn push_pending( + &mut self, + event: Box, + priority: Priority, + ) { + let sequence: u64 = self.pending_sequence; + self.pending_sequence += 1; + self.push_record( + Queued { + payload: Payload::Pending(sequence, event), + stream: DeliveryStream::Events, + bytes: PENDING_EVENT_BYTES, + }, + priority, + ); + } + pub(crate) fn queued(&self) -> usize { self.records } diff --git a/crates/sqlbuild-recall/src/writer/_helpers/session.rs b/crates/sqlbuild-recall/src/writer/_helpers/session.rs index bdf58bd515..955cd027fa 100644 --- a/crates/sqlbuild-recall/src/writer/_helpers/session.rs +++ b/crates/sqlbuild-recall/src/writer/_helpers/session.rs @@ -14,7 +14,7 @@ use crate::records::main::parse_event::parse_event; use crate::records::models::OutputStream; use crate::writer::_helpers::delivery::locked; use crate::writer::_helpers::queue::{Payload, Queued, SqlInput}; -use crate::writer::constants::{JSON_SCALAR_BYTES, MAX_ARTIFACT_BYTES, PENDING_EVENT_BYTES}; +use crate::writer::constants::{JSON_SCALAR_BYTES, MAX_ARTIFACT_BYTES}; use crate::writer::models::{ DeliveryStream, DeliverySummary, Priority, RecallSession, SetupOutcome, }; @@ -120,13 +120,8 @@ impl RecallSession { identity.insert("schema_version".to_owned(), Value::from(schema_version)); let priority: Priority = event_severity(&identity).map_or(Priority::Lifecycle, event_priority); - let record = Queued { - payload: Payload::Pending(event), - stream: DeliveryStream::Events, - bytes: PENDING_EVENT_BYTES, - }; let mut state = locked(&self.shared.state); - state.push_record(record, priority); + state.push_pending(event, priority); self.shared.wake.notify_one(); } diff --git a/crates/sqlbuild-recall/src/writer/tests/helpers.rs b/crates/sqlbuild-recall/src/writer/tests/helpers.rs index 97924d3932..07ebb6d109 100644 --- a/crates/sqlbuild-recall/src/writer/tests/helpers.rs +++ b/crates/sqlbuild-recall/src/writer/tests/helpers.rs @@ -17,7 +17,7 @@ use crate::writer::models::{ DeliveryStream, DeliverySummary, Priority, SessionConfig, SessionStore, SetupOutcome, }; use crate::writer::tests::test_types::{ - CopyingReplacement, PartialStore, PrefixReplacement, RecordedSession, + CopyingReplacement, PartialStore, PrefixReplacement, RecordedSession, RecordingReplacement, }; /// The envelope of invocation `invocation_id`. @@ -542,3 +542,23 @@ pub(crate) fn written_to_core_release_store(write: fn(&std::path::Path)) -> Vec< .expect("the query is valid") .rows } + +/// The order a lifecycle replacement sees events published with `priorities` and taken in one batch. +pub(crate) fn replacement_order(priorities: &[Priority]) -> Vec { + use crate::writer::_helpers::materialize::Materializer; + + let seen = Arc::new(std::sync::Mutex::new(Vec::new())); + let mut state = WriterState::new(envelope("ordered")); + for (index, priority) in priorities.iter().enumerate() { + state.push_pending( + Box::new(RecordingReplacement { + index, + seen: Arc::clone(&seen), + }), + *priority, + ); + } + let batch = state.take_batch(usize::MAX); + drop(Materializer::new(envelope("ordered")).build(batch, None)); + std::mem::take(&mut *seen.lock().expect("the record is not poisoned")) +} diff --git a/crates/sqlbuild-recall/src/writer/tests/test_session.rs b/crates/sqlbuild-recall/src/writer/tests/test_session.rs index a78f741b66..2438b84bc4 100644 --- a/crates/sqlbuild-recall/src/writer/tests/test_session.rs +++ b/crates/sqlbuild-recall/src/writer/tests/test_session.rs @@ -2,22 +2,23 @@ use std::time::Duration; use crate::records::models::{OutputStream, RecallValue}; use crate::schema::models::Table; -use crate::writer::models::SetupOutcome; +use crate::writer::models::{Priority, SetupOutcome}; #[cfg(unix)] use crate::writer::tests::helpers::unwritable_session; use crate::writer::tests::helpers::{ artifact_counts, close_under_held_lock, event_counts, insert_heartbeat, invocation_during_discovery, output_stored_while_running, partially_inserted, - pending_event_counts, record_run, recorded_session, replaced_session, session_on_corrupt_store, - written_to_core_release_store, + pending_event_counts, record_run, recorded_session, replaced_session, replacement_order, + session_on_corrupt_store, written_to_core_release_store, }; #[cfg(unix)] use crate::writer::tests::test_types::UnwritableTestCase; use crate::writer::tests::test_types::{ ArtifactSizeTestCase, CoreReleaseStoreTestCase, CorruptStoreTestCase, DiscoveryTestCase, EventBuildTestCase, HeldLockTestCase, PartialInsertTestCase, PendingEventTestCase, - RecordedSession, ReplacementTestCase, SessionTestCase, StoredWhileRunningTestCase, + RecordedSession, ReplacementOrderTestCase, ReplacementTestCase, SessionTestCase, + StoredWhileRunningTestCase, }; const CLOSE_SLACK_SECONDS: f64 = 0.3; @@ -353,3 +354,27 @@ fn given_a_store_from_the_first_release_when_written_then_rows_land_and_columns_ ); } } + +#[test] +fn given_events_of_mixed_priority_in_one_batch_when_replaced_then_the_sink_sees_publication_order() +{ + let test_cases = [ReplacementOrderTestCase { + description: "a failure and a terminal event do not overtake earlier lifecycle events", + priorities: vec![ + Priority::Lifecycle, + Priority::FailedTerminal, + Priority::Terminal, + Priority::Lifecycle, + ], + expected_order: vec![0, 1, 2, 3], + }]; + + for test_case in test_cases { + assert_eq!( + replacement_order(&test_case.priorities), + test_case.expected_order, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-recall/src/writer/tests/test_types.rs b/crates/sqlbuild-recall/src/writer/tests/test_types.rs index ac3229898a..079a4b2457 100644 --- a/crates/sqlbuild-recall/src/writer/tests/test_types.rs +++ b/crates/sqlbuild-recall/src/writer/tests/test_types.rs @@ -214,3 +214,26 @@ pub(crate) struct CoreReleaseStoreTestCase { pub(crate) write: fn(&std::path::Path), pub(crate) expected_rows: Vec>, } + +pub(crate) struct ReplacementOrderTestCase { + pub(crate) description: &'static str, + pub(crate) priorities: Vec, + pub(crate) expected_order: Vec, +} + +/// A lifecycle event that records when its replacement runs and keeps nothing. +#[derive(Debug)] +pub(crate) struct RecordingReplacement { + pub(crate) index: usize, + pub(crate) seen: std::sync::Arc>>, +} + +impl PendingLifecycleEvent for RecordingReplacement { + fn replace(self: Box) -> ReplacedEvents { + self.seen + .lock() + .expect("the record is not poisoned") + .push(self.index); + ReplacedEvents::Published(Vec::new()) + } +} From 7fe6294ce92bf3ae8b385585fbe856da0c5f57f2 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:24:13 +0100 Subject: [PATCH 17/19] test: keep ClickHouse Recall output past the local store's default limit --- .../commands/main/clickhouse/_test_types.py | 7 ++++ .../main/clickhouse/test_clickhouse_recall.py | 35 +++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py index 7312b6365a..fd6ed9b0ae 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/_test_types.py @@ -191,3 +191,10 @@ class ClickHouseRecallFirstReleaseStoreTestCase: expected_recorded: list[list[object]] expected_notice: str expected_heartbeat_columns: tuple[tuple[object, ...], ...] + + +@dataclass(frozen=True) +class ClickHouseRecallUnlimitedOutputTestCase: + description: str + thread_lines: str + expected_minimum_bytes: int diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py index 5d0e475ce5..af62819639 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py @@ -21,6 +21,7 @@ ClickHouseRecallQueryOutputSinkTestCase, ClickHouseRecallRetentionConflictTestCase, ClickHouseRecallRetentionTestCase, + ClickHouseRecallUnlimitedOutputTestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.clickhouse.helpers import ( FIRST_RELEASE_EDITS, @@ -693,5 +694,39 @@ def test_given_a_first_release_store_when_a_select_insert_user_builds_then_histo ) +@pytest.mark.parametrize( + "test_case", + [ + ClickHouseRecallUnlimitedOutputTestCase( + description="output past the local store's 2 MiB default is kept whole", + thread_lines="200000", + expected_minimum_bytes=5 * 512 * 1024, + ), + ], + ids=lambda case: case.description, +) +def test_given_large_output_when_recorded_in_clickhouse_then_no_output_limit_applies( + test_case: ClickHouseRecallUnlimitedOutputTestCase, + tmp_path: Path, + clickhouse_e2e_config: dict[str, object], + recall_database: str, +) -> None: + project_dir: Path = write_recall_project( + parent=tmp_path, config=clickhouse_e2e_config, database=recall_database + ) + built: subprocess.CompletedProcess[bytes] = run( + project_dir, "build", ORDERS_TASK_THREAD_LINES=test_case.thread_lines + ) + + recalled: subprocess.CompletedProcess[bytes] = run( + project_dir, "recall", "output", "last", "--stream", "stdout" + ) + + assert built.returncode == 0, built.stderr + assert len(built.stdout) >= test_case.expected_minimum_bytes + assert recalled.stdout == built.stdout + assert recalled.stderr == b"" + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) From 1a6cffb7cd7dde712af82d89342b7ccbff7366df Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:24:13 +0100 Subject: [PATCH 18/19] docs: regenerate the bundled docs skill for ClickHouse Recall --- .../docs/concepts/observability/recall.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md index f995b1aac5..2b92ac0dcb 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md @@ -142,7 +142,8 @@ retention = "30d" - **Setup.** On first use Recall creates its `recall_*` MergeTree tables and the views above in that database, and adds columns that a newer SQLBuild introduces. After that, writing needs only `INSERT` and `SELECT` on the tables. If the user may not create them, the command still succeeds - and prints the complete DDL for an administrator to run. + and prints the complete DDL for an administrator to run; a store created from it accepts writers + limited to `SELECT` and `INSERT`. - **Writes.** History is sent in batches of up to 5,000 rows about once a second, one insert per table, with the same queue, shutdown budget and exit-code guarantees as the local store. Many concurrent commands can write one store. @@ -151,7 +152,7 @@ retention = "30d" is counted once. Before writing, Recall looks up the batch's keys: a key already stored with different content keeps the stored row, and the rejected copy is listed in `conflicts`. - **Retention.** Every table carries a TTL of `retention` (SQL text a week longer, and refreshed - while it is still referenced), and the views hide rows past `retention` at once. ClickHouse drops + while it is still referenced), and the views hide rows past their TTL at once. ClickHouse drops expired daily parts in the background, so rows may stay on disk for a while after the views stop showing them. Retention belongs to the store: when a project's `retention` differs from the tables' TTL, Recall warns once a day with the `ALTER TABLE … MODIFY TTL` and view statements that @@ -182,9 +183,12 @@ def recall_output(record: CommandOutputRecord, writer: RecallWriter) -> None: keeps the record ID and sequence; publish at most one record per call. - A `@lifecycle_event_sink` replacement receives each event its `event_kinds` and `min_severity` accept and keeps it with `writer.publish_lifecycle(event)`. -- A record the sink does not publish is counted as `filtered`; a sink that raises has its record - counted as `failed`, and Recall never falls back to storing the unfiltered record. The command's - exit code and output are unaffected. `output_invocations.filter_active` is `1` for invocations +- Both replacements run on Recall's background thread, so a slow one never delays the command or + other sinks by more than `shutdown_timeout`; what it has not handled by then is counted as + `dropped`. Output written before discovery finishes is held until the replacement can see it. +- A record the sink does not publish is counted as `filtered`; a sink that raises an exception has + its record counted as `failed`, and Recall never falls back to storing the unfiltered record. The + command's exit code and output are unaffected. `output_invocations.filter_active` is `1` for invocations that ran a replacement. - There is at most one replacement per stream; discovery fails with `D015` naming each `path:function` otherwise. A replacement takes only its record and the writer, not providers, and From 4796ff19cfce615cc36bbd81b92e238b0285253f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 19:33:20 +0100 Subject: [PATCH 19/19] test: approve the ClickHouse Recall reader as a raw statement boundary --- src/sqlbuild/adapter/contract/_helpers/raw_statement_audit.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlbuild/adapter/contract/_helpers/raw_statement_audit.py b/src/sqlbuild/adapter/contract/_helpers/raw_statement_audit.py index 9e4f896e16..bb79d702e1 100644 --- a/src/sqlbuild/adapter/contract/_helpers/raw_statement_audit.py +++ b/src/sqlbuild/adapter/contract/_helpers/raw_statement_audit.py @@ -24,6 +24,7 @@ "adapters/bigquery/classes/bigquery_connection.py": frozenset( {("_BigQueryConnection.query_job", "query")} ), + "adapters/clickhouse/_helpers/recall_store.py": frozenset({("run_recall_query", "query")}), "adapters/clickhouse/classes/clickhouse_connection.py": frozenset( {("_ClickHouseConnection._run", "query")} ),