From 7e84ae435809ecab90edae621aa429599503f5c7 Mon Sep 17 00:00:00 2001 From: Shubham Mishra Date: Fri, 27 Mar 2026 10:57:09 +0530 Subject: [PATCH 01/23] st_databases_tx_offset --- .../locking_tx_datastore/committed_state.rs | 15 +++---- crates/datastore/src/system_tables.rs | 42 ++++++++++++++++++- 2 files changed, 48 insertions(+), 9 deletions(-) diff --git a/crates/datastore/src/locking_tx_datastore/committed_state.rs b/crates/datastore/src/locking_tx_datastore/committed_state.rs index d5cc2099d8a..6c9fbde3f1e 100644 --- a/crates/datastore/src/locking_tx_datastore/committed_state.rs +++ b/crates/datastore/src/locking_tx_datastore/committed_state.rs @@ -19,10 +19,10 @@ use crate::{ is_built_in_meta_row, system_tables, table_id_is_reserved, StColumnRow, StConstraintData, StConstraintRow, StFields, StIndexRow, StSequenceFields, StSequenceRow, StTableFields, StTableRow, StViewRow, SystemTable, ST_CLIENT_ID, ST_CLIENT_IDX, ST_COLUMN_ID, ST_COLUMN_IDX, ST_COLUMN_NAME, ST_CONSTRAINT_ID, ST_CONSTRAINT_IDX, - ST_CONSTRAINT_NAME, ST_INDEX_ID, ST_INDEX_IDX, ST_INDEX_NAME, ST_MODULE_ID, ST_MODULE_IDX, - ST_ROW_LEVEL_SECURITY_ID, ST_ROW_LEVEL_SECURITY_IDX, ST_SCHEDULED_ID, ST_SCHEDULED_IDX, ST_SEQUENCE_ID, - ST_SEQUENCE_IDX, ST_SEQUENCE_NAME, ST_TABLE_ID, ST_TABLE_IDX, ST_VAR_ID, ST_VAR_IDX, ST_VIEW_ARG_ID, - ST_VIEW_ARG_IDX, + ST_CONSTRAINT_NAME, ST_DATABASES_TX_OFFSET_ID, ST_INDEX_ID, ST_INDEX_IDX, ST_INDEX_NAME, ST_MODULE_ID, + ST_MODULE_IDX, ST_ROW_LEVEL_SECURITY_ID, ST_ROW_LEVEL_SECURITY_IDX, ST_SCHEDULED_ID, ST_SCHEDULED_IDX, + ST_SEQUENCE_ID, ST_SEQUENCE_IDX, ST_SEQUENCE_NAME, ST_TABLE_ID, ST_TABLE_IDX, ST_VAR_ID, ST_VAR_IDX, + ST_VIEW_ARG_ID, ST_VIEW_ARG_IDX, }, traits::{EphemeralTables, TxData}, }; @@ -30,9 +30,9 @@ use crate::{ locking_tx_datastore::ViewCallInfo, system_tables::{ ST_COLUMN_ACCESSOR_ID, ST_COLUMN_ACCESSOR_IDX, ST_CONNECTION_CREDENTIALS_ID, ST_CONNECTION_CREDENTIALS_IDX, - ST_EVENT_TABLE_ID, ST_EVENT_TABLE_IDX, ST_INDEX_ACCESSOR_ID, ST_INDEX_ACCESSOR_IDX, ST_TABLE_ACCESSOR_ID, - ST_TABLE_ACCESSOR_IDX, ST_VIEW_COLUMN_ID, ST_VIEW_COLUMN_IDX, ST_VIEW_ID, ST_VIEW_IDX, ST_VIEW_PARAM_ID, - ST_VIEW_PARAM_IDX, ST_VIEW_SUB_ID, ST_VIEW_SUB_IDX, + ST_DATABASES_TX_OFFSET_IDX, ST_EVENT_TABLE_ID, ST_EVENT_TABLE_IDX, ST_INDEX_ACCESSOR_ID, ST_INDEX_ACCESSOR_IDX, + ST_TABLE_ACCESSOR_ID, ST_TABLE_ACCESSOR_IDX, ST_VIEW_COLUMN_ID, ST_VIEW_COLUMN_IDX, ST_VIEW_ID, ST_VIEW_IDX, + ST_VIEW_PARAM_ID, ST_VIEW_PARAM_IDX, ST_VIEW_SUB_ID, ST_VIEW_SUB_IDX, }, }; use anyhow::anyhow; @@ -477,6 +477,7 @@ impl CommittedState { self.create_table(ST_TABLE_ACCESSOR_ID, schemas[ST_TABLE_ACCESSOR_IDX].clone()); self.create_table(ST_INDEX_ACCESSOR_ID, schemas[ST_INDEX_ACCESSOR_IDX].clone()); self.create_table(ST_COLUMN_ACCESSOR_ID, schemas[ST_COLUMN_ACCESSOR_IDX].clone()); + self.create_table(ST_DATABASES_TX_OFFSET_ID, schemas[ST_DATABASES_TX_OFFSET_IDX].clone()); // Insert the sequences into `st_sequences` let (st_sequences, blob_store, pool) = diff --git a/crates/datastore/src/system_tables.rs b/crates/datastore/src/system_tables.rs index e75cc76b365..334d71f567a 100644 --- a/crates/datastore/src/system_tables.rs +++ b/crates/datastore/src/system_tables.rs @@ -89,6 +89,9 @@ pub const ST_INDEX_ACCESSOR_ID: TableId = TableId(19); /// The static ID of the table that maps canonical column names to accessor names pub const ST_COLUMN_ACCESSOR_ID: TableId = TableId(20); +/// The static ID of the table that tracks the last tx offset send by other databases. +pub const ST_DATABASES_TX_OFFSET_ID: TableId = TableId(21); + pub(crate) const ST_CONNECTION_CREDENTIALS_NAME: &str = "st_connection_credentials"; pub const ST_TABLE_NAME: &str = "st_table"; pub const ST_COLUMN_NAME: &str = "st_column"; @@ -109,6 +112,7 @@ pub(crate) const ST_EVENT_TABLE_NAME: &str = "st_event_table"; pub(crate) const ST_TABLE_ACCESSOR_NAME: &str = "st_table_accessor"; pub(crate) const ST_INDEX_ACCESSOR_NAME: &str = "st_index_accessor"; pub(crate) const ST_COLUMN_ACCESSOR_NAME: &str = "st_column_accessor"; +pub(crate) const ST_DATABASES_TX_OFFSET_NAME: &str = "st_databases_tx_offset"; /// Reserved range of sequence values used for system tables. /// /// Ids for user-created tables will start at `ST_RESERVED_SEQUENCE_RANGE`. @@ -182,7 +186,7 @@ pub fn is_built_in_meta_row(table_id: TableId, row: &ProductValue) -> Result false, + ST_TABLE_ACCESSOR_ID | ST_INDEX_ACCESSOR_ID | ST_COLUMN_ACCESSOR_ID | ST_DATABASES_TX_OFFSET_ID => false, TableId(..ST_RESERVED_SEQUENCE_RANGE) => { log::warn!("Unknown system table {table_id:?}"); false @@ -205,7 +209,7 @@ pub enum SystemTable { st_table_accessor, } -pub fn system_tables() -> [TableSchema; 20] { +pub fn system_tables() -> [TableSchema; 21] { [ // The order should match the `id` of the system table, that start with [ST_TABLE_IDX]. st_table_schema(), @@ -228,6 +232,7 @@ pub fn system_tables() -> [TableSchema; 20] { st_table_accessor_schema(), st_index_accessor_schema(), st_column_accessor_schema(), + st_databases_tx_offset_schema(), ] } @@ -276,6 +281,7 @@ pub(crate) const ST_EVENT_TABLE_IDX: usize = 16; pub(crate) const ST_TABLE_ACCESSOR_IDX: usize = 17; pub(crate) const ST_INDEX_ACCESSOR_IDX: usize = 18; pub(crate) const ST_COLUMN_ACCESSOR_IDX: usize = 19; +pub(crate) const ST_DATABASES_TX_OFFSET_IDX: usize = 20; macro_rules! st_fields_enum { ($(#[$attr:meta])* enum $ty_name:ident { $($name:expr, $var:ident = $discr:expr,)* }) => { @@ -450,6 +456,11 @@ st_fields_enum!(enum StColumnAccessorFields { "accessor_name", AccessorName = 2, }); +st_fields_enum!(enum StDatabasesTxOffsetFields { + "database_identity", DatabaseIdentity = 0, + "tx_offset", TxOffset = 1, +}); + /// Helper method to check that a system table has the correct fields. /// Does not check field types since those aren't included in `StFields` types. /// If anything in here is not true, the system is completely broken, so it's fine to assert. @@ -668,6 +679,17 @@ fn system_module_def() -> ModuleDef { .with_unique_constraint(st_column_accessor_table_alias_cols) .with_index_no_accessor_name(btree(st_column_accessor_table_alias_cols)); + let st_databases_tx_offset_type = builder.add_type::(); + builder + .build_table( + ST_DATABASES_TX_OFFSET_NAME, + *st_databases_tx_offset_type.as_ref().expect("should be ref"), + ) + .with_type(TableType::System) + .with_unique_constraint(StDatabasesTxOffsetFields::DatabaseIdentity) + .with_index_no_accessor_name(btree(StDatabasesTxOffsetFields::DatabaseIdentity)) + .with_primary_key(StDatabasesTxOffsetFields::DatabaseIdentity); + let result = builder .finish() .try_into() @@ -693,6 +715,7 @@ fn system_module_def() -> ModuleDef { validate_system_table::(&result, ST_TABLE_ACCESSOR_NAME); validate_system_table::(&result, ST_INDEX_ACCESSOR_NAME); validate_system_table::(&result, ST_COLUMN_ACCESSOR_NAME); + validate_system_table::(&result, ST_DATABASES_TX_OFFSET_NAME); result } @@ -741,6 +764,7 @@ lazy_static::lazy_static! { m.insert("st_index_accessor_accessor_name_key", ConstraintId(23)); m.insert("st_column_accessor_table_name_col_name_key", ConstraintId(24)); m.insert("st_column_accessor_table_name_accessor_name_key", ConstraintId(25)); + m.insert("st_databases_tx_offset_database_identity_key", ConstraintId(26)); m }; } @@ -779,6 +803,7 @@ lazy_static::lazy_static! { m.insert("st_index_accessor_accessor_name_idx_btree", IndexId(27)); m.insert("st_column_accessor_table_name_col_name_idx_btree", IndexId(28)); m.insert("st_column_accessor_table_name_accessor_name_idx_btree", IndexId(29)); + m.insert("st_databases_tx_offset_database_identity_idx_btree", IndexId(30)); m }; } @@ -940,6 +965,10 @@ fn st_column_accessor_schema() -> TableSchema { st_schema(ST_COLUMN_ACCESSOR_NAME, ST_COLUMN_ACCESSOR_ID) } +fn st_databases_tx_offset_schema() -> TableSchema { + st_schema(ST_DATABASES_TX_OFFSET_NAME, ST_DATABASES_TX_OFFSET_ID) +} + /// If `table_id` refers to a known system table, return its schema. /// /// Used when restoring from a snapshot; system tables are reinstantiated with this schema, @@ -968,6 +997,7 @@ pub(crate) fn system_table_schema(table_id: TableId) -> Option { ST_TABLE_ACCESSOR_ID => Some(st_table_accessor_schema()), ST_INDEX_ACCESSOR_ID => Some(st_index_accessor_schema()), ST_COLUMN_ACCESSOR_ID => Some(st_column_accessor_schema()), + ST_DATABASES_TX_OFFSET_ID => Some(st_databases_tx_offset_schema()), _ => None, } } @@ -1859,6 +1889,14 @@ impl From for ProductValue { } } +/// System Table [ST_DATABASES_TX_OFFST_NAME] +#[derive(Debug, Clone, PartialEq, Eq, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +pub struct StDatabasesTxOffsetRow { + pub database_identity: IdentityViaU256, + pub tx_offset: u64, +} + thread_local! { static READ_BUF: RefCell> = const { RefCell::new(Vec::new()) }; } From 9d4f21e5c086bc1e84aaa50e78767b879c64ca0d Mon Sep 17 00:00:00 2001 From: Shubham Mishra Date: Fri, 27 Mar 2026 12:40:22 +0530 Subject: [PATCH 02/23] call_from_database endpoint --- crates/client-api/src/routes/database.rs | 117 +++++++++++++++++- crates/core/src/host/host_controller.rs | 7 +- crates/core/src/host/module_host.rs | 92 +++++++++++++- crates/core/src/host/v8/mod.rs | 2 +- .../src/host/wasm_common/module_host_actor.rs | 33 ++++- .../src/locking_tx_datastore/mut_tx.rs | 47 ++++++- crates/datastore/src/system_tables.rs | 7 ++ 7 files changed, 292 insertions(+), 13 deletions(-) diff --git a/crates/client-api/src/routes/database.rs b/crates/client-api/src/routes/database.rs index 6b753a9c8fd..3b9d49ca994 100644 --- a/crates/client-api/src/routes/database.rs +++ b/crates/client-api/src/routes/database.rs @@ -216,6 +216,117 @@ pub async fn call( } } +/// Path parameters for the `call_from_database` route. +#[derive(Deserialize)] +pub struct CallFromDatabaseParams { + name_or_identity: NameOrIdentity, + reducer: String, +} + +/// Query parameters for the `call_from_database` route. +/// +/// Both fields are mandatory; a missing field results in a 400 Bad Request. +#[derive(Deserialize)] +pub struct CallFromDatabaseQuery { + /// Hex-encoded [`Identity`] of the sending database. + sender_identity: String, + /// The commitlog offset of the message on the sender side. + /// Used for at-most-once delivery via `st_databases_tx_offset`. + tx_offset: u64, +} + +/// Call a reducer on behalf of another database, with deduplication. +/// +/// Endpoint: `POST /database/:name_or_identity/call-from-database/:reducer` +/// +/// Required query params: +/// - `sender_identity` — hex-encoded identity of the sending database. +/// - `tx_offset` — the sender's commitlog offset for this message. +/// +/// Before invoking the reducer, the receiver checks `st_databases_tx_offset`. +/// If the incoming `tx_offset` is ≤ the last delivered offset for `sender_identity`, +/// the call is a duplicate and 200 OK is returned immediately without running the reducer. +/// Otherwise the reducer is invoked, the dedup index is updated atomically in the same +/// transaction, and an acknowledgment is returned on success. +pub async fn call_from_database( + State(worker_ctx): State, + Extension(auth): Extension, + Path(CallFromDatabaseParams { + name_or_identity, + reducer, + }): Path, + Query(CallFromDatabaseQuery { + sender_identity, + tx_offset, + }): Query, + TypedHeader(content_type): TypedHeader, + ByteStringBody(body): ByteStringBody, +) -> axum::response::Result { + assert_content_type_json(content_type)?; + + let caller_identity = auth.claims.identity; + + let sender_identity = Identity::from_hex(&sender_identity) + .map_err(|_| (StatusCode::BAD_REQUEST, "Invalid sender_identity: expected hex-encoded identity"))?; + + let args = FunctionArgs::Json(body); + let connection_id = generate_random_connection_id(); + + let (module, Database { owner_identity, .. }) = find_module_and_database(&worker_ctx, name_or_identity).await?; + + // Call client_connected, if defined. + module + .call_identity_connected(auth.into(), connection_id) + .await + .map_err(client_connected_error_to_response)?; + + let result = module + .call_reducer_from_database( + caller_identity, + Some(connection_id), + None, + None, + None, + &reducer, + args, + sender_identity, + tx_offset, + ) + .await; + + module + .call_identity_disconnected(caller_identity, connection_id) + .await + .map_err(client_disconnected_error_to_response)?; + + match result { + Ok(rcr) => { + let (status, body) = match rcr.outcome { + ReducerOutcome::Committed => (StatusCode::OK, "".into()), + ReducerOutcome::Deduplicated => (StatusCode::OK, "deduplicated".into()), + ReducerOutcome::Failed(errmsg) => (StatusCode::from_u16(530).unwrap(), *errmsg), + ReducerOutcome::BudgetExceeded => { + log::warn!( + "Node's energy budget exceeded for identity: {owner_identity} while executing {reducer}" + ); + (StatusCode::PAYMENT_REQUIRED, "Module energy budget exhausted.".into()) + } + }; + Ok(( + status, + TypedHeader(SpacetimeEnergyUsed(rcr.energy_used)), + TypedHeader(SpacetimeExecutionDurationMicros(rcr.execution_duration)), + body, + ) + .into_response()) + } + Err(e) => { + let (status, msg) = map_reducer_error(e, &reducer); + Err((status, msg).into()) + } + } +} + fn assert_content_type_json(content_type: headers::ContentType) -> axum::response::Result<()> { if content_type != headers::ContentType::json() { Err(axum::extract::rejection::MissingJsonContentType::default().into()) @@ -230,7 +341,7 @@ fn reducer_outcome_response( outcome: ReducerOutcome, ) -> (StatusCode, Box) { match outcome { - ReducerOutcome::Committed => (StatusCode::OK, "".into()), + ReducerOutcome::Committed | ReducerOutcome::Deduplicated => (StatusCode::OK, "".into()), ReducerOutcome::Failed(errmsg) => { // TODO: different status code? this is what cloudflare uses, sorta (StatusCode::from_u16(530).unwrap(), *errmsg) @@ -1189,6 +1300,8 @@ pub struct DatabaseRoutes { pub subscribe_get: MethodRouter, /// POST: /database/:name_or_identity/call/:reducer pub call_reducer_procedure_post: MethodRouter, + /// POST: /database/:name_or_identity/call-from-database/:reducer?sender_identity=&tx_offset= + pub call_from_database_post: MethodRouter, /// GET: /database/:name_or_identity/schema pub schema_get: MethodRouter, /// GET: /database/:name_or_identity/logs @@ -1220,6 +1333,7 @@ where identity_get: get(get_identity::), subscribe_get: get(handle_websocket::), call_reducer_procedure_post: post(call::), + call_from_database_post: post(call_from_database::), schema_get: get(schema::), logs_get: get(logs::), sql_post: post(sql::), @@ -1245,6 +1359,7 @@ where .route("/identity", self.identity_get) .route("/subscribe", self.subscribe_get) .route("/call/:reducer", self.call_reducer_procedure_post) + .route("/call-from-database/:reducer", self.call_from_database_post) .route("/schema", self.schema_get) .route("/logs", self.logs_get) .route("/sql", self.sql_post) diff --git a/crates/core/src/host/host_controller.rs b/crates/core/src/host/host_controller.rs index e67e67540eb..63b6d20de6a 100644 --- a/crates/core/src/host/host_controller.rs +++ b/crates/core/src/host/host_controller.rs @@ -160,19 +160,22 @@ pub enum ReducerOutcome { Committed, Failed(Box>), BudgetExceeded, + /// The call was identified as a duplicate via the `st_databases_tx_offset` dedup index + /// and was discarded without running the reducer. + Deduplicated, } impl ReducerOutcome { pub fn into_result(self) -> anyhow::Result<()> { match self { - Self::Committed => Ok(()), + Self::Committed | Self::Deduplicated => Ok(()), Self::Failed(e) => Err(anyhow::anyhow!(e)), Self::BudgetExceeded => Err(anyhow::anyhow!("reducer ran out of energy")), } } pub fn is_err(&self) -> bool { - !matches!(self, Self::Committed) + !matches!(self, Self::Committed | Self::Deduplicated) } } diff --git a/crates/core/src/host/module_host.rs b/crates/core/src/host/module_host.rs index 6ed0f0ec97c..ebbd7001bbd 100644 --- a/crates/core/src/host/module_host.rs +++ b/crates/core/src/host/module_host.rs @@ -589,7 +589,7 @@ pub fn call_identity_connected( // then insert into `st_client`, // but if we crashed in between, we'd be left in an inconsistent state // where the reducer had run but `st_client` was not yet updated. - ReducerOutcome::Committed => Ok(()), + ReducerOutcome::Committed | ReducerOutcome::Deduplicated => Ok(()), // If the reducer returned an error or couldn't run due to insufficient energy, // abort the connection: the module code has decided it doesn't want this client. @@ -624,6 +624,15 @@ pub struct CallReducerParams { pub timer: Option, pub reducer_id: ReducerId, pub args: ArgsTuple, + /// If set, enables at-most-once delivery semantics for database-to-database calls. + /// + /// The tuple is `(sender_database_identity, sender_tx_offset)`. + /// Before running the reducer, `st_databases_tx_offset` is consulted: + /// if `sender_tx_offset` ≤ the stored last-delivered offset for the sender, + /// the call is a duplicate and returns [`ReducerCallError::Deduplicated`]. + /// Otherwise the reducer runs, and the stored offset is updated atomically + /// within the same transaction. + pub dedup_sender: Option<(Identity, u64)>, } impl CallReducerParams { @@ -644,6 +653,7 @@ impl CallReducerParams { timer: None, reducer_id, args, + dedup_sender: None, } } } @@ -1490,9 +1500,9 @@ impl ModuleHost { .. }) => fallback(), - // If it succeeded, as mentioned above, `st_client` is already updated. + // If it succeeded (or was deduplicated), as mentioned above, `st_client` is already updated. Ok(ReducerCallResult { - outcome: ReducerOutcome::Committed, + outcome: ReducerOutcome::Committed | ReducerOutcome::Deduplicated, .. }) => Ok(()), } @@ -1567,6 +1577,7 @@ impl ModuleHost { timer, reducer_id, args, + dedup_sender: None, }) } @@ -1594,6 +1605,7 @@ impl ModuleHost { timer, reducer_id, args, + dedup_sender: None, }; self.call( @@ -1655,6 +1667,80 @@ impl ModuleHost { res } + /// Variant of [`Self::call_reducer`] for database-to-database calls. + /// + /// Behaves identically to `call_reducer`, except that it enforces at-most-once + /// delivery using the `st_databases_tx_offset` dedup index. + /// Before invoking the reducer, the receiver checks whether + /// `sender_tx_offset` ≤ the last delivered offset for `sender_database_identity`. + /// If so, the call is a duplicate and [`ReducerOutcome::Deduplicated`] is returned + /// without running the reducer. + /// Otherwise the reducer runs, and the dedup index is updated atomically + /// within the same transaction. + pub async fn call_reducer_from_database( + &self, + caller_identity: Identity, + caller_connection_id: Option, + client: Option>, + request_id: Option, + timer: Option, + reducer_name: &str, + args: FunctionArgs, + sender_database_identity: Identity, + sender_tx_offset: u64, + ) -> Result { + let res = async { + let (reducer_id, reducer_def) = self + .info + .module_def + .reducer_full(reducer_name) + .ok_or(ReducerCallError::NoSuchReducer)?; + if let Some(lifecycle) = reducer_def.lifecycle { + return Err(ReducerCallError::LifecycleReducer(lifecycle)); + } + + if reducer_def.visibility.is_private() && !self.is_database_owner(caller_identity) { + return Err(ReducerCallError::NoSuchReducer); + } + + let args = args + .into_tuple_for_def(&self.info.module_def, reducer_def) + .map_err(InvalidReducerArguments)?; + let caller_connection_id = caller_connection_id.unwrap_or(ConnectionId::ZERO); + let call_reducer_params = CallReducerParams { + timestamp: Timestamp::now(), + caller_identity, + caller_connection_id, + client, + request_id, + timer, + reducer_id, + args, + dedup_sender: Some((sender_database_identity, sender_tx_offset)), + }; + + self.call( + &reducer_def.name, + call_reducer_params, + async |p, inst| Ok(inst.call_reducer(p)), + async |p, inst| inst.call_reducer(p).await, + ) + .await? + } + .await; + + let log_message = match &res { + Err(ReducerCallError::NoSuchReducer) => Some(no_such_function_log_message("reducer", reducer_name)), + Err(ReducerCallError::Args(_)) => Some(args_error_log_message("reducer", reducer_name)), + _ => None, + }; + if let Some(log_message) = log_message { + self.inject_logs(LogLevel::Error, reducer_name, &log_message) + } + + res + } + pub async fn call_view_add_single_subscription( &self, sender: Arc, diff --git a/crates/core/src/host/v8/mod.rs b/crates/core/src/host/v8/mod.rs index 332c9c89dd3..9ab92263961 100644 --- a/crates/core/src/host/v8/mod.rs +++ b/crates/core/src/host/v8/mod.rs @@ -613,7 +613,7 @@ enum JsWorkerRequest { }, } -static_assert_size!(CallReducerParams, 192); +static_assert_size!(CallReducerParams, 256); fn send_worker_reply(ctx: &str, reply_tx: JsReplyTx, value: T, trapped: bool) { if reply_tx.send(JsWorkerReply { value, trapped }).is_err() { diff --git a/crates/core/src/host/wasm_common/module_host_actor.rs b/crates/core/src/host/wasm_common/module_host_actor.rs index a711e0f18dc..338e1c58680 100644 --- a/crates/core/src/host/wasm_common/module_host_actor.rs +++ b/crates/core/src/host/wasm_common/module_host_actor.rs @@ -2,7 +2,7 @@ use super::instrumentation::CallTimes; use super::*; use crate::client::ClientActorId; use crate::database_logger; -use crate::energy::{EnergyMonitor, FunctionBudget, FunctionFingerprint}; +use crate::energy::{EnergyMonitor, EnergyQuanta, FunctionBudget, FunctionFingerprint}; use crate::error::DBError; use crate::host::host_controller::CallProcedureReturn; use crate::host::instance_env::{InstanceEnv, TxSlot}; @@ -820,6 +820,7 @@ impl InstanceCommon { reducer_id, args, timer, + dedup_sender, } = params; let caller_connection_id_opt = (caller_connection_id != ConnectionId::ZERO).then_some(caller_connection_id); @@ -844,6 +845,25 @@ impl InstanceCommon { let workload = Workload::Reducer(ReducerContext::from(op.clone())); let tx = tx.unwrap_or_else(|| stdb.begin_mut_tx(IsolationLevel::Serializable, workload)); + + // Check the dedup index atomically within this transaction. + // If the incoming offset is ≤ the last delivered offset for this sender, + // the message is a duplicate; discard it and roll back. + if let Some((sender_identity, sender_tx_offset)) = dedup_sender { + let last_delivered = tx.get_databases_tx_offset(sender_identity); + if last_delivered.is_some_and(|last| sender_tx_offset <= last) { + let _ = stdb.rollback_mut_tx(tx); + return ( + ReducerCallResult { + outcome: ReducerOutcome::Deduplicated, + energy_used: EnergyQuanta::ZERO, + execution_duration: Duration::ZERO, + }, + false, + ); + } + } + let mut tx_slot = inst.tx_slot(); let vm_metrics = self.vm_metrics.get_for_reducer_id(reducer_id); @@ -885,12 +905,21 @@ impl InstanceCommon { Ok(return_value) => { // If this is an OnDisconnect lifecycle event, remove the client from st_clients. // We handle OnConnect events before running the reducer. - let res = match reducer_def.lifecycle { + let lifecycle_res = match reducer_def.lifecycle { Some(Lifecycle::OnDisconnect) => { tx.delete_st_client(caller_identity, caller_connection_id, info.database_identity) } _ => Ok(()), }; + // Atomically update the dedup index so this sender's tx_offset is recorded + // as delivered. This prevents re-processing if the message is retried. + let dedup_res = match dedup_sender { + Some((sender_identity, sender_tx_offset)) => { + tx.upsert_databases_tx_offset(sender_identity, sender_tx_offset) + } + None => Ok(()), + }; + let res = lifecycle_res.and(dedup_res); match res { Ok(()) => (EventStatus::Committed(DatabaseUpdate::default()), return_value), Err(err) => { diff --git a/crates/datastore/src/locking_tx_datastore/mut_tx.rs b/crates/datastore/src/locking_tx_datastore/mut_tx.rs index 7fd8bdb575d..bc531679346 100644 --- a/crates/datastore/src/locking_tx_datastore/mut_tx.rs +++ b/crates/datastore/src/locking_tx_datastore/mut_tx.rs @@ -21,10 +21,11 @@ use crate::{ error::{IndexError, SequenceError, TableError}, system_tables::{ with_sys_table_buf, StClientFields, StClientRow, StColumnAccessorFields, StColumnAccessorRow, StColumnFields, - StColumnRow, StConstraintFields, StConstraintRow, StEventTableRow, StFields as _, StIndexAccessorFields, - StIndexAccessorRow, StIndexFields, StIndexRow, StRowLevelSecurityFields, StRowLevelSecurityRow, - StScheduledFields, StScheduledRow, StSequenceFields, StSequenceRow, StTableAccessorFields, StTableAccessorRow, - StTableFields, StTableRow, SystemTable, ST_CLIENT_ID, ST_COLUMN_ACCESSOR_ID, ST_COLUMN_ID, ST_CONSTRAINT_ID, + StColumnRow, StConstraintFields, StConstraintRow, StDatabasesTxOffsetFields, StDatabasesTxOffsetRow, + StEventTableRow, StFields as _, StIndexAccessorFields, StIndexAccessorRow, StIndexFields, StIndexRow, + StRowLevelSecurityFields, StRowLevelSecurityRow, StScheduledFields, StScheduledRow, StSequenceFields, + StSequenceRow, StTableAccessorFields, StTableAccessorRow, StTableFields, StTableRow, SystemTable, + ST_CLIENT_ID, ST_COLUMN_ACCESSOR_ID, ST_COLUMN_ID, ST_CONSTRAINT_ID, ST_DATABASES_TX_OFFSET_ID, ST_EVENT_TABLE_ID, ST_INDEX_ACCESSOR_ID, ST_INDEX_ID, ST_ROW_LEVEL_SECURITY_ID, ST_SCHEDULED_ID, ST_SEQUENCE_ID, ST_TABLE_ACCESSOR_ID, ST_TABLE_ID, }, @@ -2690,6 +2691,44 @@ impl MutTxId { .map(|row| row.pointer()) } + /// Look up the last delivered tx offset for `sender_identity` in `st_databases_tx_offset`. + /// + /// Returns `None` if no entry exists for this sender (i.e., no message has been delivered yet). + pub fn get_databases_tx_offset(&self, sender_identity: Identity) -> Option { + self.iter_by_col_eq( + ST_DATABASES_TX_OFFSET_ID, + StDatabasesTxOffsetFields::DatabaseIdentity.col_id(), + &IdentityViaU256::from(sender_identity).into(), + ) + .expect("failed to read from st_databases_tx_offset system table") + .next() + .and_then(|row_ref| StDatabasesTxOffsetRow::try_from(row_ref).ok()) + .map(|row| row.tx_offset) + } + + /// Update the last delivered tx offset for `sender_identity` in `st_databases_tx_offset`. + /// + /// If an entry already exists, it is replaced; otherwise a new entry is inserted. + pub fn upsert_databases_tx_offset(&mut self, sender_identity: Identity, tx_offset: u64) -> Result<()> { + // Delete the existing row if present. + self.delete_col_eq( + ST_DATABASES_TX_OFFSET_ID, + StDatabasesTxOffsetFields::DatabaseIdentity.col_id(), + &IdentityViaU256::from(sender_identity).into(), + )?; + let row = StDatabasesTxOffsetRow { + database_identity: sender_identity.into(), + tx_offset, + }; + self.insert_via_serialize_bsatn(ST_DATABASES_TX_OFFSET_ID, &row) + .map(|_| ()) + .inspect_err(|e| { + log::error!( + "upsert_databases_tx_offset: failed to upsert tx offset for {sender_identity} to {tx_offset}: {e}" + ); + }) + } + pub fn insert_via_serialize_bsatn<'a, T: Serialize>( &'a mut self, table_id: TableId, diff --git a/crates/datastore/src/system_tables.rs b/crates/datastore/src/system_tables.rs index 334d71f567a..db5045030f2 100644 --- a/crates/datastore/src/system_tables.rs +++ b/crates/datastore/src/system_tables.rs @@ -1897,6 +1897,13 @@ pub struct StDatabasesTxOffsetRow { pub tx_offset: u64, } +impl TryFrom> for StDatabasesTxOffsetRow { + type Error = DatastoreError; + fn try_from(row: RowRef<'_>) -> Result { + read_via_bsatn(row) + } +} + thread_local! { static READ_BUF: RefCell> = const { RefCell::new(Vec::new()) }; } From 06bc9a9ece98b70cbe9b5150798a7929ca6a6953 Mon Sep 17 00:00:00 2001 From: Shubham Mishra Date: Fri, 27 Mar 2026 17:10:12 +0530 Subject: [PATCH 03/23] idc runtime --- crates/client-api/src/routes/database.rs | 31 +- crates/core/src/host/host_controller.rs | 22 ++ crates/core/src/host/idc_runtime.rs | 367 ++++++++++++++++++ crates/core/src/host/instance_env.rs | 69 +++- crates/core/src/host/mod.rs | 1 + crates/core/src/host/module_common.rs | 10 +- crates/core/src/host/module_host.rs | 16 +- crates/core/src/host/v8/mod.rs | 8 +- .../src/host/wasm_common/module_host_actor.rs | 62 ++- crates/core/src/module_host_context.rs | 2 + .../locking_tx_datastore/committed_state.rs | 7 +- .../src/locking_tx_datastore/mut_tx.rs | 141 +++++-- crates/datastore/src/system_tables.rs | 123 ++++-- crates/datastore/src/traits.rs | 2 + crates/table/src/table.rs | 14 + 15 files changed, 783 insertions(+), 92 deletions(-) create mode 100644 crates/core/src/host/idc_runtime.rs diff --git a/crates/client-api/src/routes/database.rs b/crates/client-api/src/routes/database.rs index 3b9d49ca994..71a2606f4b8 100644 --- a/crates/client-api/src/routes/database.rs +++ b/crates/client-api/src/routes/database.rs @@ -230,9 +230,9 @@ pub struct CallFromDatabaseParams { pub struct CallFromDatabaseQuery { /// Hex-encoded [`Identity`] of the sending database. sender_identity: String, - /// The commitlog offset of the message on the sender side. - /// Used for at-most-once delivery via `st_databases_tx_offset`. - tx_offset: u64, + /// The inter-database message ID from the sender's st_msg_id. + /// Used for at-most-once delivery via `st_inbound_msg_id`. + msg_id: u64, } /// Call a reducer on behalf of another database, with deduplication. @@ -241,10 +241,10 @@ pub struct CallFromDatabaseQuery { /// /// Required query params: /// - `sender_identity` — hex-encoded identity of the sending database. -/// - `tx_offset` — the sender's commitlog offset for this message. +/// - `msg_id` — the inter-database message ID from the sender's st_msg_id. /// -/// Before invoking the reducer, the receiver checks `st_databases_tx_offset`. -/// If the incoming `tx_offset` is ≤ the last delivered offset for `sender_identity`, +/// Before invoking the reducer, the receiver checks `st_inbound_msg_id`. +/// If the incoming `msg_id` is ≤ the last delivered msg_id for `sender_identity`, /// the call is a duplicate and 200 OK is returned immediately without running the reducer. /// Otherwise the reducer is invoked, the dedup index is updated atomically in the same /// transaction, and an acknowledgment is returned on success. @@ -257,19 +257,22 @@ pub async fn call_from_database( }): Path, Query(CallFromDatabaseQuery { sender_identity, - tx_offset, + msg_id, }): Query, TypedHeader(content_type): TypedHeader, - ByteStringBody(body): ByteStringBody, + body: axum::body::Bytes, ) -> axum::response::Result { - assert_content_type_json(content_type)?; + // IDC callers send BSATN (application/octet-stream). + if content_type != headers::ContentType::octet_stream() { + return Err((StatusCode::UNSUPPORTED_MEDIA_TYPE, "Expected application/octet-stream").into()); + } let caller_identity = auth.claims.identity; let sender_identity = Identity::from_hex(&sender_identity) .map_err(|_| (StatusCode::BAD_REQUEST, "Invalid sender_identity: expected hex-encoded identity"))?; - let args = FunctionArgs::Json(body); + let args = FunctionArgs::Bsatn(body); let connection_id = generate_random_connection_id(); let (module, Database { owner_identity, .. }) = find_module_and_database(&worker_ctx, name_or_identity).await?; @@ -290,7 +293,7 @@ pub async fn call_from_database( &reducer, args, sender_identity, - tx_offset, + msg_id, ) .await; @@ -304,7 +307,9 @@ pub async fn call_from_database( let (status, body) = match rcr.outcome { ReducerOutcome::Committed => (StatusCode::OK, "".into()), ReducerOutcome::Deduplicated => (StatusCode::OK, "deduplicated".into()), - ReducerOutcome::Failed(errmsg) => (StatusCode::from_u16(530).unwrap(), *errmsg), + // 422 = reducer ran but returned Err; IDC runtime uses this to distinguish + // reducer failures from transport errors (which it retries). + ReducerOutcome::Failed(errmsg) => (StatusCode::UNPROCESSABLE_ENTITY, *errmsg), ReducerOutcome::BudgetExceeded => { log::warn!( "Node's energy budget exceeded for identity: {owner_identity} while executing {reducer}" @@ -1300,7 +1305,7 @@ pub struct DatabaseRoutes { pub subscribe_get: MethodRouter, /// POST: /database/:name_or_identity/call/:reducer pub call_reducer_procedure_post: MethodRouter, - /// POST: /database/:name_or_identity/call-from-database/:reducer?sender_identity=&tx_offset= + /// POST: /database/:name_or_identity/call-from-database/:reducer?sender_identity=&msg_id= pub call_from_database_post: MethodRouter, /// GET: /database/:name_or_identity/schema pub schema_get: MethodRouter, diff --git a/crates/core/src/host/host_controller.rs b/crates/core/src/host/host_controller.rs index 63b6d20de6a..cd2e715eed0 100644 --- a/crates/core/src/host/host_controller.rs +++ b/crates/core/src/host/host_controller.rs @@ -1,3 +1,4 @@ +use super::idc_runtime::{IdcRuntime, IdcRuntimeConfig, IdcRuntimeStarter, IdcSender}; use super::module_host::{EventStatus, ModuleHost, ModuleInfo, NoSuchModule}; use super::scheduler::SchedulerStarter; use super::wasmtime::WasmtimeRuntime; @@ -709,6 +710,7 @@ async fn make_module_host( runtimes: Arc, replica_ctx: Arc, scheduler: Scheduler, + idc_sender: IdcSender, program: Program, energy_monitor: Arc, unregister: impl Fn() + Send + Sync + 'static, @@ -724,6 +726,7 @@ async fn make_module_host( let mcc = ModuleCreationContext { replica_ctx, scheduler, + idc_sender, program_hash: program.hash, energy_monitor, }; @@ -761,6 +764,7 @@ struct LaunchedModule { module_host: ModuleHost, scheduler: Scheduler, scheduler_starter: SchedulerStarter, + idc_starter: IdcRuntimeStarter, } struct ModuleLauncher { @@ -797,10 +801,12 @@ impl ModuleLauncher { .await .map(Arc::new)?; let (scheduler, scheduler_starter) = Scheduler::open(replica_ctx.relational_db().clone()); + let (idc_starter, idc_sender) = IdcRuntime::open(); let (program, module_host) = make_module_host( self.runtimes.clone(), replica_ctx.clone(), scheduler.clone(), + idc_sender.clone(), self.program, self.energy_monitor, self.on_panic, @@ -817,6 +823,7 @@ impl ModuleLauncher { module_host, scheduler, scheduler_starter, + idc_starter, }, )) } @@ -882,6 +889,9 @@ struct Host { /// Handle to the task responsible for cleaning up old views. /// The task is aborted when [`Host`] is dropped. view_cleanup_task: AbortHandle, + /// IDC runtime: delivers outbound inter-database messages from `st_msg_id`. + /// Stopped when [`Host`] is dropped. + _idc_runtime: IdcRuntime, } impl Host { @@ -1075,6 +1085,7 @@ impl Host { module_host, scheduler, scheduler_starter, + idc_starter, } = launched; // Disconnect dangling clients. @@ -1101,6 +1112,14 @@ impl Host { let disk_metrics_recorder_task = tokio::spawn(metric_reporter(replica_ctx.clone())).abort_handle(); let view_cleanup_task = spawn_view_cleanup_loop(replica_ctx.relational_db().clone()); + let idc_runtime = idc_starter.start( + replica_ctx.relational_db().clone(), + IdcRuntimeConfig { + sender_identity: replica_ctx.database_identity, + }, + module_host.downgrade(), + ); + let module = watch::Sender::new(module_host); Ok(Host { @@ -1110,6 +1129,7 @@ impl Host { disk_metrics_recorder_task, tx_metrics_recorder_task, view_cleanup_task, + _idc_runtime: idc_runtime, }) } @@ -1182,11 +1202,13 @@ impl Host { ) -> anyhow::Result { let replica_ctx = &self.replica_ctx; let (scheduler, scheduler_starter) = Scheduler::open(self.replica_ctx.relational_db().clone()); + let (_idc_starter, idc_sender) = IdcRuntime::open(); let (program, module) = make_module_host( runtimes, replica_ctx.clone(), scheduler.clone(), + idc_sender, program, energy_monitor, on_panic, diff --git a/crates/core/src/host/idc_runtime.rs b/crates/core/src/host/idc_runtime.rs new file mode 100644 index 00000000000..4a31b376d2e --- /dev/null +++ b/crates/core/src/host/idc_runtime.rs @@ -0,0 +1,367 @@ +/// Inter-Database Communication (IDC) Runtime +/// +/// Background tokio task that: +/// 1. Loads Pending entries from `st_msg_id` on startup. +/// 2. Accepts immediate notifications via an mpsc channel when new outbox rows are inserted. +/// 3. Delivers each message in msg_id order via HTTP POST to +/// `http://localhost:80/v1/database/{target_db}/call-from-database/{reducer}?sender_identity=&msg_id=` +/// 4. On transport errors (network, 5xx, 4xx except 422/402): retries infinitely with exponential +/// backoff, blocking only the affected target database (other targets continue unaffected). +/// 5. On reducer errors (HTTP 422) or budget exceeded (HTTP 402): records the result, marks DONE, +/// and calls the configured `on_result_reducer` on the local database (if any). +/// 6. Enforces sequential delivery per target database: msg N+1 is only delivered after N is done. +use crate::db::relational_db::RelationalDB; +use crate::host::module_host::WeakModuleHost; +use crate::host::FunctionArgs; +use spacetimedb_datastore::execution_context::Workload; +use spacetimedb_datastore::system_tables::{StMsgIdRow, ST_MSG_ID_ID}; +use spacetimedb_datastore::traits::IsolationLevel; +use spacetimedb_lib::Identity; +use std::collections::{HashMap, VecDeque}; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use tokio::sync::mpsc; + +const IDC_HTTP_PORT: u16 = 80; +const INITIAL_BACKOFF: Duration = Duration::from_millis(100); +const MAX_BACKOFF: Duration = Duration::from_secs(30); +/// How long to wait before polling again when there is no work. +const POLL_INTERVAL: Duration = Duration::from_millis(500); + +/// A sender that notifies the IDC runtime of a new outbox row. +/// +/// Sending `()` wakes the runtime to deliver pending messages immediately +/// rather than waiting for the next poll cycle. +pub type IdcSender = mpsc::UnboundedSender<()>; + +/// The identity of this (sender) database, set when the IDC runtime is started. +pub struct IdcRuntimeConfig { + pub sender_identity: Identity, +} + +/// A handle that, when dropped, stops the IDC runtime background task. +pub struct IdcRuntime { + _abort: tokio::task::AbortHandle, +} + +/// Holds the receiver side of the notification channel until the runtime is started. +/// +/// Mirrors the `SchedulerStarter` pattern: create the channel before the module is +/// loaded (so the sender can be stored in `InstanceEnv`), then call [`IdcRuntimeStarter::start`] +/// once the DB is ready. +pub struct IdcRuntimeStarter { + rx: mpsc::UnboundedReceiver<()>, +} + +impl IdcRuntimeStarter { + /// Spawn the IDC runtime background task. + pub fn start( + self, + db: Arc, + config: IdcRuntimeConfig, + module_host: WeakModuleHost, + ) -> IdcRuntime { + let abort = tokio::spawn(run_idc_loop(db, config, module_host, self.rx)).abort_handle(); + IdcRuntime { _abort: abort } + } +} + +impl IdcRuntime { + /// Open the IDC channel, returning a starter and a sender. + /// + /// Store the sender in `ModuleCreationContext` so it reaches `InstanceEnv`. + /// After the module is ready, call [`IdcRuntimeStarter::start`] to spawn the loop. + pub fn open() -> (IdcRuntimeStarter, IdcSender) { + let (tx, rx) = mpsc::unbounded_channel(); + (IdcRuntimeStarter { rx }, tx) + } +} + +/// Per-target-database delivery state. +struct TargetState { + queue: VecDeque, + /// When `Some`, this target is in backoff and should not be retried until this instant. + blocked_until: Option, + /// Current backoff duration for this target (doubles on each transport error). + backoff: Duration, +} + +impl TargetState { + fn new() -> Self { + Self { + queue: VecDeque::new(), + blocked_until: None, + backoff: INITIAL_BACKOFF, + } + } + + fn is_ready(&self) -> bool { + match self.blocked_until { + None => true, + Some(until) => Instant::now() >= until, + } + } + + fn record_transport_error(&mut self) { + self.blocked_until = Some(Instant::now() + self.backoff); + self.backoff = (self.backoff * 2).min(MAX_BACKOFF); + } + + fn record_success(&mut self) { + self.blocked_until = None; + self.backoff = INITIAL_BACKOFF; + } +} + +/// Outcome of a delivery attempt. +enum DeliveryOutcome { + /// Reducer succeeded (HTTP 200). + Success, + /// Reducer ran but returned Err (HTTP 422). + ReducerError(String), + /// Budget exceeded (HTTP 402). + BudgetExceeded, + /// Transport error: network failure, unexpected HTTP status, etc. Caller should retry. + TransportError(String), +} + +/// Main IDC loop: maintain per-target queues and deliver messages. +async fn run_idc_loop( + db: Arc, + config: IdcRuntimeConfig, + module_host: WeakModuleHost, + mut notify_rx: mpsc::UnboundedReceiver<()>, +) { + let client = reqwest::Client::new(); + + // Per-target-database delivery state. + let mut targets: HashMap = HashMap::new(); + + // On startup, load any pending messages that survived a restart. + load_pending_into_targets(&db, &mut targets); + + loop { + // Deliver one message per ready target, then re-check. + let mut any_delivered = true; + while any_delivered { + any_delivered = false; + for state in targets.values_mut() { + if !state.is_ready() { + continue; + } + let Some(row) = state.queue.front().cloned() else { + continue; + }; + let outcome = attempt_delivery(&client, &config, &row).await; + match outcome { + DeliveryOutcome::TransportError(reason) => { + log::warn!( + "idc_runtime: transport error delivering msg_id={} to {}: {reason}", + row.msg_id, + hex::encode(Identity::from(row.target_db_identity).to_byte_array()), + ); + state.record_transport_error(); + // Do NOT pop the front — keep retrying this message for this target. + } + outcome => { + state.queue.pop_front(); + state.record_success(); + any_delivered = true; + let (result_status, result_payload) = outcome_to_result(&outcome); + finalize_message(&db, &module_host, &row, result_status, result_payload).await; + } + } + } + } + + // Compute how long to sleep: min over all blocked targets' unblock times. + let next_unblock = targets + .values() + .filter_map(|s| s.blocked_until) + .min() + .map(|t| t.saturating_duration_since(Instant::now())); + let sleep_duration = next_unblock.unwrap_or(POLL_INTERVAL).min(POLL_INTERVAL); + + // Wait for a notification or the next retry time. + tokio::select! { + _ = notify_rx.recv() => { + // Drain all pending notifications (coalesce bursts). + while notify_rx.try_recv().is_ok() {} + } + _ = tokio::time::sleep(sleep_duration) => {} + } + + // Reload pending messages from DB (catches anything missed and handles restart recovery). + load_pending_into_targets(&db, &mut targets); + } +} + +/// Decode the delivery outcome into `(result_status, result_payload)` for recording. +fn outcome_to_result(outcome: &DeliveryOutcome) -> (u8, String) { + use spacetimedb_datastore::system_tables::st_inbound_msg_id_result_status; + match outcome { + DeliveryOutcome::Success => (st_inbound_msg_id_result_status::SUCCESS, String::new()), + DeliveryOutcome::ReducerError(msg) => (st_inbound_msg_id_result_status::REDUCER_ERROR, msg.clone()), + DeliveryOutcome::BudgetExceeded => ( + st_inbound_msg_id_result_status::REDUCER_ERROR, + "budget exceeded".to_string(), + ), + DeliveryOutcome::TransportError(_) => unreachable!("transport errors never finalize"), + } +} + +/// Finalize a delivered message: call the on_result reducer (if any), then delete from ST_MSG_ID. +async fn finalize_message( + db: &RelationalDB, + module_host: &WeakModuleHost, + row: &StMsgIdRow, + _result_status: u8, + result_payload: String, +) { + // Call the on_result reducer if configured. + if !row.on_result_reducer.is_empty() { + let Some(host) = module_host.upgrade() else { + log::warn!( + "idc_runtime: module host gone, cannot call on_result reducer '{}' for msg_id={}", + row.on_result_reducer, + row.msg_id, + ); + delete_message(db, row.msg_id); + return; + }; + + // Encode (result_payload: String) as BSATN args. + // The on_result reducer is expected to accept a single String argument. + let args_bytes = match spacetimedb_sats::bsatn::to_vec(&result_payload) { + Ok(b) => b, + Err(e) => { + log::error!( + "idc_runtime: failed to encode on_result args for msg_id={}: {e}", + row.msg_id + ); + delete_message(db, row.msg_id); + return; + } + }; + + let caller_identity = Identity::ZERO; // system call + let result = host + .call_reducer( + caller_identity, + None, // no connection_id + None, // no client sender + None, // no request_id + None, // no timer + &row.on_result_reducer, + FunctionArgs::Bsatn(bytes::Bytes::from(args_bytes)), + ) + .await; + + match result { + Ok(_) => { + log::debug!( + "idc_runtime: on_result reducer '{}' called for msg_id={}", + row.on_result_reducer, + row.msg_id, + ); + } + Err(e) => { + log::error!( + "idc_runtime: on_result reducer '{}' failed for msg_id={}: {e:?}", + row.on_result_reducer, + row.msg_id, + ); + } + } + } + + // Delete the row regardless of whether on_result succeeded or failed. + delete_message(db, row.msg_id); +} + +/// Load all messages from ST_MSG_ID into the per-target queues. +/// +/// A row's presence in the table means it has not yet been processed. +/// Messages that are already in a target's queue (by msg_id) are not re-added. +fn load_pending_into_targets(db: &RelationalDB, targets: &mut HashMap) { + let tx = db.begin_tx(Workload::Internal); + let rows: Vec = db + .iter(&tx, ST_MSG_ID_ID) + .map(|iter| iter.filter_map(|row_ref| StMsgIdRow::try_from(row_ref).ok()).collect()) + .unwrap_or_else(|e| { + log::error!("idc_runtime: failed to read pending messages: {e}"); + Vec::new() + }); + drop(tx); + + // Sort by msg_id ascending so delivery order is preserved. + let mut sorted = rows; + sorted.sort_by_key(|r| r.msg_id); + + for row in sorted { + let target_id = Identity::from(row.target_db_identity); + let state = targets.entry(target_id).or_insert_with(TargetState::new); + // Only add if not already in the queue (avoid duplicates after reload). + let already_queued = state.queue.iter().any(|r| r.msg_id == row.msg_id); + if !already_queued { + state.queue.push_back(row); + } + } +} + +/// Attempt a single HTTP delivery of a message. +async fn attempt_delivery( + client: &reqwest::Client, + config: &IdcRuntimeConfig, + row: &StMsgIdRow, +) -> DeliveryOutcome { + let target_identity = Identity::from(row.target_db_identity); + let target_db_hex = hex::encode(target_identity.to_byte_array()); + let sender_hex = hex::encode(config.sender_identity.to_byte_array()); + + let url = format!( + "http://localhost:{IDC_HTTP_PORT}/v1/database/{target_db_hex}/call-from-database/{}?sender_identity={sender_hex}&msg_id={}", + row.target_reducer, row.msg_id, + ); + + let result = client + .post(&url) + .header("Content-Type", "application/octet-stream") + .body(row.args_bsatn.clone()) + .send() + .await; + + match result { + Err(e) => DeliveryOutcome::TransportError(e.to_string()), + Ok(resp) => { + let status = resp.status(); + if status.is_success() { + // HTTP 200: reducer committed successfully. + DeliveryOutcome::Success + } else if status.as_u16() == 422 { + // HTTP 422: reducer ran but returned Err(...). + let body = resp.text().await.unwrap_or_default(); + DeliveryOutcome::ReducerError(body) + } else if status.as_u16() == 402 { + // HTTP 402: budget exceeded. + DeliveryOutcome::BudgetExceeded + } else { + // Any other error (503, 404, etc.) is a transport error: retry. + DeliveryOutcome::TransportError(format!("HTTP {status}")) + } + } + } +} + +/// Delete a message from ST_MSG_ID within a new transaction. +fn delete_message(db: &RelationalDB, msg_id: u64) { + let mut tx = db.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); + if let Err(e) = tx.delete_msg_id(msg_id) { + log::error!("idc_runtime: failed to delete msg_id={msg_id}: {e}"); + let _ = db.rollback_mut_tx(tx); + return; + } + if let Err(e) = db.commit_tx(tx) { + log::error!("idc_runtime: failed to commit delete for msg_id={msg_id}: {e}"); + } +} diff --git a/crates/core/src/host/instance_env.rs b/crates/core/src/host/instance_env.rs index 311c34775fd..1f1956fd221 100644 --- a/crates/core/src/host/instance_env.rs +++ b/crates/core/src/host/instance_env.rs @@ -1,3 +1,4 @@ +use super::idc_runtime::IdcSender; use super::scheduler::{get_schedule_from_row, ScheduleError, Scheduler}; use crate::database_logger::{BacktraceFrame, BacktraceProvider, LogLevel, ModuleBacktrace, Record}; use crate::db::relational_db::{MutTx, RelationalDB}; @@ -40,6 +41,8 @@ use std::vec::IntoIter; pub struct InstanceEnv { pub replica_ctx: Arc, pub scheduler: Scheduler, + /// Sender to notify the IDC runtime of new outbox rows. + pub idc_sender: IdcSender, pub tx: TxSlot, /// The timestamp the current function began running. pub start_time: Timestamp, @@ -224,10 +227,11 @@ impl ChunkedWriter { // Generic 'instance environment' delegated to from various host types. impl InstanceEnv { - pub fn new(replica_ctx: Arc, scheduler: Scheduler) -> Self { + pub fn new(replica_ctx: Arc, scheduler: Scheduler, idc_sender: IdcSender) -> Self { Self { replica_ctx, scheduler, + idc_sender, tx: TxSlot::default(), start_time: Timestamp::now(), start_instant: Instant::now(), @@ -385,6 +389,10 @@ impl InstanceEnv { self.schedule_row(stdb, tx, table_id, row_ptr)?; } + if insert_flags.is_outbox_table { + self.enqueue_outbox_row(stdb, tx, table_id, row_ptr)?; + } + // Note, we update the metric for bytes written after the insert. // This is to capture auto-inc columns. tx.metrics.bytes_written += buffer.len(); @@ -424,6 +432,59 @@ impl InstanceEnv { Ok(()) } + /// Enqueue an outbox row into ST_MSG_ID atomically within the current transaction, + /// and notify the IDC runtime so it delivers without waiting for the next poll cycle. + /// + /// Outbox tables follow the naming convention `__outbox_`. + /// The first column (col 0) must hold the target database Identity (BSATN-encoded as U256). + /// The remaining bytes of the row's BSATN encoding are passed as `args_bsatn` to the + /// remote reducer. + #[cold] + #[inline(never)] + fn enqueue_outbox_row( + &self, + _stdb: &RelationalDB, + tx: &mut MutTx, + table_id: TableId, + row_ptr: RowPointer, + ) -> Result<(), NodesError> { + use spacetimedb_datastore::locking_tx_datastore::state_view::StateView as _; + + // Get table name to extract reducer name. + let schema = tx.schema_for_table(table_id).map_err(DBError::from)?; + let table_name = schema.table_name.to_string(); + let reducer_name = table_name.strip_prefix("__outbox_").unwrap_or(&table_name).to_string(); + + let row_ref = tx.get(table_id, row_ptr).map_err(DBError::from)?.unwrap(); + + let pv = row_ref.to_product_value(); + + // Col 0 must be the target database Identity, stored as AlgebraicValue::U256. + let target_db_identity = match pv.elements.first() { + Some(AlgebraicValue::U256(u)) => Identity::from_u256(**u), + other => { + return Err(NodesError::Internal(Box::new(DBError::Other(anyhow::anyhow!( + "outbox table {table_name}: expected col 0 to be U256 (Identity), got {other:?}" + ))))); + } + }; + + // Remaining elements are the args for the remote reducer. + let args_bsatn = pv.elements[1..].iter().fold(Vec::new(), |mut acc, elem| { + bsatn::to_writer(&mut acc, elem).expect("writing outbox row args to BSATN should never fail"); + acc + }); + + // TODO: populate on_result_reducer from TableDef once that field is added. + tx.insert_st_msg_id(table_id.0, target_db_identity, reducer_name, args_bsatn, String::new()) + .map_err(DBError::from)?; + + // Wake the IDC runtime immediately so it doesn't wait for the next poll cycle. + let _ = self.idc_sender.send(()); + + Ok(()) + } + pub fn update(&self, table_id: TableId, index_id: IndexId, buffer: &mut [u8]) -> Result { let stdb = self.relational_db(); let tx = &mut *self.get_tx()?; @@ -1360,7 +1421,11 @@ mod test { fn instance_env(db: Arc) -> Result<(InstanceEnv, tokio::runtime::Runtime)> { let (scheduler, _) = Scheduler::open(db.clone()); let (replica_context, runtime) = replica_ctx(db)?; - Ok((InstanceEnv::new(Arc::new(replica_context), scheduler), runtime)) + let (_, idc_sender) = crate::host::idc_runtime::IdcRuntime::open(); + Ok(( + InstanceEnv::new(Arc::new(replica_context), scheduler, idc_sender), + runtime, + )) } /// An in-memory `RelationalDB` for testing. diff --git a/crates/core/src/host/mod.rs b/crates/core/src/host/mod.rs index 0daa9c359bc..5125c35e6de 100644 --- a/crates/core/src/host/mod.rs +++ b/crates/core/src/host/mod.rs @@ -12,6 +12,7 @@ use spacetimedb_schema::def::ModuleDef; mod disk_storage; mod host_controller; +pub mod idc_runtime; mod module_common; #[allow(clippy::too_many_arguments)] pub mod module_host; diff --git a/crates/core/src/host/module_common.rs b/crates/core/src/host/module_common.rs index bff864759f3..5337897ec75 100644 --- a/crates/core/src/host/module_common.rs +++ b/crates/core/src/host/module_common.rs @@ -4,6 +4,7 @@ use crate::{ energy::EnergyMonitor, host::{ + idc_runtime::IdcSender, module_host::ModuleInfo, wasm_common::{module_host_actor::DescribeError, DESCRIBE_MODULE_DUNDER}, Scheduler, @@ -37,7 +38,7 @@ pub fn build_common_module_from_raw( replica_ctx.subscriptions.clone(), ); - Ok(ModuleCommon::new(replica_ctx, mcc.scheduler, info, mcc.energy_monitor)) + Ok(ModuleCommon::new(replica_ctx, mcc.scheduler, mcc.idc_sender, info, mcc.energy_monitor)) } /// Non-runtime-specific parts of a module. @@ -45,6 +46,7 @@ pub fn build_common_module_from_raw( pub(crate) struct ModuleCommon { replica_context: Arc, scheduler: Scheduler, + idc_sender: IdcSender, info: Arc, energy_monitor: Arc, } @@ -54,12 +56,14 @@ impl ModuleCommon { fn new( replica_context: Arc, scheduler: Scheduler, + idc_sender: IdcSender, info: Arc, energy_monitor: Arc, ) -> Self { Self { replica_context, scheduler, + idc_sender, info, energy_monitor, } @@ -89,6 +93,10 @@ impl ModuleCommon { pub fn scheduler(&self) -> &Scheduler { &self.scheduler } + + pub fn idc_sender(&self) -> IdcSender { + self.idc_sender.clone() + } } /// Runs the describer of modules in `run` and does some logging around it. diff --git a/crates/core/src/host/module_host.rs b/crates/core/src/host/module_host.rs index ebbd7001bbd..e9b823be575 100644 --- a/crates/core/src/host/module_host.rs +++ b/crates/core/src/host/module_host.rs @@ -626,11 +626,11 @@ pub struct CallReducerParams { pub args: ArgsTuple, /// If set, enables at-most-once delivery semantics for database-to-database calls. /// - /// The tuple is `(sender_database_identity, sender_tx_offset)`. - /// Before running the reducer, `st_databases_tx_offset` is consulted: - /// if `sender_tx_offset` ≤ the stored last-delivered offset for the sender, + /// The tuple is `(sender_database_identity, sender_msg_id)`. + /// Before running the reducer, `st_inbound_msg_id` is consulted: + /// if `sender_msg_id` ≤ the stored last-delivered msg_id for the sender, /// the call is a duplicate and returns [`ReducerCallError::Deduplicated`]. - /// Otherwise the reducer runs, and the stored offset is updated atomically + /// Otherwise the reducer runs, and the stored msg_id is updated atomically /// within the same transaction. pub dedup_sender: Option<(Identity, u64)>, } @@ -1670,9 +1670,9 @@ impl ModuleHost { /// Variant of [`Self::call_reducer`] for database-to-database calls. /// /// Behaves identically to `call_reducer`, except that it enforces at-most-once - /// delivery using the `st_databases_tx_offset` dedup index. + /// delivery using the `st_inbound_msg_id` dedup index. /// Before invoking the reducer, the receiver checks whether - /// `sender_tx_offset` ≤ the last delivered offset for `sender_database_identity`. + /// `sender_msg_id` ≤ the last delivered msg_id for `sender_database_identity`. /// If so, the call is a duplicate and [`ReducerOutcome::Deduplicated`] is returned /// without running the reducer. /// Otherwise the reducer runs, and the dedup index is updated atomically @@ -1687,7 +1687,7 @@ impl ModuleHost { reducer_name: &str, args: FunctionArgs, sender_database_identity: Identity, - sender_tx_offset: u64, + sender_msg_id: u64, ) -> Result { let res = async { let (reducer_id, reducer_def) = self @@ -1716,7 +1716,7 @@ impl ModuleHost { timer, reducer_id, args, - dedup_sender: Some((sender_database_identity, sender_tx_offset)), + dedup_sender: Some((sender_database_identity, sender_msg_id)), }; self.call( diff --git a/crates/core/src/host/v8/mod.rs b/crates/core/src/host/v8/mod.rs index 9ab92263961..c5d1964bc5b 100644 --- a/crates/core/src/host/v8/mod.rs +++ b/crates/core/src/host/v8/mod.rs @@ -1062,11 +1062,11 @@ async fn spawn_instance_worker( scope_with_context!(let scope, &mut isolate, Context::new(scope, Default::default())); // Setup the instance environment. - let (replica_ctx, scheduler) = match &module_or_mcc { - Either::Left(module) => (module.replica_ctx(), module.scheduler()), - Either::Right(mcc) => (&mcc.replica_ctx, &mcc.scheduler), + let (replica_ctx, scheduler, idc_sender) = match &module_or_mcc { + Either::Left(module) => (module.replica_ctx(), module.scheduler(), module.idc_sender()), + Either::Right(mcc) => (&mcc.replica_ctx, &mcc.scheduler, mcc.idc_sender.clone()), }; - let instance_env = InstanceEnv::new(replica_ctx.clone(), scheduler.clone()); + let instance_env = InstanceEnv::new(replica_ctx.clone(), scheduler.clone(), idc_sender); scope.set_slot(JsInstanceEnv::new(instance_env)); catch_exception(scope, |scope| Ok(builtins::evaluate_builtins(scope)?)) diff --git a/crates/core/src/host/wasm_common/module_host_actor.rs b/crates/core/src/host/wasm_common/module_host_actor.rs index 338e1c58680..5e381ee0c44 100644 --- a/crates/core/src/host/wasm_common/module_host_actor.rs +++ b/crates/core/src/host/wasm_common/module_host_actor.rs @@ -34,6 +34,7 @@ use core::time::Duration; use prometheus::{Histogram, IntCounter, IntGauge}; use spacetimedb_auth::identity::ConnectionAuthCtx; use spacetimedb_datastore::db_metrics::DB_METRICS; +use spacetimedb_datastore::system_tables::st_inbound_msg_id_result_status; use spacetimedb_datastore::error::{DatastoreError, ViewError}; use spacetimedb_datastore::execution_context::{self, ReducerContext, Workload}; use spacetimedb_datastore::locking_tx_datastore::{FuncCallType, MutTxId, ViewCallInfo}; @@ -375,7 +376,7 @@ impl WasmModuleHostActor { func_names }; let uninit_instance = module.instantiate_pre()?; - let instance_env = InstanceEnv::new(mcc.replica_ctx.clone(), mcc.scheduler.clone()); + let instance_env = InstanceEnv::new(mcc.replica_ctx.clone(), mcc.scheduler.clone(), mcc.idc_sender.clone()); let mut instance = uninit_instance.instantiate(instance_env, &func_names)?; let desc = instance.extract_descriptions()?; @@ -422,7 +423,7 @@ impl WasmModuleHostActor { pub fn create_instance(&self) -> WasmModuleInstance { let common = &self.common; - let env = InstanceEnv::new(common.replica_ctx().clone(), common.scheduler().clone()); + let env = InstanceEnv::new(common.replica_ctx().clone(), common.scheduler().clone(), common.idc_sender()); // this shouldn't fail, since we already called module.create_instance() // before and it didn't error, and ideally they should be deterministic let mut instance = self @@ -825,7 +826,7 @@ impl InstanceCommon { let caller_connection_id_opt = (caller_connection_id != ConnectionId::ZERO).then_some(caller_connection_id); let replica_ctx = inst.replica_ctx(); - let stdb = replica_ctx.relational_db(); + let stdb = replica_ctx.relational_db().clone(); let info = self.info.clone(); let reducer_def = info.module_def.reducer_by_id(reducer_id); let reducer_name = &reducer_def.name; @@ -847,15 +848,23 @@ impl InstanceCommon { let tx = tx.unwrap_or_else(|| stdb.begin_mut_tx(IsolationLevel::Serializable, workload)); // Check the dedup index atomically within this transaction. - // If the incoming offset is ≤ the last delivered offset for this sender, - // the message is a duplicate; discard it and roll back. - if let Some((sender_identity, sender_tx_offset)) = dedup_sender { - let last_delivered = tx.get_databases_tx_offset(sender_identity); - if last_delivered.is_some_and(|last| sender_tx_offset <= last) { + // If the incoming msg_id is ≤ the last delivered msg_id for this sender, + // the message is a duplicate; discard it and return the stored result. + if let Some((sender_identity, sender_msg_id)) = dedup_sender { + let stored = tx.get_inbound_msg_id_row(sender_identity); + if stored.as_ref().is_some_and(|r| sender_msg_id <= r.last_msg_id) { let _ = stdb.rollback_mut_tx(tx); + let stored = stored.unwrap(); + let outcome = match stored.result_status { + s if s == st_inbound_msg_id_result_status::SUCCESS => ReducerOutcome::Committed, + s if s == st_inbound_msg_id_result_status::REDUCER_ERROR => { + ReducerOutcome::Failed(Box::new(stored.result_payload.into())) + } + _ => ReducerOutcome::Deduplicated, + }; return ( ReducerCallResult { - outcome: ReducerOutcome::Deduplicated, + outcome, energy_used: EnergyQuanta::ZERO, execution_duration: Duration::ZERO, }, @@ -914,14 +923,19 @@ impl InstanceCommon { // Atomically update the dedup index so this sender's tx_offset is recorded // as delivered. This prevents re-processing if the message is retried. let dedup_res = match dedup_sender { - Some((sender_identity, sender_tx_offset)) => { - tx.upsert_databases_tx_offset(sender_identity, sender_tx_offset) - } + Some((sender_identity, sender_msg_id)) => tx.upsert_inbound_last_msg_id( + sender_identity, + sender_msg_id, + st_inbound_msg_id_result_status::SUCCESS, + String::new(), + ), None => Ok(()), }; let res = lifecycle_res.and(dedup_res); match res { - Ok(()) => (EventStatus::Committed(DatabaseUpdate::default()), return_value), + Ok(()) => { + (EventStatus::Committed(DatabaseUpdate::default()), return_value) + } Err(err) => { let err = err.to_string(); log_reducer_error( @@ -979,6 +993,28 @@ impl InstanceCommon { }; let event = commit_and_broadcast_event(&info.subscriptions, client, event, out.tx).event; + // For IDC (database-to-database) calls: if the reducer failed with a user error, + // record the failure in st_inbound_msg_id in a separate tx (since the reducer tx + // was rolled back). This allows the sending database to receive the error on dedup + // rather than re-running the reducer. + if let (Some((sender_identity, sender_msg_id)), EventStatus::FailedUser(err)) = + (dedup_sender, &event.status) + { + let err_msg = err.clone(); + let mut dedup_tx = stdb.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); + if let Err(e) = dedup_tx.upsert_inbound_last_msg_id( + sender_identity, + sender_msg_id, + st_inbound_msg_id_result_status::REDUCER_ERROR, + err_msg, + ) { + log::error!("IDC: failed to record reducer error in dedup table for sender {sender_identity}: {e}"); + let _ = stdb.rollback_mut_tx(dedup_tx); + } else if let Err(e) = stdb.commit_tx(dedup_tx) { + log::error!("IDC: failed to commit dedup error record for sender {sender_identity}: {e}"); + } + } + let res = ReducerCallResult { outcome: ReducerOutcome::from(&event.status), energy_used: energy_quanta_used, diff --git a/crates/core/src/module_host_context.rs b/crates/core/src/module_host_context.rs index 50f8c258a37..a3f39c3cb13 100644 --- a/crates/core/src/module_host_context.rs +++ b/crates/core/src/module_host_context.rs @@ -1,4 +1,5 @@ use crate::energy::EnergyMonitor; +use crate::host::idc_runtime::IdcSender; use crate::host::scheduler::Scheduler; use crate::replica_context::ReplicaContext; use spacetimedb_sats::hash::Hash; @@ -7,6 +8,7 @@ use std::sync::Arc; pub struct ModuleCreationContext { pub replica_ctx: Arc, pub scheduler: Scheduler, + pub idc_sender: IdcSender, pub program_hash: Hash, pub energy_monitor: Arc, } diff --git a/crates/datastore/src/locking_tx_datastore/committed_state.rs b/crates/datastore/src/locking_tx_datastore/committed_state.rs index 6c9fbde3f1e..faaec67eac9 100644 --- a/crates/datastore/src/locking_tx_datastore/committed_state.rs +++ b/crates/datastore/src/locking_tx_datastore/committed_state.rs @@ -19,7 +19,7 @@ use crate::{ is_built_in_meta_row, system_tables, table_id_is_reserved, StColumnRow, StConstraintData, StConstraintRow, StFields, StIndexRow, StSequenceFields, StSequenceRow, StTableFields, StTableRow, StViewRow, SystemTable, ST_CLIENT_ID, ST_CLIENT_IDX, ST_COLUMN_ID, ST_COLUMN_IDX, ST_COLUMN_NAME, ST_CONSTRAINT_ID, ST_CONSTRAINT_IDX, - ST_CONSTRAINT_NAME, ST_DATABASES_TX_OFFSET_ID, ST_INDEX_ID, ST_INDEX_IDX, ST_INDEX_NAME, ST_MODULE_ID, + ST_CONSTRAINT_NAME, ST_INBOUND_MSG_ID_ID, ST_INDEX_ID, ST_INDEX_IDX, ST_INDEX_NAME, ST_MODULE_ID, ST_MODULE_IDX, ST_ROW_LEVEL_SECURITY_ID, ST_ROW_LEVEL_SECURITY_IDX, ST_SCHEDULED_ID, ST_SCHEDULED_IDX, ST_SEQUENCE_ID, ST_SEQUENCE_IDX, ST_SEQUENCE_NAME, ST_TABLE_ID, ST_TABLE_IDX, ST_VAR_ID, ST_VAR_IDX, ST_VIEW_ARG_ID, ST_VIEW_ARG_IDX, @@ -30,7 +30,7 @@ use crate::{ locking_tx_datastore::ViewCallInfo, system_tables::{ ST_COLUMN_ACCESSOR_ID, ST_COLUMN_ACCESSOR_IDX, ST_CONNECTION_CREDENTIALS_ID, ST_CONNECTION_CREDENTIALS_IDX, - ST_DATABASES_TX_OFFSET_IDX, ST_EVENT_TABLE_ID, ST_EVENT_TABLE_IDX, ST_INDEX_ACCESSOR_ID, ST_INDEX_ACCESSOR_IDX, + ST_INBOUND_MSG_ID_IDX, ST_MSG_ID_ID, ST_MSG_ID_IDX, ST_EVENT_TABLE_ID, ST_EVENT_TABLE_IDX, ST_INDEX_ACCESSOR_ID, ST_INDEX_ACCESSOR_IDX, ST_TABLE_ACCESSOR_ID, ST_TABLE_ACCESSOR_IDX, ST_VIEW_COLUMN_ID, ST_VIEW_COLUMN_IDX, ST_VIEW_ID, ST_VIEW_IDX, ST_VIEW_PARAM_ID, ST_VIEW_PARAM_IDX, ST_VIEW_SUB_ID, ST_VIEW_SUB_IDX, }, @@ -477,7 +477,8 @@ impl CommittedState { self.create_table(ST_TABLE_ACCESSOR_ID, schemas[ST_TABLE_ACCESSOR_IDX].clone()); self.create_table(ST_INDEX_ACCESSOR_ID, schemas[ST_INDEX_ACCESSOR_IDX].clone()); self.create_table(ST_COLUMN_ACCESSOR_ID, schemas[ST_COLUMN_ACCESSOR_IDX].clone()); - self.create_table(ST_DATABASES_TX_OFFSET_ID, schemas[ST_DATABASES_TX_OFFSET_IDX].clone()); + self.create_table(ST_INBOUND_MSG_ID_ID, schemas[ST_INBOUND_MSG_ID_IDX].clone()); + self.create_table(ST_MSG_ID_ID, schemas[ST_MSG_ID_IDX].clone()); // Insert the sequences into `st_sequences` let (st_sequences, blob_store, pool) = diff --git a/crates/datastore/src/locking_tx_datastore/mut_tx.rs b/crates/datastore/src/locking_tx_datastore/mut_tx.rs index bc531679346..0d339161819 100644 --- a/crates/datastore/src/locking_tx_datastore/mut_tx.rs +++ b/crates/datastore/src/locking_tx_datastore/mut_tx.rs @@ -20,14 +20,14 @@ use crate::{ use crate::{ error::{IndexError, SequenceError, TableError}, system_tables::{ - with_sys_table_buf, StClientFields, StClientRow, StColumnAccessorFields, StColumnAccessorRow, StColumnFields, - StColumnRow, StConstraintFields, StConstraintRow, StDatabasesTxOffsetFields, StDatabasesTxOffsetRow, - StEventTableRow, StFields as _, StIndexAccessorFields, StIndexAccessorRow, StIndexFields, StIndexRow, - StRowLevelSecurityFields, StRowLevelSecurityRow, StScheduledFields, StScheduledRow, StSequenceFields, - StSequenceRow, StTableAccessorFields, StTableAccessorRow, StTableFields, StTableRow, SystemTable, - ST_CLIENT_ID, ST_COLUMN_ACCESSOR_ID, ST_COLUMN_ID, ST_CONSTRAINT_ID, ST_DATABASES_TX_OFFSET_ID, - ST_EVENT_TABLE_ID, ST_INDEX_ACCESSOR_ID, ST_INDEX_ID, ST_ROW_LEVEL_SECURITY_ID, ST_SCHEDULED_ID, - ST_SEQUENCE_ID, ST_TABLE_ACCESSOR_ID, ST_TABLE_ID, + with_sys_table_buf, StClientFields, StClientRow, StColumnAccessorFields, StColumnAccessorRow, + StColumnFields, StColumnRow, StConstraintFields, StConstraintRow, StEventTableRow, StFields as _, + StInboundMsgIdFields, StInboundMsgIdRow, StIndexAccessorFields, StIndexAccessorRow, StIndexFields, StIndexRow, + StMsgIdFields, StMsgIdRow, StRowLevelSecurityFields, StRowLevelSecurityRow, StScheduledFields, StScheduledRow, + StSequenceFields, StSequenceRow, StTableAccessorFields, StTableAccessorRow, StTableFields, StTableRow, + SystemTable, ST_CLIENT_ID, ST_COLUMN_ACCESSOR_ID, ST_COLUMN_ID, ST_CONSTRAINT_ID, ST_EVENT_TABLE_ID, + ST_INBOUND_MSG_ID_ID, ST_INDEX_ACCESSOR_ID, ST_INDEX_ID, ST_MSG_ID_ID, ST_ROW_LEVEL_SECURITY_ID, + ST_SCHEDULED_ID, ST_SEQUENCE_ID, ST_TABLE_ACCESSOR_ID, ST_TABLE_ID, }, }; use crate::{execution_context::ExecutionContext, system_tables::StViewColumnRow}; @@ -44,6 +44,7 @@ use smallvec::SmallVec; use spacetimedb_data_structures::map::{HashMap, HashSet, IntMap}; use spacetimedb_durability::TxOffset; use spacetimedb_execution::{dml::MutDatastore, Datastore, DeltaStore, Row}; +use spacetimedb_lib::bsatn::ToBsatn; use spacetimedb_lib::{db::raw_def::v9::RawSql, metrics::ExecutionMetrics, Timestamp}; use spacetimedb_lib::{ db::{auth::StAccess, raw_def::SEQUENCE_ALLOCATION_STEP}, @@ -2691,44 +2692,137 @@ impl MutTxId { .map(|row| row.pointer()) } - /// Look up the last delivered tx offset for `sender_identity` in `st_databases_tx_offset`. + /// Look up the inbound dedup record for `sender_identity` in `st_inbound_msg_id`. /// /// Returns `None` if no entry exists for this sender (i.e., no message has been delivered yet). - pub fn get_databases_tx_offset(&self, sender_identity: Identity) -> Option { + pub fn get_inbound_msg_id_row(&self, sender_identity: Identity) -> Option { self.iter_by_col_eq( - ST_DATABASES_TX_OFFSET_ID, - StDatabasesTxOffsetFields::DatabaseIdentity.col_id(), + ST_INBOUND_MSG_ID_ID, + StInboundMsgIdFields::DatabaseIdentity.col_id(), &IdentityViaU256::from(sender_identity).into(), ) - .expect("failed to read from st_databases_tx_offset system table") + .expect("failed to read from st_inbound_msg_id system table") .next() - .and_then(|row_ref| StDatabasesTxOffsetRow::try_from(row_ref).ok()) - .map(|row| row.tx_offset) + .and_then(|row_ref| StInboundMsgIdRow::try_from(row_ref).ok()) } - /// Update the last delivered tx offset for `sender_identity` in `st_databases_tx_offset`. + /// Update the last delivered msg_id for `sender_identity` in `st_inbound_msg_id`. /// /// If an entry already exists, it is replaced; otherwise a new entry is inserted. - pub fn upsert_databases_tx_offset(&mut self, sender_identity: Identity, tx_offset: u64) -> Result<()> { + /// `result_status` and `result_payload` store the outcome of the reducer call + /// (see [st_inbound_msg_id_result_status]). + pub fn upsert_inbound_last_msg_id( + &mut self, + sender_identity: Identity, + last_msg_id: u64, + result_status: u8, + result_payload: String, + ) -> Result<()> { // Delete the existing row if present. self.delete_col_eq( - ST_DATABASES_TX_OFFSET_ID, - StDatabasesTxOffsetFields::DatabaseIdentity.col_id(), + ST_INBOUND_MSG_ID_ID, + StInboundMsgIdFields::DatabaseIdentity.col_id(), &IdentityViaU256::from(sender_identity).into(), )?; - let row = StDatabasesTxOffsetRow { + let row = StInboundMsgIdRow { database_identity: sender_identity.into(), - tx_offset, + last_msg_id, + result_status, + result_payload, }; - self.insert_via_serialize_bsatn(ST_DATABASES_TX_OFFSET_ID, &row) + self.insert_via_serialize_bsatn(ST_INBOUND_MSG_ID_ID, &row) .map(|_| ()) .inspect_err(|e| { log::error!( - "upsert_databases_tx_offset: failed to upsert tx offset for {sender_identity} to {tx_offset}: {e}" + "upsert_inbound_last_msg_id: failed to upsert last_msg_id for {sender_identity} to {last_msg_id}: {e}" ); }) } + /// Insert a new outbound inter-database message into `st_msg_id`. + /// Returns the auto-incremented msg_id assigned to this message. + pub fn insert_st_msg_id( + &mut self, + sender_table_id: u32, + target_db_identity: Identity, + target_reducer: String, + args_bsatn: Vec, + on_result_reducer: String, + ) -> Result<()> { + let row = StMsgIdRow { + msg_id: 0, // auto-incremented by the sequence + sender_table_id, + target_db_identity: target_db_identity.into(), + target_reducer, + args_bsatn, + on_result_reducer, + }; + self.insert_via_serialize_bsatn(ST_MSG_ID_ID, &row) + .map(|_| ()) + .inspect_err(|e| { + log::error!("insert_st_msg_id: failed to insert msg for {target_db_identity}: {e}"); + }) + } + + /// Retrieve all outbound messages from `st_msg_id`, ordered by msg_id ascending. + pub fn all_msg_ids(&self) -> Result> { + let mut rows: Vec = self + .iter(ST_MSG_ID_ID) + .expect("failed to read from st_msg_id system table") + .filter_map(|row_ref| StMsgIdRow::try_from(row_ref).ok()) + .collect(); + rows.sort_by_key(|r| r.msg_id); + Ok(rows) + } + + /// Delete a message from `st_msg_id` once it has been fully processed. + pub fn delete_msg_id(&mut self, msg_id: u64) -> Result<()> { + self.delete_col_eq( + ST_MSG_ID_ID, + StMsgIdFields::MsgId.col_id(), + &AlgebraicValue::U64(msg_id), + ) + .map(|_| ()) + .inspect_err(|e| { + log::error!("delete_msg_id: failed to delete msg_id={msg_id}: {e}"); + }) + } + + /// Returns the table IDs and table names of all tables that had rows inserted + /// in this transaction and whose name starts with `"__outbox_"`. + /// + /// Used by the IDC runtime to detect outbox inserts and enqueue ST_MSG_ID entries. + pub fn outbox_insert_table_ids(&self) -> Vec<(TableId, String)> { + let mut result = Vec::new(); + for table_id in self.tx_state.insert_tables.keys() { + if let Ok(schema) = self.schema_for_table(*table_id) { + let name = schema.table_name.to_string(); + if name.starts_with("__outbox_") { + result.push((*table_id, name)); + } + } + } + result + } + + /// Returns the raw BSATN bytes of all rows inserted into `table_id` in this transaction. + /// + /// Each `Vec` encodes a full outbox row. The first 32 bytes are the target + /// database identity (U256 little-endian), and the remaining bytes are the reducer args. + pub fn outbox_inserts_for_table(&mut self, table_id: TableId) -> Vec> { + let Some(inserted_table) = self.tx_state.insert_tables.get(&table_id) else { + return Vec::new(); + }; + // Collect blob-store-independent data: we hold a shared ref to insert_tables, + // so we can't also borrow committed_state mutably for the blob store. + // Use the blob store embedded in the TxState. + let blob_store = &self.tx_state.blob_store; + inserted_table + .scan_rows(blob_store) + .filter_map(|row_ref| row_ref.to_bsatn_vec().ok()) + .collect() + } + pub fn insert_via_serialize_bsatn<'a, T: Serialize>( &'a mut self, table_id: TableId, @@ -2887,6 +2981,7 @@ pub(super) fn insert<'a, const GENERATE: bool>( let insert_flags = InsertFlags { is_scheduler_table: tx_table.is_scheduler(), + is_outbox_table: tx_table.is_outbox(), }; let ok = |row_ref| Ok((gen_cols, row_ref, insert_flags)); diff --git a/crates/datastore/src/system_tables.rs b/crates/datastore/src/system_tables.rs index db5045030f2..a037100337d 100644 --- a/crates/datastore/src/system_tables.rs +++ b/crates/datastore/src/system_tables.rs @@ -89,8 +89,11 @@ pub const ST_INDEX_ACCESSOR_ID: TableId = TableId(19); /// The static ID of the table that maps canonical column names to accessor names pub const ST_COLUMN_ACCESSOR_ID: TableId = TableId(20); -/// The static ID of the table that tracks the last tx offset send by other databases. -pub const ST_DATABASES_TX_OFFSET_ID: TableId = TableId(21); +/// The static ID of the table that tracks the last inbound msg id per sender database. +pub const ST_INBOUND_MSG_ID_ID: TableId = TableId(21); + +/// The static ID of the table that tracks outbound inter-database messages. +pub const ST_MSG_ID_ID: TableId = TableId(22); pub(crate) const ST_CONNECTION_CREDENTIALS_NAME: &str = "st_connection_credentials"; pub const ST_TABLE_NAME: &str = "st_table"; @@ -112,7 +115,8 @@ pub(crate) const ST_EVENT_TABLE_NAME: &str = "st_event_table"; pub(crate) const ST_TABLE_ACCESSOR_NAME: &str = "st_table_accessor"; pub(crate) const ST_INDEX_ACCESSOR_NAME: &str = "st_index_accessor"; pub(crate) const ST_COLUMN_ACCESSOR_NAME: &str = "st_column_accessor"; -pub(crate) const ST_DATABASES_TX_OFFSET_NAME: &str = "st_databases_tx_offset"; +pub(crate) const ST_INBOUND_MSG_ID_NAME: &str = "st_inbound_msg_id"; +pub(crate) const ST_MSG_ID_NAME: &str = "st_msg_id"; /// Reserved range of sequence values used for system tables. /// /// Ids for user-created tables will start at `ST_RESERVED_SEQUENCE_RANGE`. @@ -186,7 +190,7 @@ pub fn is_built_in_meta_row(table_id: TableId, row: &ProductValue) -> Result false, + ST_TABLE_ACCESSOR_ID | ST_INDEX_ACCESSOR_ID | ST_COLUMN_ACCESSOR_ID | ST_INBOUND_MSG_ID_ID | ST_MSG_ID_ID => false, TableId(..ST_RESERVED_SEQUENCE_RANGE) => { log::warn!("Unknown system table {table_id:?}"); false @@ -209,7 +213,7 @@ pub enum SystemTable { st_table_accessor, } -pub fn system_tables() -> [TableSchema; 21] { +pub fn system_tables() -> [TableSchema; 22] { [ // The order should match the `id` of the system table, that start with [ST_TABLE_IDX]. st_table_schema(), @@ -232,7 +236,8 @@ pub fn system_tables() -> [TableSchema; 21] { st_table_accessor_schema(), st_index_accessor_schema(), st_column_accessor_schema(), - st_databases_tx_offset_schema(), + st_inbound_msg_id_schema(), + st_msg_id_schema(), ] } @@ -281,7 +286,8 @@ pub(crate) const ST_EVENT_TABLE_IDX: usize = 16; pub(crate) const ST_TABLE_ACCESSOR_IDX: usize = 17; pub(crate) const ST_INDEX_ACCESSOR_IDX: usize = 18; pub(crate) const ST_COLUMN_ACCESSOR_IDX: usize = 19; -pub(crate) const ST_DATABASES_TX_OFFSET_IDX: usize = 20; +pub(crate) const ST_INBOUND_MSG_ID_IDX: usize = 20; +pub(crate) const ST_MSG_ID_IDX: usize = 21; macro_rules! st_fields_enum { ($(#[$attr:meta])* enum $ty_name:ident { $($name:expr, $var:ident = $discr:expr,)* }) => { @@ -456,9 +462,20 @@ st_fields_enum!(enum StColumnAccessorFields { "accessor_name", AccessorName = 2, }); -st_fields_enum!(enum StDatabasesTxOffsetFields { +st_fields_enum!(enum StInboundMsgIdFields { "database_identity", DatabaseIdentity = 0, - "tx_offset", TxOffset = 1, + "last_msg_id", LastMsgId = 1, + "result_status", ResultStatus = 2, + "result_payload", ResultPayload = 3, +}); + +st_fields_enum!(enum StMsgIdFields { + "msg_id", MsgId = 0, + "sender_table_id", SenderTableId = 1, + "target_db_identity", TargetDbIdentity = 2, + "target_reducer", TargetReducer = 3, + "args_bsatn", ArgsBsatn = 4, + "on_result_reducer", OnResultReducer = 5, }); /// Helper method to check that a system table has the correct fields. @@ -679,16 +696,26 @@ fn system_module_def() -> ModuleDef { .with_unique_constraint(st_column_accessor_table_alias_cols) .with_index_no_accessor_name(btree(st_column_accessor_table_alias_cols)); - let st_databases_tx_offset_type = builder.add_type::(); + let st_inbound_msg_id_type = builder.add_type::(); builder .build_table( - ST_DATABASES_TX_OFFSET_NAME, - *st_databases_tx_offset_type.as_ref().expect("should be ref"), + ST_INBOUND_MSG_ID_NAME, + *st_inbound_msg_id_type.as_ref().expect("should be ref"), ) .with_type(TableType::System) - .with_unique_constraint(StDatabasesTxOffsetFields::DatabaseIdentity) - .with_index_no_accessor_name(btree(StDatabasesTxOffsetFields::DatabaseIdentity)) - .with_primary_key(StDatabasesTxOffsetFields::DatabaseIdentity); + .with_unique_constraint(StInboundMsgIdFields::DatabaseIdentity) + .with_index_no_accessor_name(btree(StInboundMsgIdFields::DatabaseIdentity)) + .with_primary_key(StInboundMsgIdFields::DatabaseIdentity); + + let st_msg_id_type = builder.add_type::(); + builder + .build_table( + ST_MSG_ID_NAME, + *st_msg_id_type.as_ref().expect("should be ref"), + ) + .with_type(TableType::System) + .with_auto_inc_primary_key(StMsgIdFields::MsgId) + .with_index_no_accessor_name(btree(StMsgIdFields::MsgId)); let result = builder .finish() @@ -715,7 +742,8 @@ fn system_module_def() -> ModuleDef { validate_system_table::(&result, ST_TABLE_ACCESSOR_NAME); validate_system_table::(&result, ST_INDEX_ACCESSOR_NAME); validate_system_table::(&result, ST_COLUMN_ACCESSOR_NAME); - validate_system_table::(&result, ST_DATABASES_TX_OFFSET_NAME); + validate_system_table::(&result, ST_INBOUND_MSG_ID_NAME); + validate_system_table::(&result, ST_MSG_ID_NAME); result } @@ -764,7 +792,8 @@ lazy_static::lazy_static! { m.insert("st_index_accessor_accessor_name_key", ConstraintId(23)); m.insert("st_column_accessor_table_name_col_name_key", ConstraintId(24)); m.insert("st_column_accessor_table_name_accessor_name_key", ConstraintId(25)); - m.insert("st_databases_tx_offset_database_identity_key", ConstraintId(26)); + m.insert("st_inbound_msg_id_database_identity_key", ConstraintId(26)); + m.insert("st_msg_id_msg_id_key", ConstraintId(27)); m }; } @@ -803,7 +832,8 @@ lazy_static::lazy_static! { m.insert("st_index_accessor_accessor_name_idx_btree", IndexId(27)); m.insert("st_column_accessor_table_name_col_name_idx_btree", IndexId(28)); m.insert("st_column_accessor_table_name_accessor_name_idx_btree", IndexId(29)); - m.insert("st_databases_tx_offset_database_identity_idx_btree", IndexId(30)); + m.insert("st_inbound_msg_id_database_identity_idx_btree", IndexId(30)); + m.insert("st_msg_id_msg_id_idx_btree", IndexId(31)); m }; } @@ -820,6 +850,7 @@ lazy_static::lazy_static! { m.insert("st_sequence_sequence_id_seq", SequenceId(5)); m.insert("st_view_view_id_seq", SequenceId(6)); m.insert("st_view_arg_id_seq", SequenceId(7)); + m.insert("st_msg_id_msg_id_seq", SequenceId(8)); m }; } @@ -965,8 +996,12 @@ fn st_column_accessor_schema() -> TableSchema { st_schema(ST_COLUMN_ACCESSOR_NAME, ST_COLUMN_ACCESSOR_ID) } -fn st_databases_tx_offset_schema() -> TableSchema { - st_schema(ST_DATABASES_TX_OFFSET_NAME, ST_DATABASES_TX_OFFSET_ID) +fn st_inbound_msg_id_schema() -> TableSchema { + st_schema(ST_INBOUND_MSG_ID_NAME, ST_INBOUND_MSG_ID_ID) +} + +fn st_msg_id_schema() -> TableSchema { + st_schema(ST_MSG_ID_NAME, ST_MSG_ID_ID) } /// If `table_id` refers to a known system table, return its schema. @@ -997,7 +1032,8 @@ pub(crate) fn system_table_schema(table_id: TableId) -> Option { ST_TABLE_ACCESSOR_ID => Some(st_table_accessor_schema()), ST_INDEX_ACCESSOR_ID => Some(st_index_accessor_schema()), ST_COLUMN_ACCESSOR_ID => Some(st_column_accessor_schema()), - ST_DATABASES_TX_OFFSET_ID => Some(st_databases_tx_offset_schema()), + ST_INBOUND_MSG_ID_ID => Some(st_inbound_msg_id_schema()), + ST_MSG_ID_ID => Some(st_msg_id_schema()), _ => None, } } @@ -1889,15 +1925,52 @@ impl From for ProductValue { } } -/// System Table [ST_DATABASES_TX_OFFST_NAME] +/// System Table [ST_INBOUND_MSG_ID_NAME] +/// Tracks the last message id received from each sender database for deduplication. +/// Also stores the result of the reducer call for returning on subsequent duplicate requests. #[derive(Debug, Clone, PartialEq, Eq, SpacetimeType)] #[sats(crate = spacetimedb_lib)] -pub struct StDatabasesTxOffsetRow { +pub struct StInboundMsgIdRow { pub database_identity: IdentityViaU256, - pub tx_offset: u64, + pub last_msg_id: u64, + /// See [st_inbound_msg_id_result_status] for values. + pub result_status: u8, + /// Error message if result_status is REDUCER_ERROR, empty otherwise. + pub result_payload: String, +} + +impl TryFrom> for StInboundMsgIdRow { + type Error = DatastoreError; + fn try_from(row: RowRef<'_>) -> Result { + read_via_bsatn(row) + } +} + +/// Result status values stored in [ST_INBOUND_MSG_ID_NAME] rows. +pub mod st_inbound_msg_id_result_status { + /// Reducer has been delivered but result not yet received. + pub const NOT_YET_RECEIVED: u8 = 0; + /// Reducer ran and returned Ok(()). + pub const SUCCESS: u8 = 1; + /// Reducer ran and returned Err(...). + pub const REDUCER_ERROR: u8 = 2; +} + +/// System Table [ST_MSG_ID_NAME] +/// Tracks outbound inter-database messages (outbox pattern). +#[derive(Debug, Clone, PartialEq, Eq, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +pub struct StMsgIdRow { + pub msg_id: u64, + pub sender_table_id: u32, + pub target_db_identity: IdentityViaU256, + pub target_reducer: String, + pub args_bsatn: Vec, + /// Optional reducer to call on the sender database with the result of the remote reducer call. + pub on_result_reducer: String, } -impl TryFrom> for StDatabasesTxOffsetRow { +impl TryFrom> for StMsgIdRow { type Error = DatastoreError; fn try_from(row: RowRef<'_>) -> Result { read_via_bsatn(row) diff --git a/crates/datastore/src/traits.rs b/crates/datastore/src/traits.rs index e1b99825ae2..fcf7a8caf6a 100644 --- a/crates/datastore/src/traits.rs +++ b/crates/datastore/src/traits.rs @@ -531,6 +531,8 @@ impl Program { pub struct InsertFlags { /// Is the table a scheduler table? pub is_scheduler_table: bool, + /// Is the table an outbox table? + pub is_outbox_table: bool, } /// Additional information about an update operation. diff --git a/crates/table/src/table.rs b/crates/table/src/table.rs index babcc45c1ee..2414e398df3 100644 --- a/crates/table/src/table.rs +++ b/crates/table/src/table.rs @@ -106,6 +106,11 @@ pub struct Table { /// /// This is an optimization to avoid checking the schema in e.g., `InstanceEnv::{insert, update}`. is_scheduler: bool, + /// Indicates whether this is an outbox table or not. + /// + /// Outbox tables follow the naming convention `__outbox_`. + /// This is an optimization to avoid checking the schema in e.g., `InstanceEnv::insert`. + is_outbox: bool, } type StaticLayoutInTable = Option<(StaticLayout, StaticBsatnValidator)>; @@ -218,6 +223,7 @@ impl MemoryUsage for Table { row_count, blob_store_bytes, is_scheduler, + is_outbox, } = self; inner.heap_usage() + pointer_map.heap_usage() @@ -226,6 +232,7 @@ impl MemoryUsage for Table { + row_count.heap_usage() + blob_store_bytes.heap_usage() + is_scheduler.heap_usage() + + is_outbox.heap_usage() } } @@ -542,6 +549,11 @@ impl Table { self.is_scheduler } + /// Returns whether this is an outbox table (`__outbox_`). + pub fn is_outbox(&self) -> bool { + self.is_outbox + } + /// Check if the `row` conflicts with any unique index on `self`, /// and if there is a conflict, return `Err`. /// @@ -2253,6 +2265,7 @@ impl Table { squashed_offset: SquashedOffset, pointer_map: Option, ) -> Self { + let is_outbox = schema.table_name.starts_with("__outbox_"); Self { inner: TableInner { row_layout, @@ -2261,6 +2274,7 @@ impl Table { pages: Pages::default(), }, is_scheduler: schema.schedule.is_some(), + is_outbox, schema, indexes: BTreeMap::new(), pointer_map, From 811266f7e358638f0504621dac88303e8475b5ba Mon Sep 17 00:00:00 2001 From: Shubham Mishra Date: Fri, 27 Mar 2026 17:35:19 +0530 Subject: [PATCH 04/23] fix st_msg_id syntax --- crates/core/src/host/idc_runtime.rs | 167 +++++++++++++----- crates/core/src/host/instance_env.rs | 37 ++-- crates/core/src/sql/execute.rs | 1 + .../src/locking_tx_datastore/datastore.rs | 1 + .../src/locking_tx_datastore/mut_tx.rs | 19 +- .../src/locking_tx_datastore/state_view.rs | 1 + crates/datastore/src/system_tables.rs | 24 +-- crates/physical-plan/src/plan.rs | 1 + crates/schema/src/def.rs | 6 + crates/schema/src/def/validate/v10.rs | 1 + crates/schema/src/def/validate/v9.rs | 1 + crates/schema/src/schema.rs | 11 ++ 12 files changed, 180 insertions(+), 90 deletions(-) diff --git a/crates/core/src/host/idc_runtime.rs b/crates/core/src/host/idc_runtime.rs index 4a31b376d2e..709619a936c 100644 --- a/crates/core/src/host/idc_runtime.rs +++ b/crates/core/src/host/idc_runtime.rs @@ -1,14 +1,14 @@ /// Inter-Database Communication (IDC) Runtime /// /// Background tokio task that: -/// 1. Loads Pending entries from `st_msg_id` on startup. +/// 1. Loads undelivered entries from `st_msg_id` on startup, resolving delivery data from outbox tables. /// 2. Accepts immediate notifications via an mpsc channel when new outbox rows are inserted. /// 3. Delivers each message in msg_id order via HTTP POST to /// `http://localhost:80/v1/database/{target_db}/call-from-database/{reducer}?sender_identity=&msg_id=` /// 4. On transport errors (network, 5xx, 4xx except 422/402): retries infinitely with exponential /// backoff, blocking only the affected target database (other targets continue unaffected). -/// 5. On reducer errors (HTTP 422) or budget exceeded (HTTP 402): records the result, marks DONE, -/// and calls the configured `on_result_reducer` on the local database (if any). +/// 5. On reducer errors (HTTP 422) or budget exceeded (HTTP 402): calls the configured +/// `on_result_reducer` (read from the outbox table's schema) and deletes the st_msg_id row. /// 6. Enforces sequential delivery per target database: msg N+1 is only delivered after N is done. use crate::db::relational_db::RelationalDB; use crate::host::module_host::WeakModuleHost; @@ -16,7 +16,8 @@ use crate::host::FunctionArgs; use spacetimedb_datastore::execution_context::Workload; use spacetimedb_datastore::system_tables::{StMsgIdRow, ST_MSG_ID_ID}; use spacetimedb_datastore::traits::IsolationLevel; -use spacetimedb_lib::Identity; +use spacetimedb_lib::{AlgebraicValue, Identity}; +use spacetimedb_primitives::{ColId, TableId}; use std::collections::{HashMap, VecDeque}; use std::sync::Arc; use std::time::{Duration, Instant}; @@ -77,9 +78,22 @@ impl IdcRuntime { } } +/// All data needed to deliver a single outbound message, resolved from the outbox table. +#[derive(Clone)] +struct PendingMessage { + msg_id: u64, + outbox_table_id: TableId, + row_id: u64, + target_db_identity: Identity, + target_reducer: String, + args_bsatn: Vec, + /// From the outbox table's `TableSchema::on_result_reducer`. + on_result_reducer: Option, +} + /// Per-target-database delivery state. struct TargetState { - queue: VecDeque, + queue: VecDeque, /// When `Some`, this target is in backoff and should not be retried until this instant. blocked_until: Option, /// Current backoff duration for this target (doubles on each transport error). @@ -149,16 +163,16 @@ async fn run_idc_loop( if !state.is_ready() { continue; } - let Some(row) = state.queue.front().cloned() else { + let Some(msg) = state.queue.front().cloned() else { continue; }; - let outcome = attempt_delivery(&client, &config, &row).await; + let outcome = attempt_delivery(&client, &config, &msg).await; match outcome { DeliveryOutcome::TransportError(reason) => { log::warn!( "idc_runtime: transport error delivering msg_id={} to {}: {reason}", - row.msg_id, - hex::encode(Identity::from(row.target_db_identity).to_byte_array()), + msg.msg_id, + hex::encode(msg.target_db_identity.to_byte_array()), ); state.record_transport_error(); // Do NOT pop the front — keep retrying this message for this target. @@ -168,7 +182,7 @@ async fn run_idc_loop( state.record_success(); any_delivered = true; let (result_status, result_payload) = outcome_to_result(&outcome); - finalize_message(&db, &module_host, &row, result_status, result_payload).await; + finalize_message(&db, &module_host, &msg, result_status, result_payload).await; } } } @@ -214,19 +228,19 @@ fn outcome_to_result(outcome: &DeliveryOutcome) -> (u8, String) { async fn finalize_message( db: &RelationalDB, module_host: &WeakModuleHost, - row: &StMsgIdRow, + msg: &PendingMessage, _result_status: u8, result_payload: String, ) { // Call the on_result reducer if configured. - if !row.on_result_reducer.is_empty() { + if let Some(on_result_reducer) = &msg.on_result_reducer { let Some(host) = module_host.upgrade() else { log::warn!( "idc_runtime: module host gone, cannot call on_result reducer '{}' for msg_id={}", - row.on_result_reducer, - row.msg_id, + on_result_reducer, + msg.msg_id, ); - delete_message(db, row.msg_id); + delete_message(db, msg.msg_id); return; }; @@ -237,9 +251,9 @@ async fn finalize_message( Err(e) => { log::error!( "idc_runtime: failed to encode on_result args for msg_id={}: {e}", - row.msg_id + msg.msg_id ); - delete_message(db, row.msg_id); + delete_message(db, msg.msg_id); return; } }; @@ -252,7 +266,7 @@ async fn finalize_message( None, // no client sender None, // no request_id None, // no timer - &row.on_result_reducer, + on_result_reducer, FunctionArgs::Bsatn(bytes::Bytes::from(args_bytes)), ) .await; @@ -261,50 +275,124 @@ async fn finalize_message( Ok(_) => { log::debug!( "idc_runtime: on_result reducer '{}' called for msg_id={}", - row.on_result_reducer, - row.msg_id, + on_result_reducer, + msg.msg_id, ); } Err(e) => { log::error!( "idc_runtime: on_result reducer '{}' failed for msg_id={}: {e:?}", - row.on_result_reducer, - row.msg_id, + on_result_reducer, + msg.msg_id, ); } } } // Delete the row regardless of whether on_result succeeded or failed. - delete_message(db, row.msg_id); + delete_message(db, msg.msg_id); } -/// Load all messages from ST_MSG_ID into the per-target queues. +/// Load all messages from ST_MSG_ID into the per-target queues, resolving delivery data +/// from the corresponding outbox table rows. /// -/// A row's presence in the table means it has not yet been processed. -/// Messages that are already in a target's queue (by msg_id) are not re-added. +/// A row's presence in ST_MSG_ID means it has not yet been processed. +/// Messages already in a target's queue (by msg_id) are not re-added. fn load_pending_into_targets(db: &RelationalDB, targets: &mut HashMap) { let tx = db.begin_tx(Workload::Internal); - let rows: Vec = db + + let st_msg_id_rows: Vec = db .iter(&tx, ST_MSG_ID_ID) .map(|iter| iter.filter_map(|row_ref| StMsgIdRow::try_from(row_ref).ok()).collect()) .unwrap_or_else(|e| { - log::error!("idc_runtime: failed to read pending messages: {e}"); + log::error!("idc_runtime: failed to read st_msg_id: {e}"); Vec::new() }); + + let mut pending: Vec = Vec::with_capacity(st_msg_id_rows.len()); + + for st_row in st_msg_id_rows { + let outbox_table_id = TableId(st_row.outbox_table_id); + + // Read the outbox table schema for reducer name and on_result_reducer. + let schema = match db.schema_for_table(&tx, outbox_table_id) { + Ok(s) => s, + Err(e) => { + log::error!( + "idc_runtime: cannot find schema for outbox table {:?} (msg_id={}): {e}", + outbox_table_id, + st_row.msg_id, + ); + continue; + } + }; + + let table_name = schema.table_name.to_string(); + let target_reducer = table_name + .strip_prefix("__outbox_") + .unwrap_or(&table_name) + .to_string(); + let on_result_reducer = schema.on_result_reducer.clone(); + + // Look up the outbox row by its auto-inc PK (col 0) to get target identity and args. + let outbox_row = db + .iter_by_col_eq(&tx, outbox_table_id, ColId(0), &AlgebraicValue::U64(st_row.row_id)) + .ok() + .and_then(|mut iter| iter.next()); + + let Some(outbox_row_ref) = outbox_row else { + log::error!( + "idc_runtime: outbox row not found in table {:?} for row_id={} (msg_id={})", + outbox_table_id, + st_row.row_id, + st_row.msg_id, + ); + continue; + }; + + let pv = outbox_row_ref.to_product_value(); + + // Col 1: target_db_identity (Identity stored as U256). + let target_db_identity = match pv.elements.get(1) { + Some(AlgebraicValue::U256(u)) => Identity::from_u256(**u), + other => { + log::error!( + "idc_runtime: outbox row col 1 expected U256 (Identity), got {other:?} (msg_id={})", + st_row.msg_id, + ); + continue; + } + }; + + // Cols 2+: args for the remote reducer. + let args_bsatn = pv.elements[2..].iter().fold(Vec::new(), |mut acc, elem| { + spacetimedb_sats::bsatn::to_writer(&mut acc, elem) + .expect("writing outbox row args to BSATN should never fail"); + acc + }); + + pending.push(PendingMessage { + msg_id: st_row.msg_id, + outbox_table_id, + row_id: st_row.row_id, + target_db_identity, + target_reducer, + args_bsatn, + on_result_reducer, + }); + } + drop(tx); // Sort by msg_id ascending so delivery order is preserved. - let mut sorted = rows; - sorted.sort_by_key(|r| r.msg_id); + pending.sort_by_key(|m| m.msg_id); - for row in sorted { - let target_id = Identity::from(row.target_db_identity); - let state = targets.entry(target_id).or_insert_with(TargetState::new); + for msg in pending { + let state = targets.entry(msg.target_db_identity).or_insert_with(TargetState::new); // Only add if not already in the queue (avoid duplicates after reload). - let already_queued = state.queue.iter().any(|r| r.msg_id == row.msg_id); + let already_queued = state.queue.iter().any(|m| m.msg_id == msg.msg_id); if !already_queued { - state.queue.push_back(row); + state.queue.push_back(msg); } } } @@ -313,21 +401,20 @@ fn load_pending_into_targets(db: &RelationalDB, targets: &mut HashMap DeliveryOutcome { - let target_identity = Identity::from(row.target_db_identity); - let target_db_hex = hex::encode(target_identity.to_byte_array()); + let target_db_hex = hex::encode(msg.target_db_identity.to_byte_array()); let sender_hex = hex::encode(config.sender_identity.to_byte_array()); let url = format!( "http://localhost:{IDC_HTTP_PORT}/v1/database/{target_db_hex}/call-from-database/{}?sender_identity={sender_hex}&msg_id={}", - row.target_reducer, row.msg_id, + msg.target_reducer, msg.msg_id, ); let result = client .post(&url) .header("Content-Type", "application/octet-stream") - .body(row.args_bsatn.clone()) + .body(msg.args_bsatn.clone()) .send() .await; diff --git a/crates/core/src/host/instance_env.rs b/crates/core/src/host/instance_env.rs index 1f1956fd221..24a4da4cab8 100644 --- a/crates/core/src/host/instance_env.rs +++ b/crates/core/src/host/instance_env.rs @@ -435,10 +435,13 @@ impl InstanceEnv { /// Enqueue an outbox row into ST_MSG_ID atomically within the current transaction, /// and notify the IDC runtime so it delivers without waiting for the next poll cycle. /// - /// Outbox tables follow the naming convention `__outbox_`. - /// The first column (col 0) must hold the target database Identity (BSATN-encoded as U256). - /// The remaining bytes of the row's BSATN encoding are passed as `args_bsatn` to the - /// remote reducer. + /// Outbox tables follow the naming convention `__outbox_` and have: + /// - Col 0: auto-inc primary key (u64) — stored as `row_id` in ST_MSG_ID. + /// - Col 1: target database Identity (stored as U256). + /// - Remaining cols: args for the remote reducer. + /// + /// The `on_result_reducer` and delivery data are resolved at delivery time from the + /// outbox table's schema and row, so ST_MSG_ID only stores the minimal reference. #[cold] #[inline(never)] fn enqueue_outbox_row( @@ -450,34 +453,22 @@ impl InstanceEnv { ) -> Result<(), NodesError> { use spacetimedb_datastore::locking_tx_datastore::state_view::StateView as _; - // Get table name to extract reducer name. - let schema = tx.schema_for_table(table_id).map_err(DBError::from)?; - let table_name = schema.table_name.to_string(); - let reducer_name = table_name.strip_prefix("__outbox_").unwrap_or(&table_name).to_string(); - let row_ref = tx.get(table_id, row_ptr).map_err(DBError::from)?.unwrap(); - let pv = row_ref.to_product_value(); - // Col 0 must be the target database Identity, stored as AlgebraicValue::U256. - let target_db_identity = match pv.elements.first() { - Some(AlgebraicValue::U256(u)) => Identity::from_u256(**u), + // Col 0 is the auto-inc primary key — this is the row_id we store in ST_MSG_ID. + let row_id = match pv.elements.first() { + Some(AlgebraicValue::U64(id)) => *id, other => { + let schema = tx.schema_for_table(table_id).map_err(DBError::from)?; return Err(NodesError::Internal(Box::new(DBError::Other(anyhow::anyhow!( - "outbox table {table_name}: expected col 0 to be U256 (Identity), got {other:?}" + "outbox table {}: expected col 0 to be U64 (auto-inc PK), got {other:?}", + schema.table_name ))))); } }; - // Remaining elements are the args for the remote reducer. - let args_bsatn = pv.elements[1..].iter().fold(Vec::new(), |mut acc, elem| { - bsatn::to_writer(&mut acc, elem).expect("writing outbox row args to BSATN should never fail"); - acc - }); - - // TODO: populate on_result_reducer from TableDef once that field is added. - tx.insert_st_msg_id(table_id.0, target_db_identity, reducer_name, args_bsatn, String::new()) - .map_err(DBError::from)?; + tx.insert_st_msg_id(table_id.0, row_id).map_err(DBError::from)?; // Wake the IDC runtime immediately so it doesn't wait for the next poll cycle. let _ = self.idc_sender.send(()); diff --git a/crates/core/src/sql/execute.rs b/crates/core/src/sql/execute.rs index 476a73a9a45..47eb67a02f2 100644 --- a/crates/core/src/sql/execute.rs +++ b/crates/core/src/sql/execute.rs @@ -340,6 +340,7 @@ pub(crate) mod tests { None, false, None, + None, ), )?; let schema = db.schema_for_table_mut(tx, table_id)?; diff --git a/crates/datastore/src/locking_tx_datastore/datastore.rs b/crates/datastore/src/locking_tx_datastore/datastore.rs index 53327b55c8b..bc2a8bc488f 100644 --- a/crates/datastore/src/locking_tx_datastore/datastore.rs +++ b/crates/datastore/src/locking_tx_datastore/datastore.rs @@ -1627,6 +1627,7 @@ mod tests { pk, false, None, + None, ) } diff --git a/crates/datastore/src/locking_tx_datastore/mut_tx.rs b/crates/datastore/src/locking_tx_datastore/mut_tx.rs index 0d339161819..de5c3c917cc 100644 --- a/crates/datastore/src/locking_tx_datastore/mut_tx.rs +++ b/crates/datastore/src/locking_tx_datastore/mut_tx.rs @@ -2740,27 +2740,16 @@ impl MutTxId { } /// Insert a new outbound inter-database message into `st_msg_id`. - /// Returns the auto-incremented msg_id assigned to this message. - pub fn insert_st_msg_id( - &mut self, - sender_table_id: u32, - target_db_identity: Identity, - target_reducer: String, - args_bsatn: Vec, - on_result_reducer: String, - ) -> Result<()> { + pub fn insert_st_msg_id(&mut self, outbox_table_id: u32, row_id: u64) -> Result<()> { let row = StMsgIdRow { msg_id: 0, // auto-incremented by the sequence - sender_table_id, - target_db_identity: target_db_identity.into(), - target_reducer, - args_bsatn, - on_result_reducer, + outbox_table_id, + row_id, }; self.insert_via_serialize_bsatn(ST_MSG_ID_ID, &row) .map(|_| ()) .inspect_err(|e| { - log::error!("insert_st_msg_id: failed to insert msg for {target_db_identity}: {e}"); + log::error!("insert_st_msg_id: failed to insert msg for outbox_table_id={outbox_table_id}: {e}"); }) } diff --git a/crates/datastore/src/locking_tx_datastore/state_view.rs b/crates/datastore/src/locking_tx_datastore/state_view.rs index 448217dc713..42d7937eb90 100644 --- a/crates/datastore/src/locking_tx_datastore/state_view.rs +++ b/crates/datastore/src/locking_tx_datastore/state_view.rs @@ -268,6 +268,7 @@ pub trait StateView { table_primary_key, is_event, table_alias, + None, )) } diff --git a/crates/datastore/src/system_tables.rs b/crates/datastore/src/system_tables.rs index a037100337d..56b2851b4e2 100644 --- a/crates/datastore/src/system_tables.rs +++ b/crates/datastore/src/system_tables.rs @@ -471,11 +471,8 @@ st_fields_enum!(enum StInboundMsgIdFields { st_fields_enum!(enum StMsgIdFields { "msg_id", MsgId = 0, - "sender_table_id", SenderTableId = 1, - "target_db_identity", TargetDbIdentity = 2, - "target_reducer", TargetReducer = 3, - "args_bsatn", ArgsBsatn = 4, - "on_result_reducer", OnResultReducer = 5, + "outbox_table_id", OutboxTableId = 1, + "row_id", RowId = 2, }); /// Helper method to check that a system table has the correct fields. @@ -1957,17 +1954,20 @@ pub mod st_inbound_msg_id_result_status { } /// System Table [ST_MSG_ID_NAME] -/// Tracks outbound inter-database messages (outbox pattern). +/// Tracks undelivered outbound inter-database messages (outbox pattern). +/// A row's presence means the message has not yet been fully processed. +/// Once delivery and the on_result callback complete, the row is deleted. +/// +/// Delivery data (target identity, reducer name, args) is read from the outbox table row. +/// The on_result_reducer is read from the outbox table's schema (TableSchema::on_result_reducer). #[derive(Debug, Clone, PartialEq, Eq, SpacetimeType)] #[sats(crate = spacetimedb_lib)] pub struct StMsgIdRow { pub msg_id: u64, - pub sender_table_id: u32, - pub target_db_identity: IdentityViaU256, - pub target_reducer: String, - pub args_bsatn: Vec, - /// Optional reducer to call on the sender database with the result of the remote reducer call. - pub on_result_reducer: String, + /// The TableId of the `__outbox_` table that generated this message. + pub outbox_table_id: u32, + /// The auto-inc primary key (col 0) of the row in the outbox table. + pub row_id: u64, } impl TryFrom> for StMsgIdRow { diff --git a/crates/physical-plan/src/plan.rs b/crates/physical-plan/src/plan.rs index 0db3e7910ce..53487aa457b 100644 --- a/crates/physical-plan/src/plan.rs +++ b/crates/physical-plan/src/plan.rs @@ -1547,6 +1547,7 @@ mod tests { primary_key.map(ColId::from), false, None, + None, ))) } diff --git a/crates/schema/src/def.rs b/crates/schema/src/def.rs index 89c201e3f85..ae202e4a7b4 100644 --- a/crates/schema/src/def.rs +++ b/crates/schema/src/def.rs @@ -692,6 +692,10 @@ pub struct TableDef { /// Event tables persist to the commitlog but are not merged into committed state. /// Their rows are only visible to V2 subscribers in the transaction that inserted them. pub is_event: bool, + + /// For outbox tables (`__outbox_`): the name of the local reducer to call + /// with the result of the remote reducer invocation. Empty string means no callback. + pub on_result_reducer: Option, } impl TableDef { @@ -751,6 +755,7 @@ impl From for RawTableDefV10 { table_access, is_event, accessor_name, + on_result_reducer: _, // not represented in V10 wire format } = val; RawTableDefV10 { @@ -793,6 +798,7 @@ impl From for TableDef { table_access: if is_public { Public } else { Private }, is_event: false, accessor_name, + on_result_reducer: None, } } } diff --git a/crates/schema/src/def/validate/v10.rs b/crates/schema/src/def/validate/v10.rs index 7ebbaae06d4..7c03606c738 100644 --- a/crates/schema/src/def/validate/v10.rs +++ b/crates/schema/src/def/validate/v10.rs @@ -510,6 +510,7 @@ impl<'a> ModuleValidatorV10<'a> { table_access, is_event, accessor_name: identifier(raw_table_name)?, + on_result_reducer: None, // TODO: parse from raw table def once wire format supports it }) } diff --git a/crates/schema/src/def/validate/v9.rs b/crates/schema/src/def/validate/v9.rs index d040435afe5..2e04f9a013c 100644 --- a/crates/schema/src/def/validate/v9.rs +++ b/crates/schema/src/def/validate/v9.rs @@ -324,6 +324,7 @@ impl ModuleValidatorV9<'_> { table_access, is_event: false, // V9 does not support event tables accessor_name: name, + on_result_reducer: None, // V9 does not support on_result_reducer }) } diff --git a/crates/schema/src/schema.rs b/crates/schema/src/schema.rs index e62b9dc6963..26920f04fdc 100644 --- a/crates/schema/src/schema.rs +++ b/crates/schema/src/schema.rs @@ -201,6 +201,10 @@ pub struct TableSchema { /// Whether this is an event table. pub is_event: bool, + /// For outbox tables (`__outbox_`): the name of the local reducer to call + /// with the result of the remote reducer invocation. `None` means no callback. + pub on_result_reducer: Option, + /// Cache for `row_type_for_table` in the data store. pub row_type: ProductType, } @@ -227,6 +231,7 @@ impl TableSchema { primary_key: Option, is_event: bool, alias: Option, + on_result_reducer: Option, ) -> Self { Self { row_type: columns_to_row_type(&columns), @@ -243,6 +248,7 @@ impl TableSchema { primary_key, is_event, alias, + on_result_reducer, } } @@ -281,6 +287,7 @@ impl TableSchema { None, false, None, + None, ) } @@ -803,6 +810,7 @@ impl TableSchema { view_primary_key, false, None, + None, ) } @@ -974,6 +982,7 @@ impl TableSchema { if *is_anonymous { view_primary_key } else { None }, false, Some(accessor_name.clone()), + None, ) } } @@ -1005,6 +1014,7 @@ impl Schema for TableSchema { table_access, is_event, accessor_name, + on_result_reducer, .. } = def; @@ -1045,6 +1055,7 @@ impl Schema for TableSchema { *primary_key, *is_event, Some(accessor_name.clone()), + on_result_reducer.clone(), ) } From f036abde5e90f041e78efe0be3bfc3a5f73b9df7 Mon Sep 17 00:00:00 2001 From: Shubham Mishra Date: Fri, 27 Mar 2026 19:08:17 +0530 Subject: [PATCH 05/23] hacky macro --- crates/bindings-macro/src/lib.rs | 2 + crates/bindings-macro/src/table.rs | 94 +++++++++++++++++++++++++++ crates/bindings/src/rt.rs | 19 ++++++ crates/bindings/src/table.rs | 9 +++ crates/lib/src/db/raw_def/v10.rs | 76 ++++++++++++++++++++++ crates/schema/src/def.rs | 46 ++++++++++--- crates/schema/src/def/validate/v10.rs | 81 ++++++++++++++++++++++- crates/schema/src/def/validate/v9.rs | 2 +- crates/schema/src/error.rs | 20 ++++++ crates/schema/src/schema.rs | 9 ++- 10 files changed, 346 insertions(+), 12 deletions(-) diff --git a/crates/bindings-macro/src/lib.rs b/crates/bindings-macro/src/lib.rs index 8fa9705ca0e..fdea5f64b86 100644 --- a/crates/bindings-macro/src/lib.rs +++ b/crates/bindings-macro/src/lib.rs @@ -132,6 +132,8 @@ mod sym { symbol!(update); symbol!(default); symbol!(event); + symbol!(outbox); + symbol!(on_result); symbol!(u8); symbol!(i8); diff --git a/crates/bindings-macro/src/table.rs b/crates/bindings-macro/src/table.rs index 1cbba4c5ca1..bc0a9177334 100644 --- a/crates/bindings-macro/src/table.rs +++ b/crates/bindings-macro/src/table.rs @@ -22,6 +22,16 @@ pub(crate) struct TableArgs { accessor: Ident, indices: Vec, event: Option, + outbox: Option, +} + +/// Parsed from `outbox(remote_reducer_fn)` optionally followed by `on_result(local_reducer_fn)`. +struct OutboxArg { + span: Span, + /// Path to the remote-side reducer function (used only for its name via `FnInfo::NAME`). + remote_reducer: Path, + /// Path to the local `on_result` reducer, if any. + on_result_reducer: Option, } enum TableAccess { @@ -82,6 +92,7 @@ impl TableArgs { let mut name: Option = None; let mut indices = Vec::new(); let mut event = None; + let mut outbox: Option = None; syn::meta::parser(|meta| { match_meta!(match meta { sym::public => { @@ -149,6 +160,30 @@ If you're migrating from SpacetimeDB 1.*, replace `name = {sym}` with `accessor check_duplicate(&event, &meta)?; event = Some(meta.path.span()); } + sym::outbox => { + check_duplicate_msg(&outbox, &meta, "already specified outbox")?; + outbox = Some(OutboxArg::parse_meta(meta)?); + } + sym::on_result => { + // `on_result` must be specified alongside `outbox`. + // We parse it here and attach it to the outbox arg below. + let span = meta.path.span(); + let on_result_path = OutboxArg::parse_single_path_meta(meta)?; + match &mut outbox { + Some(ob) => { + if ob.on_result_reducer.is_some() { + return Err(syn::Error::new(span, "already specified on_result")); + } + ob.on_result_reducer = Some(on_result_path); + } + None => { + return Err(syn::Error::new( + span, + "on_result requires outbox to be specified first: `outbox(remote_reducer), on_result(local_reducer)`", + )) + } + } + } }); Ok(()) }) @@ -188,6 +223,7 @@ If you're migrating from SpacetimeDB 1.*, replace `name = {name_str_value:?}` wi indices, name, event, + outbox, }) } } @@ -231,6 +267,64 @@ impl ScheduledArg { } } +impl OutboxArg { + /// Parse `outbox(remote_reducer_path)`. + /// + /// `on_result` is parsed separately via `parse_single_path_meta` and attached afterwards. + fn parse_meta(meta: ParseNestedMeta) -> syn::Result { + let span = meta.path.span(); + let mut remote_reducer: Option = None; + + meta.parse_nested_meta(|meta| { + if meta.input.peek(syn::Token![=]) || meta.input.peek(syn::token::Paren) { + Err(meta.error("outbox takes a single function path, e.g. `outbox(my_remote_reducer)`")) + } else { + check_duplicate_msg( + &remote_reducer, + &meta, + "can only specify one remote reducer for outbox", + )?; + remote_reducer = Some(meta.path); + Ok(()) + } + })?; + + let remote_reducer = remote_reducer.ok_or_else(|| { + syn::Error::new(span, "outbox requires a remote reducer: `outbox(my_remote_reducer)`") + })?; + + Ok(Self { + span, + remote_reducer, + on_result_reducer: None, + }) + } + + /// Parse `on_result(local_reducer_path)` and return the path. + fn parse_single_path_meta(meta: ParseNestedMeta) -> syn::Result { + let span = meta.path.span(); + let mut result: Option = None; + + meta.parse_nested_meta(|meta| { + if meta.input.peek(syn::Token![=]) || meta.input.peek(syn::token::Paren) { + Err(meta.error("on_result takes a single function path, e.g. `on_result(my_local_reducer)`")) + } else { + check_duplicate_msg( + &result, + &meta, + "can only specify one on_result reducer", + )?; + result = Some(meta.path); + Ok(()) + } + })?; + + result.ok_or_else(|| { + syn::Error::new(span, "on_result requires a local reducer: `on_result(my_local_reducer)`") + }) + } +} + impl IndexArg { fn parse_meta(meta: ParseNestedMeta) -> syn::Result { let mut accessor = None; diff --git a/crates/bindings/src/rt.rs b/crates/bindings/src/rt.rs index d6d55eba5f4..906d9231bdf 100644 --- a/crates/bindings/src/rt.rs +++ b/crates/bindings/src/rt.rs @@ -533,6 +533,17 @@ impl TableColumn for T {} /// Assert that the primary_key column of a scheduled table is a u64. pub const fn assert_scheduled_table_primary_key() {} +/// Verify at compile time that a function has the correct signature for an outbox `on_result` reducer. +/// +/// The reducer must accept `(OutboxRow, Result<(), String>)` as its user-supplied arguments: +/// `fn on_result(ctx: &ReducerContext, request: OutboxRow, result: Result<(), String>)` +pub const fn outbox_typecheck<'de, OutboxRow>(_x: impl Reducer<'de, (OutboxRow, Result<(), String>)>) +where + OutboxRow: spacetimedb_lib::SpacetimeType + Serialize + Deserialize<'de>, +{ + core::mem::forget(_x); +} + mod sealed { pub trait Sealed {} } @@ -763,6 +774,14 @@ pub fn register_table() { table.finish(); + if let Some(outbox) = T::OUTBOX { + module.inner.add_outbox( + T::TABLE_NAME, + outbox.remote_reducer_name, + outbox.on_result_reducer_name, + ); + } + module.inner.add_explicit_names(T::explicit_names()); }) } diff --git a/crates/bindings/src/table.rs b/crates/bindings/src/table.rs index a4c25c039e2..a759e702160 100644 --- a/crates/bindings/src/table.rs +++ b/crates/bindings/src/table.rs @@ -134,6 +134,7 @@ pub trait TableInternal: Sized { const SEQUENCES: &'static [u16]; const SCHEDULE: Option> = None; const IS_EVENT: bool = false; + const OUTBOX: Option> = None; /// Returns the ID of this table. fn table_id() -> TableId; @@ -161,6 +162,14 @@ pub struct ScheduleDesc<'a> { pub scheduled_at_column: u16, } +/// Describes the outbox configuration of a table, for inter-database communication. +pub struct OutboxDesc<'a> { + /// The name of the remote reducer to invoke on the target database. + pub remote_reducer_name: &'a str, + /// The local reducer to call with the delivery result, if any. + pub on_result_reducer_name: Option<&'a str>, +} + #[derive(Debug, Clone)] pub struct ColumnDefault { pub col_id: u16, diff --git a/crates/lib/src/db/raw_def/v10.rs b/crates/lib/src/db/raw_def/v10.rs index a801ea286be..1e615c6b0b9 100644 --- a/crates/lib/src/db/raw_def/v10.rs +++ b/crates/lib/src/db/raw_def/v10.rs @@ -89,6 +89,13 @@ pub enum RawModuleDefV10Section { /// Names provided explicitly by the user that do not follow from the case conversion policy. ExplicitNames(ExplicitNames), + + /// Outbox table definitions. + /// + /// Each entry marks a table as an outbox table for inter-database communication, + /// specifying the remote reducer to call and optionally a local callback reducer. + /// New variant — old modules simply omit this section; old servers skip it. + Outboxes(Vec), } #[derive(Debug, Clone, Copy, Default, SpacetimeType)] @@ -264,6 +271,31 @@ pub struct RawColumnDefaultValueV10 { pub value: Box<[u8]>, } +/// Marks a table as an outbox table for inter-database communication. +/// +/// The table must have: +/// - Col 0: `u64` with `#[primary_key] #[auto_inc]` — the row ID stored in `st_msg_id`. +/// - Col 1: `Identity` (encoded as U256) — the target database identity. +/// - Remaining cols: arguments forwarded verbatim to the remote reducer. +/// +/// The `remote_reducer` is the name of the reducer to call on the target database. +/// If `on_result_reducer` is set, that local reducer is called when delivery completes, +/// with signature `fn on_result(ctx: &ReducerContext, request: OutboxRow, result: Result<(), String>)`. +#[derive(Debug, Clone, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "test", derive(PartialEq, Eq, PartialOrd, Ord))] +pub struct RawOutboxDefV10 { + /// The `source_name` of the outbox table (as given in `accessor = ...`). + pub table_name: RawIdentifier, + + /// The name of the reducer to call on the target database. + pub remote_reducer: RawIdentifier, + + /// The name of the local reducer to call with the delivery result. + /// If `None`, no callback is made after delivery. + pub on_result_reducer: Option, +} + /// A reducer definition. #[derive(Debug, Clone, SpacetimeType)] #[sats(crate = crate)] @@ -584,6 +616,14 @@ impl RawModuleDefV10 { .expect("Tables section must exist for tests") } + /// Get the outboxes section, if present. + pub fn outboxes(&self) -> Option<&Vec> { + self.sections.iter().find_map(|s| match s { + RawModuleDefV10Section::Outboxes(outboxes) => Some(outboxes), + _ => None, + }) + } + // Get the row-level security section, if present. pub fn row_level_security(&self) -> Option<&Vec> { self.sections.iter().find_map(|s| match s { @@ -785,6 +825,24 @@ impl RawModuleDefV10Builder { } } + /// Get mutable access to the outboxes section, creating it if missing. + fn outboxes_mut(&mut self) -> &mut Vec { + let idx = self + .module + .sections + .iter() + .position(|s| matches!(s, RawModuleDefV10Section::Outboxes(_))) + .unwrap_or_else(|| { + self.module.sections.push(RawModuleDefV10Section::Outboxes(Vec::new())); + self.module.sections.len() - 1 + }); + + match &mut self.module.sections[idx] { + RawModuleDefV10Section::Outboxes(outboxes) => outboxes, + _ => unreachable!("Just ensured Outboxes section exists"), + } + } + /// Get mutable access to the case conversion policy, creating it if missing. fn explicit_names_mut(&mut self) -> &mut ExplicitNames { let idx = self @@ -1040,6 +1098,24 @@ impl RawModuleDefV10Builder { }); } + /// Register an outbox table for inter-database communication. + /// + /// `table_name` is the `source_name` of the table (i.e. `accessor =` value). + /// `remote_reducer` is the reducer to call on the target database. + /// `on_result_reducer` is an optional local reducer called with the delivery result. + pub fn add_outbox( + &mut self, + table_name: impl Into, + remote_reducer: impl Into, + on_result_reducer: Option>, + ) { + self.outboxes_mut().push(RawOutboxDefV10 { + table_name: table_name.into(), + remote_reducer: remote_reducer.into(), + on_result_reducer: on_result_reducer.map(Into::into), + }); + } + /// Add a row-level security policy to the module. /// /// The `sql` expression should be a valid SQL expression that will be used to filter rows. diff --git a/crates/schema/src/def.rs b/crates/schema/src/def.rs index ae202e4a7b4..7e89c048518 100644 --- a/crates/schema/src/def.rs +++ b/crates/schema/src/def.rs @@ -33,8 +33,8 @@ use spacetimedb_data_structures::map::{Equivalent, HashMap}; use spacetimedb_lib::db::raw_def; use spacetimedb_lib::db::raw_def::v10::{ ExplicitNames, RawConstraintDefV10, RawIndexDefV10, RawLifeCycleReducerDefV10, RawModuleDefV10, - RawModuleDefV10Section, RawProcedureDefV10, RawReducerDefV10, RawRowLevelSecurityDefV10, RawScheduleDefV10, - RawScopedTypeNameV10, RawSequenceDefV10, RawTableDefV10, RawTypeDefV10, RawViewDefV10, + RawModuleDefV10Section, RawOutboxDefV10, RawProcedureDefV10, RawReducerDefV10, RawRowLevelSecurityDefV10, + RawScheduleDefV10, RawScopedTypeNameV10, RawSequenceDefV10, RawTableDefV10, RawTypeDefV10, RawViewDefV10, }; use spacetimedb_lib::db::raw_def::v9::{ Lifecycle, RawColumnDefaultValueV9, RawConstraintDataV9, RawConstraintDefV9, RawIndexAlgorithm, RawIndexDefV9, @@ -518,9 +518,10 @@ impl From for RawModuleDefV10 { sections.push(RawModuleDefV10Section::Types(raw_types)); } - // Collect schedules from tables (V10 stores them in a separate section). + // Collect schedules and outboxes from tables (V10 stores them in separate sections). // Also collect ExplicitNames for tables: accessor_name → source_name, name → canonical_name. let mut schedules = Vec::new(); + let mut outboxes: Vec = Vec::new(); let raw_tables: Vec = tables .into_values() .map(|td| { @@ -537,6 +538,13 @@ impl From for RawModuleDefV10 { function_name: sched.function_name.into(), }); } + if let Some(ref outbox) = td.outbox { + outboxes.push(RawOutboxDefV10 { + table_name: td.accessor_name.clone().into(), + remote_reducer: outbox.remote_reducer.clone().into(), + on_result_reducer: outbox.on_result_reducer.clone().map(Into::into), + }); + } td.into() }) .collect(); @@ -593,6 +601,10 @@ impl From for RawModuleDefV10 { sections.push(RawModuleDefV10Section::Schedules(schedules)); } + if !outboxes.is_empty() { + sections.push(RawModuleDefV10Section::Outboxes(outboxes)); + } + if !raw_lifecycle.is_empty() { sections.push(RawModuleDefV10Section::LifeCycleReducers(raw_lifecycle)); } @@ -693,9 +705,26 @@ pub struct TableDef { /// Their rows are only visible to V2 subscribers in the transaction that inserted them. pub is_event: bool, - /// For outbox tables (`__outbox_`): the name of the local reducer to call - /// with the result of the remote reducer invocation. Empty string means no callback. - pub on_result_reducer: Option, + /// Outbox configuration for inter-database communication. + /// `None` unless this table was declared with `#[spacetimedb::table(outbox(...))]`. + pub outbox: Option, +} + +/// Configuration for an outbox table used in inter-database communication. +/// +/// An outbox table must have: +/// - Col 0: `u64` with `#[primary_key] #[auto_inc]` — row ID tracked in `st_msg_id`. +/// - Col 1: `Identity` — target database identity. +/// - Remaining cols: arguments forwarded to `remote_reducer` on the target database. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OutboxDef { + /// The name of the reducer to call on the target database. + pub remote_reducer: Identifier, + + /// The local reducer called with the delivery result, if any. + /// + /// Signature: `fn on_result(ctx: &ReducerContext, request: OutboxRow, result: Result<(), String>)`. + pub on_result_reducer: Option, } impl TableDef { @@ -723,6 +752,7 @@ impl From for RawTableDefV9 { table_type, table_access, is_event: _, // V9 does not support event tables; ignore when converting back + outbox: _, // V9 does not support outbox tables; ignore when converting back .. } = val; @@ -755,7 +785,7 @@ impl From for RawTableDefV10 { table_access, is_event, accessor_name, - on_result_reducer: _, // not represented in V10 wire format + outbox: _, // V10 stores outbox definitions in a separate Outboxes section; handled in From. } = val; RawTableDefV10 { @@ -798,7 +828,7 @@ impl From for TableDef { table_access: if is_public { Public } else { Private }, is_event: false, accessor_name, - on_result_reducer: None, + outbox: None, } } } diff --git a/crates/schema/src/def/validate/v10.rs b/crates/schema/src/def/validate/v10.rs index 7c03606c738..b54d3274ffb 100644 --- a/crates/schema/src/def/validate/v10.rs +++ b/crates/schema/src/def/validate/v10.rs @@ -188,6 +188,12 @@ pub fn validate(def: RawModuleDefV10) -> Result { }) .unwrap_or_else(|| Ok(Vec::new())); + // Collect raw outbox definitions from the Outboxes section for post-validation attachment. + let raw_outboxes: Vec = def + .outboxes() + .map(|v| v.to_vec()) + .unwrap_or_default(); + // Validate lifecycle reducers - they reference reducers by name let lifecycle_validations = reducers .as_ref() @@ -242,6 +248,9 @@ pub fn validate(def: RawModuleDefV10) -> Result { // Attach schedules to their respective tables attach_schedules_to_tables(&mut tables, schedules)?; + // Attach outbox definitions to their respective tables + attach_outboxes_to_tables(&mut tables, raw_outboxes)?; + check_scheduled_functions_exist(&mut tables, &reducers, &procedures)?; change_scheduled_functions_and_lifetimes_visibility(&tables, &mut reducers, &mut procedures)?; assign_query_view_primary_keys(&tables, &mut views); @@ -510,7 +519,7 @@ impl<'a> ModuleValidatorV10<'a> { table_access, is_event, accessor_name: identifier(raw_table_name)?, - on_result_reducer: None, // TODO: parse from raw table def once wire format supports it + outbox: None, // V10 attaches outbox defs from the Outboxes section; handled in validate(). }) } @@ -831,6 +840,76 @@ fn attach_schedules_to_tables( Ok(()) } +/// Attach outbox definitions from the `Outboxes` section to their respective tables. +/// +/// Also validates outbox table structure: +/// - Col 0: `u64` with `#[primary_key] #[auto_inc]` +/// - Col 1: `Identity` stored as U256 +/// - At least 2 columns total +fn attach_outboxes_to_tables( + tables: &mut HashMap, + raw_outboxes: Vec, +) -> Result<()> { + use spacetimedb_sats::AlgebraicType; + + for raw in raw_outboxes { + let table_ident = identifier(raw.table_name.clone())?; + + // Find the table by its accessor_name (source_name in the wire format). + let table = tables + .values_mut() + .find(|t| *t.accessor_name == *table_ident) + .ok_or_else(|| ValidationError::OutboxTableNotFound { + table_name: raw.table_name.clone(), + })?; + + if table.outbox.is_some() { + return Err(ValidationError::DuplicateOutbox { + table: table.name.clone(), + } + .into()); + } + + // Validate col count. + if table.columns.len() < 2 { + return Err(ValidationError::OutboxTooFewColumns { + table: table.name.clone(), + } + .into()); + } + + // Validate col 0: must be u64 (auto_inc + PK is enforced by the macro; we just check the type). + let col0_ty = &table.columns[0].ty; + if *col0_ty != AlgebraicType::U64 { + return Err(ValidationError::OutboxInvalidIdColumn { + table: table.name.clone(), + found: col0_ty.clone().into(), + } + .into()); + } + + // Validate col 1: must be U256 (Identity wire representation). + let col1_ty = &table.columns[1].ty; + if *col1_ty != AlgebraicType::U256 { + return Err(ValidationError::OutboxInvalidTargetColumn { + table: table.name.clone(), + found: col1_ty.clone().into(), + } + .into()); + } + + let remote_reducer = identifier(raw.remote_reducer)?; + let on_result_reducer = raw.on_result_reducer.map(identifier).transpose()?; + + table.outbox = Some(crate::def::OutboxDef { + remote_reducer, + on_result_reducer, + }); + } + + Ok(()) +} + fn assign_query_view_primary_keys(tables: &IdentifierMap, views: &mut IndexMap) { let primary_key_for_product_type_ref = |product_type_ref: AlgebraicTypeRef| { let mut primary_key = None; diff --git a/crates/schema/src/def/validate/v9.rs b/crates/schema/src/def/validate/v9.rs index 2e04f9a013c..87777858579 100644 --- a/crates/schema/src/def/validate/v9.rs +++ b/crates/schema/src/def/validate/v9.rs @@ -324,7 +324,7 @@ impl ModuleValidatorV9<'_> { table_access, is_event: false, // V9 does not support event tables accessor_name: name, - on_result_reducer: None, // V9 does not support on_result_reducer + outbox: None, // V9 does not support outbox tables }) } diff --git a/crates/schema/src/error.rs b/crates/schema/src/error.rs index 06f284998b5..de194f60e0c 100644 --- a/crates/schema/src/error.rs +++ b/crates/schema/src/error.rs @@ -152,6 +152,26 @@ pub enum ValidationError { ok_type: PrettyAlgebraicType, err_type: PrettyAlgebraicType, }, + #[error("outbox table {table_name} not found in module")] + OutboxTableNotFound { table_name: RawIdentifier }, + #[error("table {table} is declared as an outbox table multiple times")] + DuplicateOutbox { table: Identifier }, + #[error( + "outbox table {table} col 0 must be `u64` with `#[primary_key] #[auto_inc]`, found {found}" + )] + OutboxInvalidIdColumn { + table: Identifier, + found: PrettyAlgebraicType, + }, + #[error( + "outbox table {table} col 1 must be `Identity` (U256), found {found}" + )] + OutboxInvalidTargetColumn { + table: Identifier, + found: PrettyAlgebraicType, + }, + #[error("outbox table {table} must have at least 2 columns (id, target_identity)")] + OutboxTooFewColumns { table: Identifier }, } /// A wrapper around an `AlgebraicType` that implements `fmt::Display`. diff --git a/crates/schema/src/schema.rs b/crates/schema/src/schema.rs index 26920f04fdc..fd564f686c2 100644 --- a/crates/schema/src/schema.rs +++ b/crates/schema/src/schema.rs @@ -1014,7 +1014,7 @@ impl Schema for TableSchema { table_access, is_event, accessor_name, - on_result_reducer, + outbox, .. } = def; @@ -1041,6 +1041,11 @@ impl Schema for TableSchema { .as_ref() .map(|schedule| ScheduleSchema::from_module_def(module_def, schedule, table_id, ScheduleId::SENTINEL)); + let on_result_reducer = outbox + .as_ref() + .and_then(|o| o.on_result_reducer.as_ref()) + .map(|r| r.to_string()); + TableSchema::new( table_id, TableName::new(name.clone()), @@ -1055,7 +1060,7 @@ impl Schema for TableSchema { *primary_key, *is_event, Some(accessor_name.clone()), - on_result_reducer.clone(), + on_result_reducer, ) } From b0112b8a233c4d6b109e00cff04a574ca6349107 Mon Sep 17 00:00:00 2001 From: Shubham Mishra Date: Fri, 27 Mar 2026 19:12:21 +0530 Subject: [PATCH 06/23] fmt --- crates/bindings-macro/src/table.rs | 22 +++++++------------ crates/bindings/src/rt.rs | 8 +++---- crates/client-api/src/routes/database.rs | 8 +++++-- crates/core/src/host/idc_runtime.rs | 12 ++-------- crates/core/src/host/module_common.rs | 8 ++++++- .../src/host/wasm_common/module_host_actor.rs | 16 +++++++------- .../locking_tx_datastore/committed_state.rs | 7 +++--- .../src/locking_tx_datastore/mut_tx.rs | 8 +++---- crates/datastore/src/system_tables.rs | 9 ++++---- crates/schema/src/def.rs | 2 +- crates/schema/src/def/validate/v10.rs | 5 +---- crates/schema/src/error.rs | 8 ++----- 12 files changed, 50 insertions(+), 63 deletions(-) diff --git a/crates/bindings-macro/src/table.rs b/crates/bindings-macro/src/table.rs index bc0a9177334..2fa1f285cba 100644 --- a/crates/bindings-macro/src/table.rs +++ b/crates/bindings-macro/src/table.rs @@ -279,19 +279,14 @@ impl OutboxArg { if meta.input.peek(syn::Token![=]) || meta.input.peek(syn::token::Paren) { Err(meta.error("outbox takes a single function path, e.g. `outbox(my_remote_reducer)`")) } else { - check_duplicate_msg( - &remote_reducer, - &meta, - "can only specify one remote reducer for outbox", - )?; + check_duplicate_msg(&remote_reducer, &meta, "can only specify one remote reducer for outbox")?; remote_reducer = Some(meta.path); Ok(()) } })?; - let remote_reducer = remote_reducer.ok_or_else(|| { - syn::Error::new(span, "outbox requires a remote reducer: `outbox(my_remote_reducer)`") - })?; + let remote_reducer = remote_reducer + .ok_or_else(|| syn::Error::new(span, "outbox requires a remote reducer: `outbox(my_remote_reducer)`"))?; Ok(Self { span, @@ -309,18 +304,17 @@ impl OutboxArg { if meta.input.peek(syn::Token![=]) || meta.input.peek(syn::token::Paren) { Err(meta.error("on_result takes a single function path, e.g. `on_result(my_local_reducer)`")) } else { - check_duplicate_msg( - &result, - &meta, - "can only specify one on_result reducer", - )?; + check_duplicate_msg(&result, &meta, "can only specify one on_result reducer")?; result = Some(meta.path); Ok(()) } })?; result.ok_or_else(|| { - syn::Error::new(span, "on_result requires a local reducer: `on_result(my_local_reducer)`") + syn::Error::new( + span, + "on_result requires a local reducer: `on_result(my_local_reducer)`", + ) }) } } diff --git a/crates/bindings/src/rt.rs b/crates/bindings/src/rt.rs index 906d9231bdf..108e9a111a1 100644 --- a/crates/bindings/src/rt.rs +++ b/crates/bindings/src/rt.rs @@ -775,11 +775,9 @@ pub fn register_table() { table.finish(); if let Some(outbox) = T::OUTBOX { - module.inner.add_outbox( - T::TABLE_NAME, - outbox.remote_reducer_name, - outbox.on_result_reducer_name, - ); + module + .inner + .add_outbox(T::TABLE_NAME, outbox.remote_reducer_name, outbox.on_result_reducer_name); } module.inner.add_explicit_names(T::explicit_names()); diff --git a/crates/client-api/src/routes/database.rs b/crates/client-api/src/routes/database.rs index 71a2606f4b8..8ba7014d6e9 100644 --- a/crates/client-api/src/routes/database.rs +++ b/crates/client-api/src/routes/database.rs @@ -269,8 +269,12 @@ pub async fn call_from_database( let caller_identity = auth.claims.identity; - let sender_identity = Identity::from_hex(&sender_identity) - .map_err(|_| (StatusCode::BAD_REQUEST, "Invalid sender_identity: expected hex-encoded identity"))?; + let sender_identity = Identity::from_hex(&sender_identity).map_err(|_| { + ( + StatusCode::BAD_REQUEST, + "Invalid sender_identity: expected hex-encoded identity", + ) + })?; let args = FunctionArgs::Bsatn(body); let connection_id = generate_random_connection_id(); diff --git a/crates/core/src/host/idc_runtime.rs b/crates/core/src/host/idc_runtime.rs index 709619a936c..f117214f18e 100644 --- a/crates/core/src/host/idc_runtime.rs +++ b/crates/core/src/host/idc_runtime.rs @@ -56,12 +56,7 @@ pub struct IdcRuntimeStarter { impl IdcRuntimeStarter { /// Spawn the IDC runtime background task. - pub fn start( - self, - db: Arc, - config: IdcRuntimeConfig, - module_host: WeakModuleHost, - ) -> IdcRuntime { + pub fn start(self, db: Arc, config: IdcRuntimeConfig, module_host: WeakModuleHost) -> IdcRuntime { let abort = tokio::spawn(run_idc_loop(db, config, module_host, self.rx)).abort_handle(); IdcRuntime { _abort: abort } } @@ -328,10 +323,7 @@ fn load_pending_into_targets(db: &RelationalDB, targets: &mut HashMap WasmModuleHostActor { pub fn create_instance(&self) -> WasmModuleInstance { let common = &self.common; - let env = InstanceEnv::new(common.replica_ctx().clone(), common.scheduler().clone(), common.idc_sender()); + let env = InstanceEnv::new( + common.replica_ctx().clone(), + common.scheduler().clone(), + common.idc_sender(), + ); // this shouldn't fail, since we already called module.create_instance() // before and it didn't error, and ideally they should be deterministic let mut instance = self @@ -933,9 +937,7 @@ impl InstanceCommon { }; let res = lifecycle_res.and(dedup_res); match res { - Ok(()) => { - (EventStatus::Committed(DatabaseUpdate::default()), return_value) - } + Ok(()) => (EventStatus::Committed(DatabaseUpdate::default()), return_value), Err(err) => { let err = err.to_string(); log_reducer_error( @@ -997,9 +999,7 @@ impl InstanceCommon { // record the failure in st_inbound_msg_id in a separate tx (since the reducer tx // was rolled back). This allows the sending database to receive the error on dedup // rather than re-running the reducer. - if let (Some((sender_identity, sender_msg_id)), EventStatus::FailedUser(err)) = - (dedup_sender, &event.status) - { + if let (Some((sender_identity, sender_msg_id)), EventStatus::FailedUser(err)) = (dedup_sender, &event.status) { let err_msg = err.clone(); let mut dedup_tx = stdb.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); if let Err(e) = dedup_tx.upsert_inbound_last_msg_id( diff --git a/crates/datastore/src/locking_tx_datastore/committed_state.rs b/crates/datastore/src/locking_tx_datastore/committed_state.rs index faaec67eac9..fa89b69ddb0 100644 --- a/crates/datastore/src/locking_tx_datastore/committed_state.rs +++ b/crates/datastore/src/locking_tx_datastore/committed_state.rs @@ -30,9 +30,10 @@ use crate::{ locking_tx_datastore::ViewCallInfo, system_tables::{ ST_COLUMN_ACCESSOR_ID, ST_COLUMN_ACCESSOR_IDX, ST_CONNECTION_CREDENTIALS_ID, ST_CONNECTION_CREDENTIALS_IDX, - ST_INBOUND_MSG_ID_IDX, ST_MSG_ID_ID, ST_MSG_ID_IDX, ST_EVENT_TABLE_ID, ST_EVENT_TABLE_IDX, ST_INDEX_ACCESSOR_ID, ST_INDEX_ACCESSOR_IDX, - ST_TABLE_ACCESSOR_ID, ST_TABLE_ACCESSOR_IDX, ST_VIEW_COLUMN_ID, ST_VIEW_COLUMN_IDX, ST_VIEW_ID, ST_VIEW_IDX, - ST_VIEW_PARAM_ID, ST_VIEW_PARAM_IDX, ST_VIEW_SUB_ID, ST_VIEW_SUB_IDX, + ST_EVENT_TABLE_ID, ST_EVENT_TABLE_IDX, ST_INBOUND_MSG_ID_IDX, ST_INDEX_ACCESSOR_ID, ST_INDEX_ACCESSOR_IDX, + ST_MSG_ID_ID, ST_MSG_ID_IDX, ST_TABLE_ACCESSOR_ID, ST_TABLE_ACCESSOR_IDX, ST_VIEW_COLUMN_ID, + ST_VIEW_COLUMN_IDX, ST_VIEW_ID, ST_VIEW_IDX, ST_VIEW_PARAM_ID, ST_VIEW_PARAM_IDX, ST_VIEW_SUB_ID, + ST_VIEW_SUB_IDX, }, }; use anyhow::anyhow; diff --git a/crates/datastore/src/locking_tx_datastore/mut_tx.rs b/crates/datastore/src/locking_tx_datastore/mut_tx.rs index de5c3c917cc..5f333ff3108 100644 --- a/crates/datastore/src/locking_tx_datastore/mut_tx.rs +++ b/crates/datastore/src/locking_tx_datastore/mut_tx.rs @@ -20,10 +20,10 @@ use crate::{ use crate::{ error::{IndexError, SequenceError, TableError}, system_tables::{ - with_sys_table_buf, StClientFields, StClientRow, StColumnAccessorFields, StColumnAccessorRow, - StColumnFields, StColumnRow, StConstraintFields, StConstraintRow, StEventTableRow, StFields as _, - StInboundMsgIdFields, StInboundMsgIdRow, StIndexAccessorFields, StIndexAccessorRow, StIndexFields, StIndexRow, - StMsgIdFields, StMsgIdRow, StRowLevelSecurityFields, StRowLevelSecurityRow, StScheduledFields, StScheduledRow, + with_sys_table_buf, StClientFields, StClientRow, StColumnAccessorFields, StColumnAccessorRow, StColumnFields, + StColumnRow, StConstraintFields, StConstraintRow, StEventTableRow, StFields as _, StInboundMsgIdFields, + StInboundMsgIdRow, StIndexAccessorFields, StIndexAccessorRow, StIndexFields, StIndexRow, StMsgIdFields, + StMsgIdRow, StRowLevelSecurityFields, StRowLevelSecurityRow, StScheduledFields, StScheduledRow, StSequenceFields, StSequenceRow, StTableAccessorFields, StTableAccessorRow, StTableFields, StTableRow, SystemTable, ST_CLIENT_ID, ST_COLUMN_ACCESSOR_ID, ST_COLUMN_ID, ST_CONSTRAINT_ID, ST_EVENT_TABLE_ID, ST_INBOUND_MSG_ID_ID, ST_INDEX_ACCESSOR_ID, ST_INDEX_ID, ST_MSG_ID_ID, ST_ROW_LEVEL_SECURITY_ID, diff --git a/crates/datastore/src/system_tables.rs b/crates/datastore/src/system_tables.rs index 56b2851b4e2..aa1c702e57e 100644 --- a/crates/datastore/src/system_tables.rs +++ b/crates/datastore/src/system_tables.rs @@ -190,7 +190,9 @@ pub fn is_built_in_meta_row(table_id: TableId, row: &ProductValue) -> Result false, + ST_TABLE_ACCESSOR_ID | ST_INDEX_ACCESSOR_ID | ST_COLUMN_ACCESSOR_ID | ST_INBOUND_MSG_ID_ID | ST_MSG_ID_ID => { + false + } TableId(..ST_RESERVED_SEQUENCE_RANGE) => { log::warn!("Unknown system table {table_id:?}"); false @@ -706,10 +708,7 @@ fn system_module_def() -> ModuleDef { let st_msg_id_type = builder.add_type::(); builder - .build_table( - ST_MSG_ID_NAME, - *st_msg_id_type.as_ref().expect("should be ref"), - ) + .build_table(ST_MSG_ID_NAME, *st_msg_id_type.as_ref().expect("should be ref")) .with_type(TableType::System) .with_auto_inc_primary_key(StMsgIdFields::MsgId) .with_index_no_accessor_name(btree(StMsgIdFields::MsgId)); diff --git a/crates/schema/src/def.rs b/crates/schema/src/def.rs index 7e89c048518..807de4cbf38 100644 --- a/crates/schema/src/def.rs +++ b/crates/schema/src/def.rs @@ -752,7 +752,7 @@ impl From for RawTableDefV9 { table_type, table_access, is_event: _, // V9 does not support event tables; ignore when converting back - outbox: _, // V9 does not support outbox tables; ignore when converting back + outbox: _, // V9 does not support outbox tables; ignore when converting back .. } = val; diff --git a/crates/schema/src/def/validate/v10.rs b/crates/schema/src/def/validate/v10.rs index b54d3274ffb..cdc6e9ef909 100644 --- a/crates/schema/src/def/validate/v10.rs +++ b/crates/schema/src/def/validate/v10.rs @@ -189,10 +189,7 @@ pub fn validate(def: RawModuleDefV10) -> Result { .unwrap_or_else(|| Ok(Vec::new())); // Collect raw outbox definitions from the Outboxes section for post-validation attachment. - let raw_outboxes: Vec = def - .outboxes() - .map(|v| v.to_vec()) - .unwrap_or_default(); + let raw_outboxes: Vec = def.outboxes().map(|v| v.to_vec()).unwrap_or_default(); // Validate lifecycle reducers - they reference reducers by name let lifecycle_validations = reducers diff --git a/crates/schema/src/error.rs b/crates/schema/src/error.rs index de194f60e0c..2b074ab7a75 100644 --- a/crates/schema/src/error.rs +++ b/crates/schema/src/error.rs @@ -156,16 +156,12 @@ pub enum ValidationError { OutboxTableNotFound { table_name: RawIdentifier }, #[error("table {table} is declared as an outbox table multiple times")] DuplicateOutbox { table: Identifier }, - #[error( - "outbox table {table} col 0 must be `u64` with `#[primary_key] #[auto_inc]`, found {found}" - )] + #[error("outbox table {table} col 0 must be `u64` with `#[primary_key] #[auto_inc]`, found {found}")] OutboxInvalidIdColumn { table: Identifier, found: PrettyAlgebraicType, }, - #[error( - "outbox table {table} col 1 must be `Identity` (U256), found {found}" - )] + #[error("outbox table {table} col 1 must be `Identity` (U256), found {found}")] OutboxInvalidTargetColumn { table: Identifier, found: PrettyAlgebraicType, From 23adfc2c88642e1c181f3fcabee202b3c20288d0 Mon Sep 17 00:00:00 2001 From: Shubham Mishra Date: Fri, 27 Mar 2026 19:54:30 +0530 Subject: [PATCH 07/23] fix enqueue_outbox_row --- crates/core/src/host/instance_env.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/crates/core/src/host/instance_env.rs b/crates/core/src/host/instance_env.rs index 24a4da4cab8..ab4c2153ddb 100644 --- a/crates/core/src/host/instance_env.rs +++ b/crates/core/src/host/instance_env.rs @@ -442,8 +442,6 @@ impl InstanceEnv { /// /// The `on_result_reducer` and delivery data are resolved at delivery time from the /// outbox table's schema and row, so ST_MSG_ID only stores the minimal reference. - #[cold] - #[inline(never)] fn enqueue_outbox_row( &self, _stdb: &RelationalDB, From 5cd680b0cc4ef7760b2805dfe0a1e1f10ad17971 Mon Sep 17 00:00:00 2001 From: Shubham Mishra Date: Fri, 27 Mar 2026 21:02:10 +0530 Subject: [PATCH 08/23] system table naming --- crates/client-api/src/routes/database.rs | 8 +- crates/core/src/host/host_controller.rs | 2 +- crates/core/src/host/idc_runtime.rs | 39 ++++---- crates/core/src/host/instance_env.rs | 10 +- crates/core/src/host/module_host.rs | 4 +- .../src/host/wasm_common/module_host_actor.rs | 20 ++-- .../locking_tx_datastore/committed_state.rs | 10 +- .../src/locking_tx_datastore/mut_tx.rs | 72 +++++++------- crates/datastore/src/system_tables.rs | 96 ++++++++++--------- crates/lib/src/db/raw_def/v10.rs | 2 +- crates/schema/src/def.rs | 2 +- 11 files changed, 135 insertions(+), 130 deletions(-) diff --git a/crates/client-api/src/routes/database.rs b/crates/client-api/src/routes/database.rs index 8ba7014d6e9..ae091f56213 100644 --- a/crates/client-api/src/routes/database.rs +++ b/crates/client-api/src/routes/database.rs @@ -230,8 +230,8 @@ pub struct CallFromDatabaseParams { pub struct CallFromDatabaseQuery { /// Hex-encoded [`Identity`] of the sending database. sender_identity: String, - /// The inter-database message ID from the sender's st_msg_id. - /// Used for at-most-once delivery via `st_inbound_msg_id`. + /// The inter-database message ID from the sender's st_outbound_msg. + /// Used for at-most-once delivery via `st_inbound_msg`. msg_id: u64, } @@ -241,9 +241,9 @@ pub struct CallFromDatabaseQuery { /// /// Required query params: /// - `sender_identity` — hex-encoded identity of the sending database. -/// - `msg_id` — the inter-database message ID from the sender's st_msg_id. +/// - `msg_id` — the inter-database message ID from the sender's st_outbound_msg. /// -/// Before invoking the reducer, the receiver checks `st_inbound_msg_id`. +/// Before invoking the reducer, the receiver checks `st_inbound_msg`. /// If the incoming `msg_id` is ≤ the last delivered msg_id for `sender_identity`, /// the call is a duplicate and 200 OK is returned immediately without running the reducer. /// Otherwise the reducer is invoked, the dedup index is updated atomically in the same diff --git a/crates/core/src/host/host_controller.rs b/crates/core/src/host/host_controller.rs index cd2e715eed0..c0f6073e9db 100644 --- a/crates/core/src/host/host_controller.rs +++ b/crates/core/src/host/host_controller.rs @@ -889,7 +889,7 @@ struct Host { /// Handle to the task responsible for cleaning up old views. /// The task is aborted when [`Host`] is dropped. view_cleanup_task: AbortHandle, - /// IDC runtime: delivers outbound inter-database messages from `st_msg_id`. + /// IDC runtime: delivers outbound inter-database messages from `st_outbound_msg`. /// Stopped when [`Host`] is dropped. _idc_runtime: IdcRuntime, } diff --git a/crates/core/src/host/idc_runtime.rs b/crates/core/src/host/idc_runtime.rs index f117214f18e..3fd8dd51b2c 100644 --- a/crates/core/src/host/idc_runtime.rs +++ b/crates/core/src/host/idc_runtime.rs @@ -1,20 +1,20 @@ /// Inter-Database Communication (IDC) Runtime /// /// Background tokio task that: -/// 1. Loads undelivered entries from `st_msg_id` on startup, resolving delivery data from outbox tables. +/// 1. Loads undelivered entries from `st_outbound_msg` on startup, resolving delivery data from outbox tables. /// 2. Accepts immediate notifications via an mpsc channel when new outbox rows are inserted. /// 3. Delivers each message in msg_id order via HTTP POST to /// `http://localhost:80/v1/database/{target_db}/call-from-database/{reducer}?sender_identity=&msg_id=` /// 4. On transport errors (network, 5xx, 4xx except 422/402): retries infinitely with exponential /// backoff, blocking only the affected target database (other targets continue unaffected). /// 5. On reducer errors (HTTP 422) or budget exceeded (HTTP 402): calls the configured -/// `on_result_reducer` (read from the outbox table's schema) and deletes the st_msg_id row. +/// `on_result_reducer` (read from the outbox table's schema) and deletes the st_outbound_msg row. /// 6. Enforces sequential delivery per target database: msg N+1 is only delivered after N is done. use crate::db::relational_db::RelationalDB; use crate::host::module_host::WeakModuleHost; use crate::host::FunctionArgs; use spacetimedb_datastore::execution_context::Workload; -use spacetimedb_datastore::system_tables::{StMsgIdRow, ST_MSG_ID_ID}; +use spacetimedb_datastore::system_tables::{StOutboundMsgRow, ST_OUTBOUND_MSG_ID}; use spacetimedb_datastore::traits::IsolationLevel; use spacetimedb_lib::{AlgebraicValue, Identity}; use spacetimedb_primitives::{ColId, TableId}; @@ -207,19 +207,19 @@ async fn run_idc_loop( /// Decode the delivery outcome into `(result_status, result_payload)` for recording. fn outcome_to_result(outcome: &DeliveryOutcome) -> (u8, String) { - use spacetimedb_datastore::system_tables::st_inbound_msg_id_result_status; + use spacetimedb_datastore::system_tables::st_inbound_msg_result_status; match outcome { - DeliveryOutcome::Success => (st_inbound_msg_id_result_status::SUCCESS, String::new()), - DeliveryOutcome::ReducerError(msg) => (st_inbound_msg_id_result_status::REDUCER_ERROR, msg.clone()), + DeliveryOutcome::Success => (st_inbound_msg_result_status::SUCCESS, String::new()), + DeliveryOutcome::ReducerError(msg) => (st_inbound_msg_result_status::REDUCER_ERROR, msg.clone()), DeliveryOutcome::BudgetExceeded => ( - st_inbound_msg_id_result_status::REDUCER_ERROR, + st_inbound_msg_result_status::REDUCER_ERROR, "budget exceeded".to_string(), ), DeliveryOutcome::TransportError(_) => unreachable!("transport errors never finalize"), } } -/// Finalize a delivered message: call the on_result reducer (if any), then delete from ST_MSG_ID. +/// Finalize a delivered message: call the on_result reducer (if any), then delete from ST_OUTBOUND_MSG. async fn finalize_message( db: &RelationalDB, module_host: &WeakModuleHost, @@ -288,25 +288,28 @@ async fn finalize_message( delete_message(db, msg.msg_id); } -/// Load all messages from ST_MSG_ID into the per-target queues, resolving delivery data +/// Load all messages from ST_OUTBOUND_MSG into the per-target queues, resolving delivery data /// from the corresponding outbox table rows. /// -/// A row's presence in ST_MSG_ID means it has not yet been processed. +/// A row's presence in ST_OUTBOUND_MSG means it has not yet been processed. /// Messages already in a target's queue (by msg_id) are not re-added. fn load_pending_into_targets(db: &RelationalDB, targets: &mut HashMap) { let tx = db.begin_tx(Workload::Internal); - let st_msg_id_rows: Vec = db - .iter(&tx, ST_MSG_ID_ID) - .map(|iter| iter.filter_map(|row_ref| StMsgIdRow::try_from(row_ref).ok()).collect()) + let st_outbound_msg_rows: Vec = db + .iter(&tx, ST_OUTBOUND_MSG_ID) + .map(|iter| { + iter.filter_map(|row_ref| StOutboundMsgRow::try_from(row_ref).ok()) + .collect() + }) .unwrap_or_else(|e| { - log::error!("idc_runtime: failed to read st_msg_id: {e}"); + log::error!("idc_runtime: failed to read st_outbound_msg: {e}"); Vec::new() }); - let mut pending: Vec = Vec::with_capacity(st_msg_id_rows.len()); + let mut pending: Vec = Vec::with_capacity(st_outbound_msg_rows.len()); - for st_row in st_msg_id_rows { + for st_row in st_outbound_msg_rows { let outbox_table_id = TableId(st_row.outbox_table_id); // Read the outbox table schema for reducer name and on_result_reducer. @@ -432,10 +435,10 @@ async fn attempt_delivery( } } -/// Delete a message from ST_MSG_ID within a new transaction. +/// Delete a message from ST_OUTBOUND_MSG within a new transaction. fn delete_message(db: &RelationalDB, msg_id: u64) { let mut tx = db.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); - if let Err(e) = tx.delete_msg_id(msg_id) { + if let Err(e) = tx.delete_outbound_msg(msg_id) { log::error!("idc_runtime: failed to delete msg_id={msg_id}: {e}"); let _ = db.rollback_mut_tx(tx); return; diff --git a/crates/core/src/host/instance_env.rs b/crates/core/src/host/instance_env.rs index ab4c2153ddb..6c811c78fa1 100644 --- a/crates/core/src/host/instance_env.rs +++ b/crates/core/src/host/instance_env.rs @@ -432,16 +432,16 @@ impl InstanceEnv { Ok(()) } - /// Enqueue an outbox row into ST_MSG_ID atomically within the current transaction, + /// Enqueue an outbox row into ST_OUTBOUND_MSG atomically within the current transaction, /// and notify the IDC runtime so it delivers without waiting for the next poll cycle. /// /// Outbox tables follow the naming convention `__outbox_` and have: - /// - Col 0: auto-inc primary key (u64) — stored as `row_id` in ST_MSG_ID. + /// - Col 0: auto-inc primary key (u64) — stored as `row_id` in ST_OUTBOUND_MSG. /// - Col 1: target database Identity (stored as U256). /// - Remaining cols: args for the remote reducer. /// /// The `on_result_reducer` and delivery data are resolved at delivery time from the - /// outbox table's schema and row, so ST_MSG_ID only stores the minimal reference. + /// outbox table's schema and row, so ST_OUTBOUND_MSG only stores the minimal reference. fn enqueue_outbox_row( &self, _stdb: &RelationalDB, @@ -454,7 +454,7 @@ impl InstanceEnv { let row_ref = tx.get(table_id, row_ptr).map_err(DBError::from)?.unwrap(); let pv = row_ref.to_product_value(); - // Col 0 is the auto-inc primary key — this is the row_id we store in ST_MSG_ID. + // Col 0 is the auto-inc primary key — this is the row_id we store in ST_OUTBOUND_MSG. let row_id = match pv.elements.first() { Some(AlgebraicValue::U64(id)) => *id, other => { @@ -466,7 +466,7 @@ impl InstanceEnv { } }; - tx.insert_st_msg_id(table_id.0, row_id).map_err(DBError::from)?; + tx.insert_st_outbound_msg(table_id.0, row_id).map_err(DBError::from)?; // Wake the IDC runtime immediately so it doesn't wait for the next poll cycle. let _ = self.idc_sender.send(()); diff --git a/crates/core/src/host/module_host.rs b/crates/core/src/host/module_host.rs index e9b823be575..a582d521393 100644 --- a/crates/core/src/host/module_host.rs +++ b/crates/core/src/host/module_host.rs @@ -627,7 +627,7 @@ pub struct CallReducerParams { /// If set, enables at-most-once delivery semantics for database-to-database calls. /// /// The tuple is `(sender_database_identity, sender_msg_id)`. - /// Before running the reducer, `st_inbound_msg_id` is consulted: + /// Before running the reducer, `st_inbound_msg` is consulted: /// if `sender_msg_id` ≤ the stored last-delivered msg_id for the sender, /// the call is a duplicate and returns [`ReducerCallError::Deduplicated`]. /// Otherwise the reducer runs, and the stored msg_id is updated atomically @@ -1670,7 +1670,7 @@ impl ModuleHost { /// Variant of [`Self::call_reducer`] for database-to-database calls. /// /// Behaves identically to `call_reducer`, except that it enforces at-most-once - /// delivery using the `st_inbound_msg_id` dedup index. + /// delivery using the `st_inbound_msg` dedup index. /// Before invoking the reducer, the receiver checks whether /// `sender_msg_id` ≤ the last delivered msg_id for `sender_database_identity`. /// If so, the call is a duplicate and [`ReducerOutcome::Deduplicated`] is returned diff --git a/crates/core/src/host/wasm_common/module_host_actor.rs b/crates/core/src/host/wasm_common/module_host_actor.rs index 229b60f2ffb..b8156cda65f 100644 --- a/crates/core/src/host/wasm_common/module_host_actor.rs +++ b/crates/core/src/host/wasm_common/module_host_actor.rs @@ -37,7 +37,7 @@ use spacetimedb_datastore::db_metrics::DB_METRICS; use spacetimedb_datastore::error::{DatastoreError, ViewError}; use spacetimedb_datastore::execution_context::{self, ReducerContext, Workload}; use spacetimedb_datastore::locking_tx_datastore::{FuncCallType, MutTxId, ViewCallInfo}; -use spacetimedb_datastore::system_tables::st_inbound_msg_id_result_status; +use spacetimedb_datastore::system_tables::st_inbound_msg_result_status; use spacetimedb_datastore::traits::{IsolationLevel, Program}; use spacetimedb_execution::pipelined::PipelinedProject; use spacetimedb_lib::buffer::DecodeError; @@ -855,13 +855,13 @@ impl InstanceCommon { // If the incoming msg_id is ≤ the last delivered msg_id for this sender, // the message is a duplicate; discard it and return the stored result. if let Some((sender_identity, sender_msg_id)) = dedup_sender { - let stored = tx.get_inbound_msg_id_row(sender_identity); - if stored.as_ref().is_some_and(|r| sender_msg_id <= r.last_msg_id) { + let stored = tx.get_inbound_msg_row(sender_identity); + if stored.as_ref().is_some_and(|r| sender_msg_id <= r.last_outbound_msg) { let _ = stdb.rollback_mut_tx(tx); let stored = stored.unwrap(); let outcome = match stored.result_status { - s if s == st_inbound_msg_id_result_status::SUCCESS => ReducerOutcome::Committed, - s if s == st_inbound_msg_id_result_status::REDUCER_ERROR => { + s if s == st_inbound_msg_result_status::SUCCESS => ReducerOutcome::Committed, + s if s == st_inbound_msg_result_status::REDUCER_ERROR => { ReducerOutcome::Failed(Box::new(stored.result_payload.into())) } _ => ReducerOutcome::Deduplicated, @@ -927,10 +927,10 @@ impl InstanceCommon { // Atomically update the dedup index so this sender's tx_offset is recorded // as delivered. This prevents re-processing if the message is retried. let dedup_res = match dedup_sender { - Some((sender_identity, sender_msg_id)) => tx.upsert_inbound_last_msg_id( + Some((sender_identity, sender_msg_id)) => tx.upsert_inbound_last_msg( sender_identity, sender_msg_id, - st_inbound_msg_id_result_status::SUCCESS, + st_inbound_msg_result_status::SUCCESS, String::new(), ), None => Ok(()), @@ -996,16 +996,16 @@ impl InstanceCommon { let event = commit_and_broadcast_event(&info.subscriptions, client, event, out.tx).event; // For IDC (database-to-database) calls: if the reducer failed with a user error, - // record the failure in st_inbound_msg_id in a separate tx (since the reducer tx + // record the failure in st_inbound_msg in a separate tx (since the reducer tx // was rolled back). This allows the sending database to receive the error on dedup // rather than re-running the reducer. if let (Some((sender_identity, sender_msg_id)), EventStatus::FailedUser(err)) = (dedup_sender, &event.status) { let err_msg = err.clone(); let mut dedup_tx = stdb.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); - if let Err(e) = dedup_tx.upsert_inbound_last_msg_id( + if let Err(e) = dedup_tx.upsert_inbound_last_msg( sender_identity, sender_msg_id, - st_inbound_msg_id_result_status::REDUCER_ERROR, + st_inbound_msg_result_status::REDUCER_ERROR, err_msg, ) { log::error!("IDC: failed to record reducer error in dedup table for sender {sender_identity}: {e}"); diff --git a/crates/datastore/src/locking_tx_datastore/committed_state.rs b/crates/datastore/src/locking_tx_datastore/committed_state.rs index fa89b69ddb0..80060eca95c 100644 --- a/crates/datastore/src/locking_tx_datastore/committed_state.rs +++ b/crates/datastore/src/locking_tx_datastore/committed_state.rs @@ -19,7 +19,7 @@ use crate::{ is_built_in_meta_row, system_tables, table_id_is_reserved, StColumnRow, StConstraintData, StConstraintRow, StFields, StIndexRow, StSequenceFields, StSequenceRow, StTableFields, StTableRow, StViewRow, SystemTable, ST_CLIENT_ID, ST_CLIENT_IDX, ST_COLUMN_ID, ST_COLUMN_IDX, ST_COLUMN_NAME, ST_CONSTRAINT_ID, ST_CONSTRAINT_IDX, - ST_CONSTRAINT_NAME, ST_INBOUND_MSG_ID_ID, ST_INDEX_ID, ST_INDEX_IDX, ST_INDEX_NAME, ST_MODULE_ID, + ST_CONSTRAINT_NAME, ST_INBOUND_MSG_ID, ST_INDEX_ID, ST_INDEX_IDX, ST_INDEX_NAME, ST_MODULE_ID, ST_MODULE_IDX, ST_ROW_LEVEL_SECURITY_ID, ST_ROW_LEVEL_SECURITY_IDX, ST_SCHEDULED_ID, ST_SCHEDULED_IDX, ST_SEQUENCE_ID, ST_SEQUENCE_IDX, ST_SEQUENCE_NAME, ST_TABLE_ID, ST_TABLE_IDX, ST_VAR_ID, ST_VAR_IDX, ST_VIEW_ARG_ID, ST_VIEW_ARG_IDX, @@ -30,8 +30,8 @@ use crate::{ locking_tx_datastore::ViewCallInfo, system_tables::{ ST_COLUMN_ACCESSOR_ID, ST_COLUMN_ACCESSOR_IDX, ST_CONNECTION_CREDENTIALS_ID, ST_CONNECTION_CREDENTIALS_IDX, - ST_EVENT_TABLE_ID, ST_EVENT_TABLE_IDX, ST_INBOUND_MSG_ID_IDX, ST_INDEX_ACCESSOR_ID, ST_INDEX_ACCESSOR_IDX, - ST_MSG_ID_ID, ST_MSG_ID_IDX, ST_TABLE_ACCESSOR_ID, ST_TABLE_ACCESSOR_IDX, ST_VIEW_COLUMN_ID, + ST_EVENT_TABLE_ID, ST_EVENT_TABLE_IDX, ST_INBOUND_MSG_IDX, ST_INDEX_ACCESSOR_ID, ST_INDEX_ACCESSOR_IDX, + ST_OUTBOUND_MSG_ID, ST_OUTBOUND_MSG_IDX, ST_TABLE_ACCESSOR_ID, ST_TABLE_ACCESSOR_IDX, ST_VIEW_COLUMN_ID, ST_VIEW_COLUMN_IDX, ST_VIEW_ID, ST_VIEW_IDX, ST_VIEW_PARAM_ID, ST_VIEW_PARAM_IDX, ST_VIEW_SUB_ID, ST_VIEW_SUB_IDX, }, @@ -478,8 +478,8 @@ impl CommittedState { self.create_table(ST_TABLE_ACCESSOR_ID, schemas[ST_TABLE_ACCESSOR_IDX].clone()); self.create_table(ST_INDEX_ACCESSOR_ID, schemas[ST_INDEX_ACCESSOR_IDX].clone()); self.create_table(ST_COLUMN_ACCESSOR_ID, schemas[ST_COLUMN_ACCESSOR_IDX].clone()); - self.create_table(ST_INBOUND_MSG_ID_ID, schemas[ST_INBOUND_MSG_ID_IDX].clone()); - self.create_table(ST_MSG_ID_ID, schemas[ST_MSG_ID_IDX].clone()); + self.create_table(ST_INBOUND_MSG_ID, schemas[ST_INBOUND_MSG_IDX].clone()); + self.create_table(ST_OUTBOUND_MSG_ID, schemas[ST_OUTBOUND_MSG_IDX].clone()); // Insert the sequences into `st_sequences` let (st_sequences, blob_store, pool) = diff --git a/crates/datastore/src/locking_tx_datastore/mut_tx.rs b/crates/datastore/src/locking_tx_datastore/mut_tx.rs index 5f333ff3108..b5c72ea22b7 100644 --- a/crates/datastore/src/locking_tx_datastore/mut_tx.rs +++ b/crates/datastore/src/locking_tx_datastore/mut_tx.rs @@ -21,12 +21,12 @@ use crate::{ error::{IndexError, SequenceError, TableError}, system_tables::{ with_sys_table_buf, StClientFields, StClientRow, StColumnAccessorFields, StColumnAccessorRow, StColumnFields, - StColumnRow, StConstraintFields, StConstraintRow, StEventTableRow, StFields as _, StInboundMsgIdFields, - StInboundMsgIdRow, StIndexAccessorFields, StIndexAccessorRow, StIndexFields, StIndexRow, StMsgIdFields, - StMsgIdRow, StRowLevelSecurityFields, StRowLevelSecurityRow, StScheduledFields, StScheduledRow, + StColumnRow, StConstraintFields, StConstraintRow, StEventTableRow, StFields as _, StInboundMsgFields, + StInboundMsgRow, StIndexAccessorFields, StIndexAccessorRow, StIndexFields, StIndexRow, StOutboundMsgFields, + StOutboundMsgRow, StRowLevelSecurityFields, StRowLevelSecurityRow, StScheduledFields, StScheduledRow, StSequenceFields, StSequenceRow, StTableAccessorFields, StTableAccessorRow, StTableFields, StTableRow, SystemTable, ST_CLIENT_ID, ST_COLUMN_ACCESSOR_ID, ST_COLUMN_ID, ST_CONSTRAINT_ID, ST_EVENT_TABLE_ID, - ST_INBOUND_MSG_ID_ID, ST_INDEX_ACCESSOR_ID, ST_INDEX_ID, ST_MSG_ID_ID, ST_ROW_LEVEL_SECURITY_ID, + ST_INBOUND_MSG_ID, ST_INDEX_ACCESSOR_ID, ST_INDEX_ID, ST_OUTBOUND_MSG_ID, ST_ROW_LEVEL_SECURITY_ID, ST_SCHEDULED_ID, ST_SEQUENCE_ID, ST_TABLE_ACCESSOR_ID, ST_TABLE_ID, }, }; @@ -2692,83 +2692,83 @@ impl MutTxId { .map(|row| row.pointer()) } - /// Look up the inbound dedup record for `sender_identity` in `st_inbound_msg_id`. + /// Look up the inbound dedup record for `sender_identity` in `st_inbound_msg`. /// /// Returns `None` if no entry exists for this sender (i.e., no message has been delivered yet). - pub fn get_inbound_msg_id_row(&self, sender_identity: Identity) -> Option { + pub fn get_inbound_msg_row(&self, sender_identity: Identity) -> Option { self.iter_by_col_eq( - ST_INBOUND_MSG_ID_ID, - StInboundMsgIdFields::DatabaseIdentity.col_id(), + ST_INBOUND_MSG_ID, + StInboundMsgFields::DatabaseIdentity.col_id(), &IdentityViaU256::from(sender_identity).into(), ) - .expect("failed to read from st_inbound_msg_id system table") + .expect("failed to read from st_inbound_msg system table") .next() - .and_then(|row_ref| StInboundMsgIdRow::try_from(row_ref).ok()) + .and_then(|row_ref| StInboundMsgRow::try_from(row_ref).ok()) } - /// Update the last delivered msg_id for `sender_identity` in `st_inbound_msg_id`. + /// Update the last delivered msg_id for `sender_identity` in `st_inbound_msg`. /// /// If an entry already exists, it is replaced; otherwise a new entry is inserted. /// `result_status` and `result_payload` store the outcome of the reducer call - /// (see [st_inbound_msg_id_result_status]). - pub fn upsert_inbound_last_msg_id( + /// (see [st_inbound_msg_result_status]). + pub fn upsert_inbound_last_msg( &mut self, sender_identity: Identity, - last_msg_id: u64, + last_outbound_msg: u64, result_status: u8, result_payload: String, ) -> Result<()> { // Delete the existing row if present. self.delete_col_eq( - ST_INBOUND_MSG_ID_ID, - StInboundMsgIdFields::DatabaseIdentity.col_id(), + ST_INBOUND_MSG_ID, + StInboundMsgFields::DatabaseIdentity.col_id(), &IdentityViaU256::from(sender_identity).into(), )?; - let row = StInboundMsgIdRow { + let row = StInboundMsgRow { database_identity: sender_identity.into(), - last_msg_id, + last_outbound_msg, result_status, result_payload, }; - self.insert_via_serialize_bsatn(ST_INBOUND_MSG_ID_ID, &row) + self.insert_via_serialize_bsatn(ST_INBOUND_MSG_ID, &row) .map(|_| ()) .inspect_err(|e| { log::error!( - "upsert_inbound_last_msg_id: failed to upsert last_msg_id for {sender_identity} to {last_msg_id}: {e}" + "upsert_inbound_last_outbound_msg: failed to upsert last_outbound_msg for {sender_identity} to {last_outbound_msg}: {e}" ); }) } - /// Insert a new outbound inter-database message into `st_msg_id`. - pub fn insert_st_msg_id(&mut self, outbox_table_id: u32, row_id: u64) -> Result<()> { - let row = StMsgIdRow { + /// Insert a new outbound inter-database message into `st_outbound_msg`. + pub fn insert_st_outbound_msg(&mut self, outbox_table_id: u32, row_id: u64) -> Result<()> { + let row = StOutboundMsgRow { msg_id: 0, // auto-incremented by the sequence outbox_table_id, row_id, }; - self.insert_via_serialize_bsatn(ST_MSG_ID_ID, &row) + self.insert_via_serialize_bsatn(ST_OUTBOUND_MSG_ID, &row) .map(|_| ()) .inspect_err(|e| { - log::error!("insert_st_msg_id: failed to insert msg for outbox_table_id={outbox_table_id}: {e}"); + log::error!("insert_st_outbound_msg: failed to insert msg for outbox_table_id={outbox_table_id}: {e}"); }) } - /// Retrieve all outbound messages from `st_msg_id`, ordered by msg_id ascending. - pub fn all_msg_ids(&self) -> Result> { - let mut rows: Vec = self - .iter(ST_MSG_ID_ID) - .expect("failed to read from st_msg_id system table") - .filter_map(|row_ref| StMsgIdRow::try_from(row_ref).ok()) + /// Retrieve all outbound messages from `st_outbound_msg`, ordered by msg_id ascending. + pub fn all_outbound_msgs(&self) -> Result> { + let mut rows: Vec = self + .iter(ST_OUTBOUND_MSG_ID) + .expect("failed to read from st_outbound_msg system table") + .filter_map(|row_ref| StOutboundMsgRow::try_from(row_ref).ok()) .collect(); rows.sort_by_key(|r| r.msg_id); Ok(rows) } - /// Delete a message from `st_msg_id` once it has been fully processed. - pub fn delete_msg_id(&mut self, msg_id: u64) -> Result<()> { + /// Delete a message from `st_outbound_msg` once it has been fully processed. + pub fn delete_outbound_msg(&mut self, msg_id: u64) -> Result<()> { self.delete_col_eq( - ST_MSG_ID_ID, - StMsgIdFields::MsgId.col_id(), + ST_OUTBOUND_MSG_ID, + StOutboundMsgFields::MsgId.col_id(), &AlgebraicValue::U64(msg_id), ) .map(|_| ()) @@ -2780,7 +2780,7 @@ impl MutTxId { /// Returns the table IDs and table names of all tables that had rows inserted /// in this transaction and whose name starts with `"__outbox_"`. /// - /// Used by the IDC runtime to detect outbox inserts and enqueue ST_MSG_ID entries. + /// Used by the IDC runtime to detect outbox inserts and enqueue ST_OUTBOUND_MSG entries. pub fn outbox_insert_table_ids(&self) -> Vec<(TableId, String)> { let mut result = Vec::new(); for table_id in self.tx_state.insert_tables.keys() { diff --git a/crates/datastore/src/system_tables.rs b/crates/datastore/src/system_tables.rs index aa1c702e57e..efedacb473a 100644 --- a/crates/datastore/src/system_tables.rs +++ b/crates/datastore/src/system_tables.rs @@ -90,10 +90,10 @@ pub const ST_INDEX_ACCESSOR_ID: TableId = TableId(19); pub const ST_COLUMN_ACCESSOR_ID: TableId = TableId(20); /// The static ID of the table that tracks the last inbound msg id per sender database. -pub const ST_INBOUND_MSG_ID_ID: TableId = TableId(21); +pub const ST_INBOUND_MSG_ID: TableId = TableId(21); /// The static ID of the table that tracks outbound inter-database messages. -pub const ST_MSG_ID_ID: TableId = TableId(22); +pub const ST_OUTBOUND_MSG_ID: TableId = TableId(22); pub(crate) const ST_CONNECTION_CREDENTIALS_NAME: &str = "st_connection_credentials"; pub const ST_TABLE_NAME: &str = "st_table"; @@ -115,8 +115,8 @@ pub(crate) const ST_EVENT_TABLE_NAME: &str = "st_event_table"; pub(crate) const ST_TABLE_ACCESSOR_NAME: &str = "st_table_accessor"; pub(crate) const ST_INDEX_ACCESSOR_NAME: &str = "st_index_accessor"; pub(crate) const ST_COLUMN_ACCESSOR_NAME: &str = "st_column_accessor"; -pub(crate) const ST_INBOUND_MSG_ID_NAME: &str = "st_inbound_msg_id"; -pub(crate) const ST_MSG_ID_NAME: &str = "st_msg_id"; +pub(crate) const ST_INBOUND_MSG_NAME: &str = "st_inbound_msg"; +pub(crate) const ST_OUTBOUND_MSG_NAME: &str = "st_outbound_msg"; /// Reserved range of sequence values used for system tables. /// /// Ids for user-created tables will start at `ST_RESERVED_SEQUENCE_RANGE`. @@ -190,9 +190,11 @@ pub fn is_built_in_meta_row(table_id: TableId, row: &ProductValue) -> Result { - false - } + ST_TABLE_ACCESSOR_ID + | ST_INDEX_ACCESSOR_ID + | ST_COLUMN_ACCESSOR_ID + | ST_INBOUND_MSG_ID + | ST_OUTBOUND_MSG_ID => false, TableId(..ST_RESERVED_SEQUENCE_RANGE) => { log::warn!("Unknown system table {table_id:?}"); false @@ -238,8 +240,8 @@ pub fn system_tables() -> [TableSchema; 22] { st_table_accessor_schema(), st_index_accessor_schema(), st_column_accessor_schema(), - st_inbound_msg_id_schema(), - st_msg_id_schema(), + st_inbound_msg_schema(), + st_outbound_schema(), ] } @@ -288,8 +290,8 @@ pub(crate) const ST_EVENT_TABLE_IDX: usize = 16; pub(crate) const ST_TABLE_ACCESSOR_IDX: usize = 17; pub(crate) const ST_INDEX_ACCESSOR_IDX: usize = 18; pub(crate) const ST_COLUMN_ACCESSOR_IDX: usize = 19; -pub(crate) const ST_INBOUND_MSG_ID_IDX: usize = 20; -pub(crate) const ST_MSG_ID_IDX: usize = 21; +pub(crate) const ST_INBOUND_MSG_IDX: usize = 20; +pub(crate) const ST_OUTBOUND_MSG_IDX: usize = 21; macro_rules! st_fields_enum { ($(#[$attr:meta])* enum $ty_name:ident { $($name:expr, $var:ident = $discr:expr,)* }) => { @@ -464,14 +466,14 @@ st_fields_enum!(enum StColumnAccessorFields { "accessor_name", AccessorName = 2, }); -st_fields_enum!(enum StInboundMsgIdFields { +st_fields_enum!(enum StInboundMsgFields { "database_identity", DatabaseIdentity = 0, - "last_msg_id", LastMsgId = 1, + "last_outbound_msg", LastMsgId = 1, "result_status", ResultStatus = 2, "result_payload", ResultPayload = 3, }); -st_fields_enum!(enum StMsgIdFields { +st_fields_enum!(enum StOutboundMsgFields { "msg_id", MsgId = 0, "outbox_table_id", OutboxTableId = 1, "row_id", RowId = 2, @@ -695,23 +697,23 @@ fn system_module_def() -> ModuleDef { .with_unique_constraint(st_column_accessor_table_alias_cols) .with_index_no_accessor_name(btree(st_column_accessor_table_alias_cols)); - let st_inbound_msg_id_type = builder.add_type::(); + let st_inbound_msg_type = builder.add_type::(); builder .build_table( - ST_INBOUND_MSG_ID_NAME, - *st_inbound_msg_id_type.as_ref().expect("should be ref"), + ST_INBOUND_MSG_NAME, + *st_inbound_msg_type.as_ref().expect("should be ref"), ) .with_type(TableType::System) - .with_unique_constraint(StInboundMsgIdFields::DatabaseIdentity) - .with_index_no_accessor_name(btree(StInboundMsgIdFields::DatabaseIdentity)) - .with_primary_key(StInboundMsgIdFields::DatabaseIdentity); + .with_unique_constraint(StInboundMsgFields::DatabaseIdentity) + .with_index_no_accessor_name(btree(StInboundMsgFields::DatabaseIdentity)) + .with_primary_key(StInboundMsgFields::DatabaseIdentity); - let st_msg_id_type = builder.add_type::(); + let st_outbound_msg_type = builder.add_type::(); builder - .build_table(ST_MSG_ID_NAME, *st_msg_id_type.as_ref().expect("should be ref")) + .build_table(ST_OUTBOUND_MSG_NAME, *st_outbound_msg_type.as_ref().expect("should be ref")) .with_type(TableType::System) - .with_auto_inc_primary_key(StMsgIdFields::MsgId) - .with_index_no_accessor_name(btree(StMsgIdFields::MsgId)); + .with_auto_inc_primary_key(StOutboundMsgFields::MsgId) + .with_index_no_accessor_name(btree(StOutboundMsgFields::MsgId)); let result = builder .finish() @@ -738,8 +740,8 @@ fn system_module_def() -> ModuleDef { validate_system_table::(&result, ST_TABLE_ACCESSOR_NAME); validate_system_table::(&result, ST_INDEX_ACCESSOR_NAME); validate_system_table::(&result, ST_COLUMN_ACCESSOR_NAME); - validate_system_table::(&result, ST_INBOUND_MSG_ID_NAME); - validate_system_table::(&result, ST_MSG_ID_NAME); + validate_system_table::(&result, ST_INBOUND_MSG_NAME); + validate_system_table::(&result, ST_OUTBOUND_MSG_NAME); result } @@ -788,8 +790,8 @@ lazy_static::lazy_static! { m.insert("st_index_accessor_accessor_name_key", ConstraintId(23)); m.insert("st_column_accessor_table_name_col_name_key", ConstraintId(24)); m.insert("st_column_accessor_table_name_accessor_name_key", ConstraintId(25)); - m.insert("st_inbound_msg_id_database_identity_key", ConstraintId(26)); - m.insert("st_msg_id_msg_id_key", ConstraintId(27)); + m.insert("st_inbound_msg_database_identity_key", ConstraintId(26)); + m.insert("st_outbound_msg_msg_id_key", ConstraintId(27)); m }; } @@ -828,8 +830,8 @@ lazy_static::lazy_static! { m.insert("st_index_accessor_accessor_name_idx_btree", IndexId(27)); m.insert("st_column_accessor_table_name_col_name_idx_btree", IndexId(28)); m.insert("st_column_accessor_table_name_accessor_name_idx_btree", IndexId(29)); - m.insert("st_inbound_msg_id_database_identity_idx_btree", IndexId(30)); - m.insert("st_msg_id_msg_id_idx_btree", IndexId(31)); + m.insert("st_inbound_msg_database_identity_idx_btree", IndexId(30)); + m.insert("st_outbound_msg_msg_id_idx_btree", IndexId(31)); m }; } @@ -846,7 +848,7 @@ lazy_static::lazy_static! { m.insert("st_sequence_sequence_id_seq", SequenceId(5)); m.insert("st_view_view_id_seq", SequenceId(6)); m.insert("st_view_arg_id_seq", SequenceId(7)); - m.insert("st_msg_id_msg_id_seq", SequenceId(8)); + m.insert("st_outbound_msg_msg_id_seq", SequenceId(8)); m }; } @@ -992,12 +994,12 @@ fn st_column_accessor_schema() -> TableSchema { st_schema(ST_COLUMN_ACCESSOR_NAME, ST_COLUMN_ACCESSOR_ID) } -fn st_inbound_msg_id_schema() -> TableSchema { - st_schema(ST_INBOUND_MSG_ID_NAME, ST_INBOUND_MSG_ID_ID) +fn st_inbound_msg_schema() -> TableSchema { + st_schema(ST_INBOUND_MSG_NAME, ST_INBOUND_MSG_ID) } -fn st_msg_id_schema() -> TableSchema { - st_schema(ST_MSG_ID_NAME, ST_MSG_ID_ID) +fn st_outbound_schema() -> TableSchema { + st_schema(ST_OUTBOUND_MSG_NAME, ST_OUTBOUND_MSG_ID) } /// If `table_id` refers to a known system table, return its schema. @@ -1028,8 +1030,8 @@ pub(crate) fn system_table_schema(table_id: TableId) -> Option { ST_TABLE_ACCESSOR_ID => Some(st_table_accessor_schema()), ST_INDEX_ACCESSOR_ID => Some(st_index_accessor_schema()), ST_COLUMN_ACCESSOR_ID => Some(st_column_accessor_schema()), - ST_INBOUND_MSG_ID_ID => Some(st_inbound_msg_id_schema()), - ST_MSG_ID_ID => Some(st_msg_id_schema()), + ST_INBOUND_MSG_ID => Some(st_inbound_msg_schema()), + ST_OUTBOUND_MSG_ID => Some(st_outbound_schema()), _ => None, } } @@ -1921,29 +1923,29 @@ impl From for ProductValue { } } -/// System Table [ST_INBOUND_MSG_ID_NAME] +/// System Table [ST_INBOUND_MSG_NAME] /// Tracks the last message id received from each sender database for deduplication. /// Also stores the result of the reducer call for returning on subsequent duplicate requests. #[derive(Debug, Clone, PartialEq, Eq, SpacetimeType)] #[sats(crate = spacetimedb_lib)] -pub struct StInboundMsgIdRow { +pub struct StInboundMsgRow { pub database_identity: IdentityViaU256, - pub last_msg_id: u64, - /// See [st_inbound_msg_id_result_status] for values. + pub last_outbound_msg: u64, + /// See [st_inbound_msg_result_status] for values. pub result_status: u8, /// Error message if result_status is REDUCER_ERROR, empty otherwise. pub result_payload: String, } -impl TryFrom> for StInboundMsgIdRow { +impl TryFrom> for StInboundMsgRow { type Error = DatastoreError; fn try_from(row: RowRef<'_>) -> Result { read_via_bsatn(row) } } -/// Result status values stored in [ST_INBOUND_MSG_ID_NAME] rows. -pub mod st_inbound_msg_id_result_status { +/// Result status values stored in [ST_INBOUND_MSG_NAME] rows. +pub mod st_inbound_msg_result_status { /// Reducer has been delivered but result not yet received. pub const NOT_YET_RECEIVED: u8 = 0; /// Reducer ran and returned Ok(()). @@ -1952,7 +1954,7 @@ pub mod st_inbound_msg_id_result_status { pub const REDUCER_ERROR: u8 = 2; } -/// System Table [ST_MSG_ID_NAME] +/// System Table [ST_OUTBOUND_MSG_NAME] /// Tracks undelivered outbound inter-database messages (outbox pattern). /// A row's presence means the message has not yet been fully processed. /// Once delivery and the on_result callback complete, the row is deleted. @@ -1961,7 +1963,7 @@ pub mod st_inbound_msg_id_result_status { /// The on_result_reducer is read from the outbox table's schema (TableSchema::on_result_reducer). #[derive(Debug, Clone, PartialEq, Eq, SpacetimeType)] #[sats(crate = spacetimedb_lib)] -pub struct StMsgIdRow { +pub struct StOutboundMsgRow { pub msg_id: u64, /// The TableId of the `__outbox_` table that generated this message. pub outbox_table_id: u32, @@ -1969,7 +1971,7 @@ pub struct StMsgIdRow { pub row_id: u64, } -impl TryFrom> for StMsgIdRow { +impl TryFrom> for StOutboundMsgRow { type Error = DatastoreError; fn try_from(row: RowRef<'_>) -> Result { read_via_bsatn(row) diff --git a/crates/lib/src/db/raw_def/v10.rs b/crates/lib/src/db/raw_def/v10.rs index 1e615c6b0b9..44fe45a6889 100644 --- a/crates/lib/src/db/raw_def/v10.rs +++ b/crates/lib/src/db/raw_def/v10.rs @@ -274,7 +274,7 @@ pub struct RawColumnDefaultValueV10 { /// Marks a table as an outbox table for inter-database communication. /// /// The table must have: -/// - Col 0: `u64` with `#[primary_key] #[auto_inc]` — the row ID stored in `st_msg_id`. +/// - Col 0: `u64` with `#[primary_key] #[auto_inc]` — the row ID stored in `st_outbound_msg`. /// - Col 1: `Identity` (encoded as U256) — the target database identity. /// - Remaining cols: arguments forwarded verbatim to the remote reducer. /// diff --git a/crates/schema/src/def.rs b/crates/schema/src/def.rs index 807de4cbf38..12070c5c0ed 100644 --- a/crates/schema/src/def.rs +++ b/crates/schema/src/def.rs @@ -713,7 +713,7 @@ pub struct TableDef { /// Configuration for an outbox table used in inter-database communication. /// /// An outbox table must have: -/// - Col 0: `u64` with `#[primary_key] #[auto_inc]` — row ID tracked in `st_msg_id`. +/// - Col 0: `u64` with `#[primary_key] #[auto_inc]` — row ID tracked in `st_outbound_msg`. /// - Col 1: `Identity` — target database identity. /// - Remaining cols: arguments forwarded to `remote_reducer` on the target database. #[derive(Debug, Clone, PartialEq, Eq)] From 4d68dd604e42ca1e743f1024066450039603f8bd Mon Sep 17 00:00:00 2001 From: Shubham Mishra Date: Fri, 27 Mar 2026 22:31:29 +0530 Subject: [PATCH 09/23] fix macro --- crates/bindings-macro/src/table.rs | 43 +++++++++++++++++++ crates/bindings/src/rt.rs | 10 +++++ crates/core/src/host/idc_runtime.rs | 20 +++++++-- .../src/locking_tx_datastore/mut_tx.rs | 35 --------------- crates/schema/src/schema.rs | 28 +++++++----- crates/table/src/table.rs | 6 +-- 6 files changed, 91 insertions(+), 51 deletions(-) diff --git a/crates/bindings-macro/src/table.rs b/crates/bindings-macro/src/table.rs index 2fa1f285cba..ba41bc57abc 100644 --- a/crates/bindings-macro/src/table.rs +++ b/crates/bindings-macro/src/table.rs @@ -1131,6 +1131,9 @@ pub(crate) fn table_impl(mut args: TableArgs, item: &syn::DeriveInput) -> syn::R } }; + // Save a clone before the schedule closure captures `primary_key_column` by move. + let primary_key_column_for_outbox = primary_key_column.clone(); + let (schedule, schedule_typecheck) = args .scheduled .as_ref() @@ -1189,6 +1192,44 @@ pub(crate) fn table_impl(mut args: TableArgs, item: &syn::DeriveInput) -> syn::R .unzip(); let schedule = schedule.into_iter(); + let (outbox, outbox_typecheck) = args + .outbox + .as_ref() + .map(|ob| { + let primary_key_column = primary_key_column_for_outbox.ok_or_else(|| { + syn::Error::new( + ob.span, + "outbox tables must have a `#[primary_key] #[auto_inc]` u64 column at position 0 (the row_id)", + ) + })?; + if primary_key_column.index != 0 { + return Err(syn::Error::new( + ob.span, + "outbox tables must have the `#[primary_key] #[auto_inc]` column as the first column (col 0)", + )); + } + + let remote_reducer = &ob.remote_reducer; + let on_result_reducer_name = match &ob.on_result_reducer { + Some(r) => quote!(Some(<#r as spacetimedb::rt::FnInfo>::NAME)), + None => quote!(None), + }; + let desc = quote!(spacetimedb::table::OutboxDesc { + remote_reducer_name: <#remote_reducer as spacetimedb::rt::FnInfo>::NAME, + on_result_reducer_name: #on_result_reducer_name, + }); + + let primary_key_ty = primary_key_column.ty; + let typecheck = quote! { + spacetimedb::rt::assert_outbox_table_primary_key::<#primary_key_ty>(); + }; + + Ok((desc, typecheck)) + }) + .transpose()? + .unzip(); + let outbox = outbox.into_iter(); + let unique_err = if !unique_columns.is_empty() { quote!(spacetimedb::UniqueConstraintViolation) } else { @@ -1222,6 +1263,7 @@ pub(crate) fn table_impl(mut args: TableArgs, item: &syn::DeriveInput) -> syn::R #(const PRIMARY_KEY: Option = Some(#primary_col_id);)* const SEQUENCES: &'static [u16] = &[#(#sequence_col_ids),*]; #(const SCHEDULE: Option> = Some(#schedule);)* + #(const OUTBOX: Option> = Some(#outbox);)* #table_id_from_name_func #default_fn @@ -1373,6 +1415,7 @@ pub(crate) fn table_impl(mut args: TableArgs, item: &syn::DeriveInput) -> syn::R const _: () = { #(let _ = <#field_types as spacetimedb::rt::TableColumn>::_ITEM;)* #schedule_typecheck + #outbox_typecheck #default_type_check }; diff --git a/crates/bindings/src/rt.rs b/crates/bindings/src/rt.rs index 108e9a111a1..7e24468e8ce 100644 --- a/crates/bindings/src/rt.rs +++ b/crates/bindings/src/rt.rs @@ -533,6 +533,9 @@ impl TableColumn for T {} /// Assert that the primary_key column of a scheduled table is a u64. pub const fn assert_scheduled_table_primary_key() {} +/// Assert that the primary_key column of an outbox table is a u64. +pub const fn assert_outbox_table_primary_key() {} + /// Verify at compile time that a function has the correct signature for an outbox `on_result` reducer. /// /// The reducer must accept `(OutboxRow, Result<(), String>)` as its user-supplied arguments: @@ -555,6 +558,13 @@ pub trait ScheduledTablePrimaryKey: sealed::Sealed {} impl sealed::Sealed for u64 {} impl ScheduledTablePrimaryKey for u64 {} +#[diagnostic::on_unimplemented( + message = "outbox table primary key must be a `u64`", + label = "should be `u64`, not `{Self}`" +)] +pub trait OutboxTablePrimaryKey: sealed::Sealed {} +impl OutboxTablePrimaryKey for u64 {} + /// Used in the last type parameter of `Reducer` to indicate that the /// context argument *should* be passed to the reducer logic. pub struct ContextArg; diff --git a/crates/core/src/host/idc_runtime.rs b/crates/core/src/host/idc_runtime.rs index 3fd8dd51b2c..7573b622680 100644 --- a/crates/core/src/host/idc_runtime.rs +++ b/crates/core/src/host/idc_runtime.rs @@ -77,7 +77,11 @@ impl IdcRuntime { #[derive(Clone)] struct PendingMessage { msg_id: u64, + /// Stored for future use (e.g. deleting the outbox row after delivery). + #[allow(dead_code)] outbox_table_id: TableId, + /// Stored for future use (e.g. deleting the outbox row after delivery). + #[allow(dead_code)] row_id: u64, target_db_identity: Identity, target_reducer: String, @@ -325,9 +329,19 @@ fn load_pending_into_targets(db: &RelationalDB, targets: &mut HashMap o, + None => { + log::error!( + "idc_runtime: table {:?} (msg_id={}) is not an outbox table", + schema.table_name, + st_row.msg_id, + ); + continue; + } + }; + let target_reducer = outbox_schema.remote_reducer.to_string(); + let on_result_reducer = outbox_schema.on_result_reducer.as_ref().map(|id| id.to_string()); // Look up the outbox row by its auto-inc PK (col 0) to get target identity and args. let outbox_row = db diff --git a/crates/datastore/src/locking_tx_datastore/mut_tx.rs b/crates/datastore/src/locking_tx_datastore/mut_tx.rs index b5c72ea22b7..29d4e1ea8b4 100644 --- a/crates/datastore/src/locking_tx_datastore/mut_tx.rs +++ b/crates/datastore/src/locking_tx_datastore/mut_tx.rs @@ -44,7 +44,6 @@ use smallvec::SmallVec; use spacetimedb_data_structures::map::{HashMap, HashSet, IntMap}; use spacetimedb_durability::TxOffset; use spacetimedb_execution::{dml::MutDatastore, Datastore, DeltaStore, Row}; -use spacetimedb_lib::bsatn::ToBsatn; use spacetimedb_lib::{db::raw_def::v9::RawSql, metrics::ExecutionMetrics, Timestamp}; use spacetimedb_lib::{ db::{auth::StAccess, raw_def::SEQUENCE_ALLOCATION_STEP}, @@ -2777,40 +2776,6 @@ impl MutTxId { }) } - /// Returns the table IDs and table names of all tables that had rows inserted - /// in this transaction and whose name starts with `"__outbox_"`. - /// - /// Used by the IDC runtime to detect outbox inserts and enqueue ST_OUTBOUND_MSG entries. - pub fn outbox_insert_table_ids(&self) -> Vec<(TableId, String)> { - let mut result = Vec::new(); - for table_id in self.tx_state.insert_tables.keys() { - if let Ok(schema) = self.schema_for_table(*table_id) { - let name = schema.table_name.to_string(); - if name.starts_with("__outbox_") { - result.push((*table_id, name)); - } - } - } - result - } - - /// Returns the raw BSATN bytes of all rows inserted into `table_id` in this transaction. - /// - /// Each `Vec` encodes a full outbox row. The first 32 bytes are the target - /// database identity (U256 little-endian), and the remaining bytes are the reducer args. - pub fn outbox_inserts_for_table(&mut self, table_id: TableId) -> Vec> { - let Some(inserted_table) = self.tx_state.insert_tables.get(&table_id) else { - return Vec::new(); - }; - // Collect blob-store-independent data: we hold a shared ref to insert_tables, - // so we can't also borrow committed_state mutably for the blob store. - // Use the blob store embedded in the TxState. - let blob_store = &self.tx_state.blob_store; - inserted_table - .scan_rows(blob_store) - .filter_map(|row_ref| row_ref.to_bsatn_vec().ok()) - .collect() - } pub fn insert_via_serialize_bsatn<'a, T: Serialize>( &'a mut self, diff --git a/crates/schema/src/schema.rs b/crates/schema/src/schema.rs index fd564f686c2..27349949046 100644 --- a/crates/schema/src/schema.rs +++ b/crates/schema/src/schema.rs @@ -201,9 +201,8 @@ pub struct TableSchema { /// Whether this is an event table. pub is_event: bool, - /// For outbox tables (`__outbox_`): the name of the local reducer to call - /// with the result of the remote reducer invocation. `None` means no callback. - pub on_result_reducer: Option, + /// Outbox configuration if this is an outbox table; `None` for non-outbox tables. + pub outbox: Option, /// Cache for `row_type_for_table` in the data store. pub row_type: ProductType, @@ -231,7 +230,7 @@ impl TableSchema { primary_key: Option, is_event: bool, alias: Option, - on_result_reducer: Option, + outbox: Option, ) -> Self { Self { row_type: columns_to_row_type(&columns), @@ -248,7 +247,7 @@ impl TableSchema { primary_key, is_event, alias, - on_result_reducer, + outbox, } } @@ -1041,10 +1040,10 @@ impl Schema for TableSchema { .as_ref() .map(|schedule| ScheduleSchema::from_module_def(module_def, schedule, table_id, ScheduleId::SENTINEL)); - let on_result_reducer = outbox - .as_ref() - .and_then(|o| o.on_result_reducer.as_ref()) - .map(|r| r.to_string()); + let outbox_schema = outbox.as_ref().map(|o| OutboxSchema { + remote_reducer: o.remote_reducer.clone(), + on_result_reducer: o.on_result_reducer.clone(), + }); TableSchema::new( table_id, @@ -1060,7 +1059,7 @@ impl Schema for TableSchema { *primary_key, *is_event, Some(accessor_name.clone()), - on_result_reducer, + outbox_schema, ) } @@ -1443,6 +1442,15 @@ impl Schema for ScheduleSchema { } } +/// Marks a table as an outbox table for inter-database communication. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OutboxSchema { + /// The name of the reducer to invoke on the target database. + pub remote_reducer: Identifier, + /// The local reducer called with the delivery result, if any. + pub on_result_reducer: Option, +} + /// A struct representing the schema of a database index. #[derive(Debug, Clone, PartialEq, Eq)] pub struct IndexSchema { diff --git a/crates/table/src/table.rs b/crates/table/src/table.rs index 2414e398df3..7ce6e23ee9e 100644 --- a/crates/table/src/table.rs +++ b/crates/table/src/table.rs @@ -108,7 +108,7 @@ pub struct Table { is_scheduler: bool, /// Indicates whether this is an outbox table or not. /// - /// Outbox tables follow the naming convention `__outbox_`. + /// Set from `schema.outbox.is_some()`. /// This is an optimization to avoid checking the schema in e.g., `InstanceEnv::insert`. is_outbox: bool, } @@ -549,7 +549,7 @@ impl Table { self.is_scheduler } - /// Returns whether this is an outbox table (`__outbox_`). + /// Returns whether this is an outbox table (i.e., `schema.outbox.is_some()`). pub fn is_outbox(&self) -> bool { self.is_outbox } @@ -2265,7 +2265,7 @@ impl Table { squashed_offset: SquashedOffset, pointer_map: Option, ) -> Self { - let is_outbox = schema.table_name.starts_with("__outbox_"); + let is_outbox = schema.outbox.is_some(); Self { inner: TableInner { row_layout, From 6fced4200ed35441c543ffa3978baaf97544811b Mon Sep 17 00:00:00 2001 From: Shubham Mishra Date: Tue, 7 Apr 2026 13:02:22 +0530 Subject: [PATCH 10/23] fixes --- Cargo.lock | 1 + crates/client-api/src/routes/database.rs | 12 +- crates/core/src/host/host_controller.rs | 23 ++-- .../src/host/{idc_runtime.rs => idc_actor.rs} | 75 +++++++------ crates/core/src/host/instance_env.rs | 14 +-- crates/core/src/host/mod.rs | 2 +- crates/core/src/host/module_common.rs | 8 +- crates/core/src/host/module_host.rs | 89 +++++++++++++++ crates/core/src/host/v8/mod.rs | 48 ++++++++ .../src/host/wasm_common/module_host_actor.rs | 35 +++++- crates/core/src/module_host_context.rs | 4 +- crates/smoketests/Cargo.toml | 1 + crates/smoketests/src/lib.rs | 11 ++ crates/smoketests/tests/smoketests/idc.rs | 103 ++++++++++++++++++ crates/smoketests/tests/smoketests/mod.rs | 1 + 15 files changed, 363 insertions(+), 64 deletions(-) rename crates/core/src/host/{idc_runtime.rs => idc_actor.rs} (86%) create mode 100644 crates/smoketests/tests/smoketests/idc.rs diff --git a/Cargo.lock b/Cargo.lock index 03618187864..4a666d7fa2d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8578,6 +8578,7 @@ dependencies = [ "serde_json", "spacetimedb-core", "spacetimedb-guard", + "spacetimedb-lib 2.1.0", "tempfile", "tokio", "tokio-postgres", diff --git a/crates/client-api/src/routes/database.rs b/crates/client-api/src/routes/database.rs index ae091f56213..655b40f08ec 100644 --- a/crates/client-api/src/routes/database.rs +++ b/crates/client-api/src/routes/database.rs @@ -287,7 +287,7 @@ pub async fn call_from_database( .await .map_err(client_connected_error_to_response)?; - let result = module + let mut result = module .call_reducer_from_database( caller_identity, Some(connection_id), @@ -301,6 +301,14 @@ pub async fn call_from_database( ) .await; + if let Ok(rcr) = result.as_mut() + && let Some(tx_offset) = rcr.tx_offset.as_mut() + && let Some(mut durable_offset) = module.durable_tx_offset() + { + let tx_offset = tx_offset.await.map_err(|_| log_and_500("transaction aborted"))?; + durable_offset.wait_for(tx_offset).await.map_err(log_and_500)?; + } + module .call_identity_disconnected(caller_identity, connection_id) .await @@ -311,7 +319,7 @@ pub async fn call_from_database( let (status, body) = match rcr.outcome { ReducerOutcome::Committed => (StatusCode::OK, "".into()), ReducerOutcome::Deduplicated => (StatusCode::OK, "deduplicated".into()), - // 422 = reducer ran but returned Err; IDC runtime uses this to distinguish + // 422 = reducer ran but returned Err; the IDC actor uses this to distinguish // reducer failures from transport errors (which it retries). ReducerOutcome::Failed(errmsg) => (StatusCode::UNPROCESSABLE_ENTITY, *errmsg), ReducerOutcome::BudgetExceeded => { diff --git a/crates/core/src/host/host_controller.rs b/crates/core/src/host/host_controller.rs index c0f6073e9db..5389086909c 100644 --- a/crates/core/src/host/host_controller.rs +++ b/crates/core/src/host/host_controller.rs @@ -1,4 +1,4 @@ -use super::idc_runtime::{IdcRuntime, IdcRuntimeConfig, IdcRuntimeStarter, IdcSender}; +use super::idc_actor::{IdcActor, IdcActorConfig, IdcActorSender, IdcActorStarter}; use super::module_host::{EventStatus, ModuleHost, ModuleInfo, NoSuchModule}; use super::scheduler::SchedulerStarter; use super::wasmtime::WasmtimeRuntime; @@ -133,11 +133,12 @@ impl HostRuntimes { } } -#[derive(Clone, Debug)] +#[derive(Debug)] pub struct ReducerCallResult { pub outcome: ReducerOutcome, pub energy_used: EnergyQuanta, pub execution_duration: Duration, + pub tx_offset: Option, } impl ReducerCallResult { @@ -710,7 +711,7 @@ async fn make_module_host( runtimes: Arc, replica_ctx: Arc, scheduler: Scheduler, - idc_sender: IdcSender, + idc_sender: IdcActorSender, program: Program, energy_monitor: Arc, unregister: impl Fn() + Send + Sync + 'static, @@ -764,7 +765,7 @@ struct LaunchedModule { module_host: ModuleHost, scheduler: Scheduler, scheduler_starter: SchedulerStarter, - idc_starter: IdcRuntimeStarter, + idc_starter: IdcActorStarter, } struct ModuleLauncher { @@ -801,7 +802,7 @@ impl ModuleLauncher { .await .map(Arc::new)?; let (scheduler, scheduler_starter) = Scheduler::open(replica_ctx.relational_db().clone()); - let (idc_starter, idc_sender) = IdcRuntime::open(); + let (idc_starter, idc_sender) = IdcActor::open(); let (program, module_host) = make_module_host( self.runtimes.clone(), replica_ctx.clone(), @@ -889,9 +890,9 @@ struct Host { /// Handle to the task responsible for cleaning up old views. /// The task is aborted when [`Host`] is dropped. view_cleanup_task: AbortHandle, - /// IDC runtime: delivers outbound inter-database messages from `st_outbound_msg`. + /// IDC actor: delivers outbound inter-database messages from `st_outbound_msg`. /// Stopped when [`Host`] is dropped. - _idc_runtime: IdcRuntime, + _idc_actor: IdcActor, } impl Host { @@ -1112,9 +1113,9 @@ impl Host { let disk_metrics_recorder_task = tokio::spawn(metric_reporter(replica_ctx.clone())).abort_handle(); let view_cleanup_task = spawn_view_cleanup_loop(replica_ctx.relational_db().clone()); - let idc_runtime = idc_starter.start( + let idc_actor = idc_starter.start( replica_ctx.relational_db().clone(), - IdcRuntimeConfig { + IdcActorConfig { sender_identity: replica_ctx.database_identity, }, module_host.downgrade(), @@ -1129,7 +1130,7 @@ impl Host { disk_metrics_recorder_task, tx_metrics_recorder_task, view_cleanup_task, - _idc_runtime: idc_runtime, + _idc_actor: idc_actor, }) } @@ -1202,7 +1203,7 @@ impl Host { ) -> anyhow::Result { let replica_ctx = &self.replica_ctx; let (scheduler, scheduler_starter) = Scheduler::open(self.replica_ctx.relational_db().clone()); - let (_idc_starter, idc_sender) = IdcRuntime::open(); + let (_idc_starter, idc_sender) = IdcActor::open(); let (program, module) = make_module_host( runtimes, diff --git a/crates/core/src/host/idc_runtime.rs b/crates/core/src/host/idc_actor.rs similarity index 86% rename from crates/core/src/host/idc_runtime.rs rename to crates/core/src/host/idc_actor.rs index 7573b622680..06ab9c0c357 100644 --- a/crates/core/src/host/idc_runtime.rs +++ b/crates/core/src/host/idc_actor.rs @@ -1,4 +1,4 @@ -/// Inter-Database Communication (IDC) Runtime +/// Inter-Database Communication (IDC) Actor /// /// Background tokio task that: /// 1. Loads undelivered entries from `st_outbound_msg` on startup, resolving delivery data from outbox tables. @@ -29,47 +29,47 @@ const MAX_BACKOFF: Duration = Duration::from_secs(30); /// How long to wait before polling again when there is no work. const POLL_INTERVAL: Duration = Duration::from_millis(500); -/// A sender that notifies the IDC runtime of a new outbox row. +/// A sender that notifies the IDC actor of a new outbox row. /// -/// Sending `()` wakes the runtime to deliver pending messages immediately +/// Sending `()` wakes the actor to deliver pending messages immediately /// rather than waiting for the next poll cycle. -pub type IdcSender = mpsc::UnboundedSender<()>; +pub type IdcActorSender = mpsc::UnboundedSender<()>; -/// The identity of this (sender) database, set when the IDC runtime is started. -pub struct IdcRuntimeConfig { +/// The identity of this (sender) database, set when the IDC actor is started. +pub struct IdcActorConfig { pub sender_identity: Identity, } -/// A handle that, when dropped, stops the IDC runtime background task. -pub struct IdcRuntime { +/// A handle that, when dropped, stops the IDC actor background task. +pub struct IdcActor { _abort: tokio::task::AbortHandle, } -/// Holds the receiver side of the notification channel until the runtime is started. +/// Holds the receiver side of the notification channel until the actor is started. /// /// Mirrors the `SchedulerStarter` pattern: create the channel before the module is -/// loaded (so the sender can be stored in `InstanceEnv`), then call [`IdcRuntimeStarter::start`] +/// loaded (so the sender can be stored in `InstanceEnv`), then call [`IdcActorStarter::start`] /// once the DB is ready. -pub struct IdcRuntimeStarter { +pub struct IdcActorStarter { rx: mpsc::UnboundedReceiver<()>, } -impl IdcRuntimeStarter { - /// Spawn the IDC runtime background task. - pub fn start(self, db: Arc, config: IdcRuntimeConfig, module_host: WeakModuleHost) -> IdcRuntime { +impl IdcActorStarter { + /// Spawn the IDC actor background task. + pub fn start(self, db: Arc, config: IdcActorConfig, module_host: WeakModuleHost) -> IdcActor { let abort = tokio::spawn(run_idc_loop(db, config, module_host, self.rx)).abort_handle(); - IdcRuntime { _abort: abort } + IdcActor { _abort: abort } } } -impl IdcRuntime { +impl IdcActor { /// Open the IDC channel, returning a starter and a sender. /// /// Store the sender in `ModuleCreationContext` so it reaches `InstanceEnv`. - /// After the module is ready, call [`IdcRuntimeStarter::start`] to spawn the loop. - pub fn open() -> (IdcRuntimeStarter, IdcSender) { + /// After the module is ready, call [`IdcActorStarter::start`] to spawn the loop. + pub fn open() -> (IdcActorStarter, IdcActorSender) { let (tx, rx) = mpsc::unbounded_channel(); - (IdcRuntimeStarter { rx }, tx) + (IdcActorStarter { rx }, tx) } } @@ -138,10 +138,10 @@ enum DeliveryOutcome { TransportError(String), } -/// Main IDC loop: maintain per-target queues and deliver messages. +/// Main IDC actor loop: maintain per-target queues and deliver messages. async fn run_idc_loop( db: Arc, - config: IdcRuntimeConfig, + config: IdcActorConfig, module_host: WeakModuleHost, mut notify_rx: mpsc::UnboundedReceiver<()>, ) { @@ -169,7 +169,7 @@ async fn run_idc_loop( match outcome { DeliveryOutcome::TransportError(reason) => { log::warn!( - "idc_runtime: transport error delivering msg_id={} to {}: {reason}", + "idc_actor: transport error delivering msg_id={} to {}: {reason}", msg.msg_id, hex::encode(msg.target_db_identity.to_byte_array()), ); @@ -224,6 +224,9 @@ fn outcome_to_result(outcome: &DeliveryOutcome) -> (u8, String) { } /// Finalize a delivered message: call the on_result reducer (if any), then delete from ST_OUTBOUND_MSG. +/// +/// On the happy path, `on_result_reducer` success and deletion of `st_outbound_msg` +/// are committed atomically in the same reducer transaction. async fn finalize_message( db: &RelationalDB, module_host: &WeakModuleHost, @@ -235,7 +238,7 @@ async fn finalize_message( if let Some(on_result_reducer) = &msg.on_result_reducer { let Some(host) = module_host.upgrade() else { log::warn!( - "idc_runtime: module host gone, cannot call on_result reducer '{}' for msg_id={}", + "idc_actor: module host gone, cannot call on_result reducer '{}' for msg_id={}", on_result_reducer, msg.msg_id, ); @@ -249,7 +252,7 @@ async fn finalize_message( Ok(b) => b, Err(e) => { log::error!( - "idc_runtime: failed to encode on_result args for msg_id={}: {e}", + "idc_actor: failed to encode on_result args for msg_id={}: {e}", msg.msg_id ); delete_message(db, msg.msg_id); @@ -259,7 +262,7 @@ async fn finalize_message( let caller_identity = Identity::ZERO; // system call let result = host - .call_reducer( + .call_reducer_delete_outbound_on_success( caller_identity, None, // no connection_id None, // no client sender @@ -267,20 +270,22 @@ async fn finalize_message( None, // no timer on_result_reducer, FunctionArgs::Bsatn(bytes::Bytes::from(args_bytes)), + msg.msg_id, ) .await; match result { Ok(_) => { log::debug!( - "idc_runtime: on_result reducer '{}' called for msg_id={}", + "idc_actor: on_result reducer '{}' called for msg_id={}", on_result_reducer, msg.msg_id, ); + return; } Err(e) => { log::error!( - "idc_runtime: on_result reducer '{}' failed for msg_id={}: {e:?}", + "idc_actor: on_result reducer '{}' failed for msg_id={}: {e:?}", on_result_reducer, msg.msg_id, ); @@ -307,7 +312,7 @@ fn load_pending_into_targets(db: &RelationalDB, targets: &mut HashMap s, Err(e) => { log::error!( - "idc_runtime: cannot find schema for outbox table {:?} (msg_id={}): {e}", + "idc_actor: cannot find schema for outbox table {:?} (msg_id={}): {e}", outbox_table_id, st_row.msg_id, ); @@ -333,7 +338,7 @@ fn load_pending_into_targets(db: &RelationalDB, targets: &mut HashMap o, None => { log::error!( - "idc_runtime: table {:?} (msg_id={}) is not an outbox table", + "idc_actor: table {:?} (msg_id={}) is not an outbox table", schema.table_name, st_row.msg_id, ); @@ -351,7 +356,7 @@ fn load_pending_into_targets(db: &RelationalDB, targets: &mut HashMap Identity::from_u256(**u), other => { log::error!( - "idc_runtime: outbox row col 1 expected U256 (Identity), got {other:?} (msg_id={})", + "idc_actor: outbox row col 1 expected U256 (Identity), got {other:?} (msg_id={})", st_row.msg_id, ); continue; @@ -409,7 +414,7 @@ fn load_pending_into_targets(db: &RelationalDB, targets: &mut HashMap DeliveryOutcome { let target_db_hex = hex::encode(msg.target_db_identity.to_byte_array()); @@ -453,11 +458,11 @@ async fn attempt_delivery( fn delete_message(db: &RelationalDB, msg_id: u64) { let mut tx = db.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); if let Err(e) = tx.delete_outbound_msg(msg_id) { - log::error!("idc_runtime: failed to delete msg_id={msg_id}: {e}"); + log::error!("idc_actor: failed to delete msg_id={msg_id}: {e}"); let _ = db.rollback_mut_tx(tx); return; } if let Err(e) = db.commit_tx(tx) { - log::error!("idc_runtime: failed to commit delete for msg_id={msg_id}: {e}"); + log::error!("idc_actor: failed to commit delete for msg_id={msg_id}: {e}"); } } diff --git a/crates/core/src/host/instance_env.rs b/crates/core/src/host/instance_env.rs index da5230e9866..217415878f5 100644 --- a/crates/core/src/host/instance_env.rs +++ b/crates/core/src/host/instance_env.rs @@ -1,4 +1,4 @@ -use super::idc_runtime::IdcSender; +use super::idc_actor::IdcActorSender; use super::scheduler::{get_schedule_from_row, ScheduleError, Scheduler}; use crate::database_logger::{BacktraceFrame, BacktraceProvider, LogLevel, ModuleBacktrace, Record}; use crate::db::relational_db::{MutTx, RelationalDB}; @@ -41,8 +41,8 @@ use std::vec::IntoIter; pub struct InstanceEnv { pub replica_ctx: Arc, pub scheduler: Scheduler, - /// Sender to notify the IDC runtime of new outbox rows. - pub idc_sender: IdcSender, + /// Sender to notify the IDC actor of new outbox rows. + pub idc_sender: IdcActorSender, pub tx: TxSlot, /// The timestamp the current function began running. pub start_time: Timestamp, @@ -227,7 +227,7 @@ impl ChunkedWriter { // Generic 'instance environment' delegated to from various host types. impl InstanceEnv { - pub fn new(replica_ctx: Arc, scheduler: Scheduler, idc_sender: IdcSender) -> Self { + pub fn new(replica_ctx: Arc, scheduler: Scheduler, idc_sender: IdcActorSender) -> Self { Self { replica_ctx, scheduler, @@ -433,7 +433,7 @@ impl InstanceEnv { } /// Enqueue an outbox row into ST_OUTBOUND_MSG atomically within the current transaction, - /// and notify the IDC runtime so it delivers without waiting for the next poll cycle. + /// and notify the IDC actor so it delivers without waiting for the next poll cycle. /// /// Outbox tables follow the naming convention `__outbox_` and have: /// - Col 0: auto-inc primary key (u64) — stored as `row_id` in ST_OUTBOUND_MSG. @@ -468,7 +468,7 @@ impl InstanceEnv { tx.insert_st_outbound_msg(table_id.0, row_id).map_err(DBError::from)?; - // Wake the IDC runtime immediately so it doesn't wait for the next poll cycle. + // Wake the IDC actor immediately so it doesn't wait for the next poll cycle. let _ = self.idc_sender.send(()); Ok(()) @@ -1416,7 +1416,7 @@ mod test { fn instance_env(db: Arc) -> Result<(InstanceEnv, tokio::runtime::Runtime)> { let (scheduler, _) = Scheduler::open(db.clone()); let (replica_context, runtime) = replica_ctx(db)?; - let (_, idc_sender) = crate::host::idc_runtime::IdcRuntime::open(); + let (_, idc_sender) = crate::host::idc_actor::IdcActor::open(); Ok(( InstanceEnv::new(Arc::new(replica_context), scheduler, idc_sender), runtime, diff --git a/crates/core/src/host/mod.rs b/crates/core/src/host/mod.rs index 5125c35e6de..ec3133cc5dd 100644 --- a/crates/core/src/host/mod.rs +++ b/crates/core/src/host/mod.rs @@ -12,7 +12,7 @@ use spacetimedb_schema::def::ModuleDef; mod disk_storage; mod host_controller; -pub mod idc_runtime; +pub mod idc_actor; mod module_common; #[allow(clippy::too_many_arguments)] pub mod module_host; diff --git a/crates/core/src/host/module_common.rs b/crates/core/src/host/module_common.rs index 0f1ff4b75cb..548922234d9 100644 --- a/crates/core/src/host/module_common.rs +++ b/crates/core/src/host/module_common.rs @@ -4,7 +4,7 @@ use crate::{ energy::EnergyMonitor, host::{ - idc_runtime::IdcSender, + idc_actor::IdcActorSender, module_host::ModuleInfo, wasm_common::{module_host_actor::DescribeError, DESCRIBE_MODULE_DUNDER}, Scheduler, @@ -52,7 +52,7 @@ pub fn build_common_module_from_raw( pub(crate) struct ModuleCommon { replica_context: Arc, scheduler: Scheduler, - idc_sender: IdcSender, + idc_sender: IdcActorSender, info: Arc, energy_monitor: Arc, } @@ -62,7 +62,7 @@ impl ModuleCommon { fn new( replica_context: Arc, scheduler: Scheduler, - idc_sender: IdcSender, + idc_sender: IdcActorSender, info: Arc, energy_monitor: Arc, ) -> Self { @@ -100,7 +100,7 @@ impl ModuleCommon { &self.scheduler } - pub fn idc_sender(&self) -> IdcSender { + pub fn idc_sender(&self) -> IdcActorSender { self.idc_sender.clone() } } diff --git a/crates/core/src/host/module_host.rs b/crates/core/src/host/module_host.rs index a582d521393..4ac99831775 100644 --- a/crates/core/src/host/module_host.rs +++ b/crates/core/src/host/module_host.rs @@ -1617,6 +1617,43 @@ impl ModuleHost { .await? } + async fn call_reducer_delete_outbound_on_success_inner( + &self, + caller_identity: Identity, + caller_connection_id: Option, + client: Option>, + request_id: Option, + timer: Option, + reducer_id: ReducerId, + reducer_def: &ReducerDef, + args: FunctionArgs, + msg_id: u64, + ) -> Result { + let args = args + .into_tuple_for_def(&self.info.module_def, reducer_def) + .map_err(InvalidReducerArguments)?; + let caller_connection_id = caller_connection_id.unwrap_or(ConnectionId::ZERO); + let call_reducer_params = CallReducerParams { + timestamp: Timestamp::now(), + caller_identity, + caller_connection_id, + client, + request_id, + timer, + reducer_id, + args, + dedup_sender: None, + }; + + self.call( + &reducer_def.name, + (call_reducer_params, msg_id), + async |(p, msg_id), inst| Ok(inst.call_reducer_delete_outbound_on_success(p, msg_id)), + async |(p, msg_id), inst| inst.call_reducer_delete_outbound_on_success(p, msg_id).await, + ) + .await? + } + pub async fn call_reducer( &self, caller_identity: Identity, @@ -1667,6 +1704,58 @@ impl ModuleHost { res } + pub async fn call_reducer_delete_outbound_on_success( + &self, + caller_identity: Identity, + caller_connection_id: Option, + client: Option>, + request_id: Option, + timer: Option, + reducer_name: &str, + args: FunctionArgs, + msg_id: u64, + ) -> Result { + let res = async { + let (reducer_id, reducer_def) = self + .info + .module_def + .reducer_full(reducer_name) + .ok_or(ReducerCallError::NoSuchReducer)?; + if let Some(lifecycle) = reducer_def.lifecycle { + return Err(ReducerCallError::LifecycleReducer(lifecycle)); + } + + if reducer_def.visibility.is_private() && !self.is_database_owner(caller_identity) { + return Err(ReducerCallError::NoSuchReducer); + } + + self.call_reducer_delete_outbound_on_success_inner( + caller_identity, + caller_connection_id, + client, + request_id, + timer, + reducer_id, + reducer_def, + args, + msg_id, + ) + .await + } + .await; + + let log_message = match &res { + Err(ReducerCallError::NoSuchReducer) => Some(no_such_function_log_message("reducer", reducer_name)), + Err(ReducerCallError::Args(_)) => Some(args_error_log_message("reducer", reducer_name)), + _ => None, + }; + if let Some(log_message) = log_message { + self.inject_logs(LogLevel::Error, reducer_name, &log_message) + } + + res + } + /// Variant of [`Self::call_reducer`] for database-to-database calls. /// /// Behaves identically to `call_reducer`, except that it enforces at-most-once diff --git a/crates/core/src/host/v8/mod.rs b/crates/core/src/host/v8/mod.rs index c5d1964bc5b..993ed9f7d95 100644 --- a/crates/core/src/host/v8/mod.rs +++ b/crates/core/src/host/v8/mod.rs @@ -468,6 +468,20 @@ impl JsInstance { .unwrap_or_else(|_| panic!("worker should stay live while calling a reducer")) } + pub async fn call_reducer_delete_outbound_on_success( + &self, + params: CallReducerParams, + msg_id: u64, + ) -> ReducerCallResult { + self.send_request(|reply_tx| JsWorkerRequest::CallReducerDeleteOutboundOnSuccess { + reply_tx, + params, + msg_id, + }) + .await + .unwrap_or_else(|_| panic!("worker should stay live while calling a reducer")) + } + pub async fn clear_all_clients(&self) -> anyhow::Result<()> { self.send_request(JsWorkerRequest::ClearAllClients) .await @@ -572,6 +586,12 @@ enum JsWorkerRequest { reply_tx: JsReplyTx, params: CallReducerParams, }, + /// See [`JsInstance::call_reducer_delete_outbound_on_success`]. + CallReducerDeleteOutboundOnSuccess { + reply_tx: JsReplyTx, + params: CallReducerParams, + msg_id: u64, + }, /// See [`JsInstance::call_view`]. CallView { reply_tx: JsReplyTx, @@ -888,6 +908,23 @@ impl JsInstanceLane { .map_err(|_| ReducerCallError::WorkerError(instance_lane_worker_error("call_reducer"))) } + pub async fn call_reducer_delete_outbound_on_success( + &self, + params: CallReducerParams, + msg_id: u64, + ) -> Result { + self.run_once("call_reducer", |inst: JsInstance| async move { + inst.send_request(|reply_tx| JsWorkerRequest::CallReducerDeleteOutboundOnSuccess { + reply_tx, + params, + msg_id, + }) + .await + }) + .await + .map_err(|_| ReducerCallError::WorkerError(instance_lane_worker_error("call_reducer"))) + } + /// Clear all instance-lane client state exactly once. pub async fn clear_all_clients(&self) -> anyhow::Result<()> { self.run_once("clear_all_clients", |inst: JsInstance| async move { @@ -1151,6 +1188,17 @@ async fn spawn_instance_worker( send_worker_reply("call_reducer", reply_tx, res, trapped); should_exit = trapped; } + JsWorkerRequest::CallReducerDeleteOutboundOnSuccess { reply_tx, params, msg_id } => { + let (res, trapped) = instance_common.call_reducer_with_tx_and_success_action( + None, + params, + &mut inst, + |tx| tx.delete_outbound_msg(msg_id).map_err(anyhow::Error::from), + ); + worker_trapped.store(trapped, Ordering::Relaxed); + send_worker_reply("call_reducer", reply_tx, res, trapped); + should_exit = trapped; + } JsWorkerRequest::CallView { reply_tx, cmd } => { let (res, trapped) = instance_common.handle_cmd(cmd, &mut inst); worker_trapped.store(trapped, Ordering::Relaxed); diff --git a/crates/core/src/host/wasm_common/module_host_actor.rs b/crates/core/src/host/wasm_common/module_host_actor.rs index b8156cda65f..ae611b2a7b5 100644 --- a/crates/core/src/host/wasm_common/module_host_actor.rs +++ b/crates/core/src/host/wasm_common/module_host_actor.rs @@ -472,6 +472,17 @@ impl WasmModuleInstance { res } + pub fn call_reducer_delete_outbound_on_success(&mut self, params: CallReducerParams, msg_id: u64) -> ReducerCallResult { + let (res, trapped) = crate::callgrind_flag::invoke_allowing_callgrind(|| { + self.common + .call_reducer_with_tx_and_success_action(None, params, &mut self.instance, |tx| { + tx.delete_outbound_msg(msg_id).map_err(anyhow::Error::from) + }) + }); + self.trapped = trapped; + res + } + pub fn clear_all_clients(&self) -> anyhow::Result<()> { self.common.clear_all_clients() } @@ -816,6 +827,19 @@ impl InstanceCommon { params: CallReducerParams, inst: &mut I, ) -> (ReducerCallResult, bool) { + self.call_reducer_with_tx_and_success_action(tx, params, inst, |_tx| Ok(())) + } + + pub(crate) fn call_reducer_with_tx_and_success_action( + &mut self, + tx: Option, + params: CallReducerParams, + inst: &mut I, + on_success: F, + ) -> (ReducerCallResult, bool) + where + F: FnOnce(&mut MutTxId) -> anyhow::Result<()>, + { let CallReducerParams { timestamp, caller_identity, @@ -871,6 +895,7 @@ impl InstanceCommon { outcome, energy_used: EnergyQuanta::ZERO, execution_duration: Duration::ZERO, + tx_offset: None, }, false, ); @@ -935,7 +960,10 @@ impl InstanceCommon { ), None => Ok(()), }; - let res = lifecycle_res.and(dedup_res); + let res = lifecycle_res + .map_err(anyhow::Error::from) + .and(dedup_res.map_err(anyhow::Error::from)) + .and_then(|_| on_success(&mut tx)); match res { Ok(()) => (EventStatus::Committed(DatabaseUpdate::default()), return_value), Err(err) => { @@ -993,7 +1021,9 @@ impl InstanceCommon { request_id, timer, }; - let event = commit_and_broadcast_event(&info.subscriptions, client, event, out.tx).event; + let committed = commit_and_broadcast_event(&info.subscriptions, client, event, out.tx); + let tx_offset = committed.tx_offset; + let event = committed.event; // For IDC (database-to-database) calls: if the reducer failed with a user error, // record the failure in st_inbound_msg in a separate tx (since the reducer tx @@ -1019,6 +1049,7 @@ impl InstanceCommon { outcome: ReducerOutcome::from(&event.status), energy_used: energy_quanta_used, execution_duration: total_duration, + tx_offset: Some(tx_offset), }; (res, trapped) diff --git a/crates/core/src/module_host_context.rs b/crates/core/src/module_host_context.rs index a3f39c3cb13..cd8293fb78c 100644 --- a/crates/core/src/module_host_context.rs +++ b/crates/core/src/module_host_context.rs @@ -1,5 +1,5 @@ use crate::energy::EnergyMonitor; -use crate::host::idc_runtime::IdcSender; +use crate::host::idc_actor::IdcActorSender; use crate::host::scheduler::Scheduler; use crate::replica_context::ReplicaContext; use spacetimedb_sats::hash::Hash; @@ -8,7 +8,7 @@ use std::sync::Arc; pub struct ModuleCreationContext { pub replica_ctx: Arc, pub scheduler: Scheduler, - pub idc_sender: IdcSender, + pub idc_sender: IdcActorSender, pub program_hash: Hash, pub energy_monitor: Arc, } diff --git a/crates/smoketests/Cargo.toml b/crates/smoketests/Cargo.toml index bfa718e7daa..380d6907c27 100644 --- a/crates/smoketests/Cargo.toml +++ b/crates/smoketests/Cargo.toml @@ -16,6 +16,7 @@ which = "8.0.0" [dev-dependencies] spacetimedb-core.workspace = true +spacetimedb-lib.workspace = true cargo_metadata.workspace = true assert_cmd = "2" predicates = "3" diff --git a/crates/smoketests/src/lib.rs b/crates/smoketests/src/lib.rs index aa54aea04ef..62be4c578e6 100644 --- a/crates/smoketests/src/lib.rs +++ b/crates/smoketests/src/lib.rs @@ -1314,6 +1314,17 @@ log = "0.4" self.api_call_internal(method, path, body, "") } + /// Makes an HTTP API call with an optional body and explicit extra headers. + pub fn api_call_with_body_and_headers( + &self, + method: &str, + path: &str, + body: Option<&[u8]>, + extra_headers: &str, + ) -> Result { + self.api_call_internal(method, path, body, extra_headers) + } + /// Makes an HTTP API call with a JSON body. pub fn api_call_json(&self, method: &str, path: &str, json_body: &str) -> Result { self.api_call_internal( diff --git a/crates/smoketests/tests/smoketests/idc.rs b/crates/smoketests/tests/smoketests/idc.rs new file mode 100644 index 00000000000..1f9be3eef40 --- /dev/null +++ b/crates/smoketests/tests/smoketests/idc.rs @@ -0,0 +1,103 @@ +use spacetimedb_lib::bsatn; +use spacetimedb_smoketests::Smoketest; + +fn idc_path(database_identity: &str, reducer: &str, sender_identity: &str, msg_id: u64) -> String { + format!( + "/v1/database/{database_identity}/call-from-database/{reducer}?sender_identity={sender_identity}&msg_id={msg_id}" + ) +} + +fn post_idc(test: &Smoketest, reducer: &str, sender_identity: &str, msg_id: u64, body: &[u8]) -> (u16, String) { + let database_identity = test.database_identity.as_ref().expect("No database published"); + let response = test + .api_call_with_body_and_headers( + "POST", + &idc_path(database_identity, reducer, sender_identity, msg_id), + Some(body), + "Content-Type: application/octet-stream\r\n", + ) + .expect("IDC HTTP call failed"); + let text = response.text().expect("IDC response body should be valid UTF-8"); + (response.status_code, text) +} + +#[test] +fn test_call_from_database_deduplicates_successful_reducer() { + let module_code = r#" +use spacetimedb::{ReducerContext, Table}; + +#[spacetimedb::table(accessor = deliveries, public)] +pub struct Delivery { + #[primary_key] + name: String, + count: u64, +} + +#[spacetimedb::reducer(init)] +pub fn init(ctx: &ReducerContext) { + ctx.db.deliveries().insert(Delivery { + name: "remote".to_string(), + count: 0, + }); +} + +#[spacetimedb::reducer] +pub fn accept(ctx: &ReducerContext) { + let mut row = ctx + .db + .deliveries() + .name() + .find("remote".to_string()) + .expect("delivery row should exist"); + row.count += 1; + ctx.db.deliveries().name().update(row); +} +"#; + + let test = Smoketest::builder().module_code(module_code).build(); + let body = bsatn::to_vec(&()).expect("BSATN encoding should succeed"); + let sender_identity = "00000000000000000000000000000000000000000000000000000000000000aa"; + + let first = post_idc(&test, "accept", sender_identity, 7, &body); + let second = post_idc(&test, "accept", sender_identity, 7, &body); + + assert_eq!(first.0, 200, "first IDC call should succeed: {first:?}"); + assert_eq!(first.1, ""); + assert_eq!(second.0, 200, "duplicate IDC call should still succeed: {second:?}"); + assert_eq!(second.1, ""); + + let output = test.sql("SELECT name, count FROM deliveries").unwrap(); + assert!( + output.contains(r#""remote" | 1"#), + "expected the deduplicated reducer to leave a single delivery row, got:\n{output}", + ); +} + +#[test] +fn test_call_from_database_replays_stored_reducer_error_without_rerunning() { + let module_code = r#" +use spacetimedb::{log, ReducerContext}; + +#[spacetimedb::reducer] +pub fn always_fail(_ctx: &ReducerContext) -> Result<(), String> { + log::info!("IDC failing reducer executed"); + Err("boom".to_string()) +} +"#; + + let test = Smoketest::builder().module_code(module_code).build(); + let body = bsatn::to_vec(&()).expect("BSATN encoding should succeed"); + let sender_identity = "00000000000000000000000000000000000000000000000000000000000000bb"; + + let first = post_idc(&test, "always_fail", sender_identity, 9, &body); + let second = post_idc(&test, "always_fail", sender_identity, 9, &body); + + assert_eq!(first.0, 422, "first IDC call should surface reducer error: {first:?}"); + assert_eq!(first.1.trim(), "boom"); + assert_eq!(second.0, 422, "duplicate IDC call should replay reducer error: {second:?}"); + assert_eq!(second.1.trim(), "boom"); + + let logs = test.logs(100).unwrap(); + let executions = logs.iter().filter(|line| line.contains("IDC failing reducer executed")).count(); + assert_eq!(executions, 1, "duplicate IDC request should not rerun the reducer"); +} diff --git a/crates/smoketests/tests/smoketests/mod.rs b/crates/smoketests/tests/smoketests/mod.rs index f5053652dd3..847138e1516 100644 --- a/crates/smoketests/tests/smoketests/mod.rs +++ b/crates/smoketests/tests/smoketests/mod.rs @@ -19,6 +19,7 @@ mod domains; mod fail_initial_publish; mod filtering; mod http_egress; +mod idc; mod logs_level_filter; mod module_nested_op; mod modules; From a337bec99fb73d1759f0b2babab7e69de3b92ecf Mon Sep 17 00:00:00 2001 From: Shubham Mishra Date: Wed, 8 Apr 2026 19:38:38 +0530 Subject: [PATCH 11/23] use Identity --- crates/client-api/src/routes/database.rs | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/crates/client-api/src/routes/database.rs b/crates/client-api/src/routes/database.rs index 655b40f08ec..5a5bf9abe0c 100644 --- a/crates/client-api/src/routes/database.rs +++ b/crates/client-api/src/routes/database.rs @@ -228,8 +228,8 @@ pub struct CallFromDatabaseParams { /// Both fields are mandatory; a missing field results in a 400 Bad Request. #[derive(Deserialize)] pub struct CallFromDatabaseQuery { - /// Hex-encoded [`Identity`] of the sending database. - sender_identity: String, + /// [`Identity`] of the sending database, parsed from a hex query string. + sender_identity: Identity, /// The inter-database message ID from the sender's st_outbound_msg. /// Used for at-most-once delivery via `st_inbound_msg`. msg_id: u64, @@ -269,13 +269,6 @@ pub async fn call_from_database( let caller_identity = auth.claims.identity; - let sender_identity = Identity::from_hex(&sender_identity).map_err(|_| { - ( - StatusCode::BAD_REQUEST, - "Invalid sender_identity: expected hex-encoded identity", - ) - })?; - let args = FunctionArgs::Bsatn(body); let connection_id = generate_random_connection_id(); From 904471b4fb4e9833a2ca3e37de02426471f6ced7 Mon Sep 17 00:00:00 2001 From: Shubham Mishra Date: Thu, 9 Apr 2026 13:46:12 +0530 Subject: [PATCH 12/23] outbox macro change --- crates/bindings-macro/src/table.rs | 66 +++++++-- crates/core/src/host/host_controller.rs | 9 ++ crates/core/src/host/idc_actor.rs | 72 +++++---- crates/core/src/host/v8/mod.rs | 16 +- .../src/host/wasm_common/module_host_actor.rs | 6 +- .../locking_tx_datastore/committed_state.rs | 8 +- .../src/locking_tx_datastore/mut_tx.rs | 1 - crates/datastore/src/system_tables.rs | 5 +- crates/schema/src/def/validate/v10.rs | 32 +++- crates/smoketests/spacetime.json | 3 + crates/smoketests/tests/smoketests/idc.rs | 10 +- crates/standalone/src/lib.rs | 4 + crates/standalone/src/subcommands/start.rs | 4 +- smoketests/tests/outbox.py | 138 ++++++++++++++++++ 14 files changed, 314 insertions(+), 60 deletions(-) create mode 100644 crates/smoketests/spacetime.json create mode 100644 smoketests/tests/outbox.py diff --git a/crates/bindings-macro/src/table.rs b/crates/bindings-macro/src/table.rs index ba41bc57abc..fd8ac7196dd 100644 --- a/crates/bindings-macro/src/table.rs +++ b/crates/bindings-macro/src/table.rs @@ -25,15 +25,26 @@ pub(crate) struct TableArgs { outbox: Option, } -/// Parsed from `outbox(remote_reducer_fn)` optionally followed by `on_result(local_reducer_fn)`. +/// Parsed from `outbox(remote_reducer_fn, on_result = local_reducer_fn)`. +/// +/// For backwards compatibility, `on_result(local_reducer_fn)` is also accepted as a sibling +/// table attribute and attached to the previously-declared outbox. struct OutboxArg { span: Span, - /// Path to the remote-side reducer function (used only for its name via `FnInfo::NAME`). + /// Path to the remote-side reducer function, used only for its final path segment. remote_reducer: Path, /// Path to the local `on_result` reducer, if any. on_result_reducer: Option, } +fn path_tail_name(path: &Path) -> LitStr { + let segment = path + .segments + .last() + .expect("syn::Path should always contain at least one segment"); + LitStr::new(&segment.ident.to_string(), segment.ident.span()) +} + enum TableAccess { Public(Span), Private(Span), @@ -268,18 +279,44 @@ impl ScheduledArg { } impl OutboxArg { - /// Parse `outbox(remote_reducer_path)`. + /// Parse `outbox(remote_reducer_path, on_result = local_reducer_path)`. /// - /// `on_result` is parsed separately via `parse_single_path_meta` and attached afterwards. + /// For backwards compatibility, `on_result` may also be parsed separately via + /// `parse_single_path_meta` and attached afterwards. fn parse_meta(meta: ParseNestedMeta) -> syn::Result { let span = meta.path.span(); let mut remote_reducer: Option = None; + let mut on_result_reducer: Option = None; meta.parse_nested_meta(|meta| { - if meta.input.peek(syn::Token![=]) || meta.input.peek(syn::token::Paren) { - Err(meta.error("outbox takes a single function path, e.g. `outbox(my_remote_reducer)`")) + if meta.input.peek(Token![=]) { + if meta.path.is_ident("on_result") { + check_duplicate_msg( + &on_result_reducer, + &meta, + "can only specify one on_result reducer", + )?; + on_result_reducer = Some(meta.value()?.parse()?); + Ok(()) + } else { + Err(meta.error( + "outbox only supports `on_result = my_local_reducer` as a named argument", + )) + } + } else if meta.input.peek(syn::token::Paren) { + Err(meta.error( + "outbox expects a remote reducer path and optional `on_result = my_local_reducer`, e.g. `outbox(my_remote_reducer, on_result = my_local_reducer)`", + )) + } else if meta.path.is_ident("on_result") { + Err(meta.error( + "outbox `on_result` must use `=`, e.g. `outbox(my_remote_reducer, on_result = my_local_reducer)`", + )) } else { - check_duplicate_msg(&remote_reducer, &meta, "can only specify one remote reducer for outbox")?; + check_duplicate_msg( + &remote_reducer, + &meta, + "can only specify one remote reducer for outbox", + )?; remote_reducer = Some(meta.path); Ok(()) } @@ -291,7 +328,7 @@ impl OutboxArg { Ok(Self { span, remote_reducer, - on_result_reducer: None, + on_result_reducer, }) } @@ -1209,19 +1246,24 @@ pub(crate) fn table_impl(mut args: TableArgs, item: &syn::DeriveInput) -> syn::R )); } - let remote_reducer = &ob.remote_reducer; - let on_result_reducer_name = match &ob.on_result_reducer { - Some(r) => quote!(Some(<#r as spacetimedb::rt::FnInfo>::NAME)), + let remote_reducer_name = path_tail_name(&ob.remote_reducer); + let on_result_reducer_name = match ob.on_result_reducer.as_ref().map(path_tail_name) { + Some(r) => quote!(Some(#r)), None => quote!(None), }; let desc = quote!(spacetimedb::table::OutboxDesc { - remote_reducer_name: <#remote_reducer as spacetimedb::rt::FnInfo>::NAME, + remote_reducer_name: #remote_reducer_name, on_result_reducer_name: #on_result_reducer_name, }); let primary_key_ty = primary_key_column.ty; + let callback_typecheck = ob + .on_result_reducer + .as_ref() + .map(|r| quote!(spacetimedb::rt::outbox_typecheck::<#original_struct_ident>(#r);)); let typecheck = quote! { spacetimedb::rt::assert_outbox_table_primary_key::<#primary_key_ty>(); + #callback_typecheck }; Ok((desc, typecheck)) diff --git a/crates/core/src/host/host_controller.rs b/crates/core/src/host/host_controller.rs index 5389086909c..610755dad3a 100644 --- a/crates/core/src/host/host_controller.rs +++ b/crates/core/src/host/host_controller.rs @@ -44,6 +44,7 @@ use spacetimedb_schema::def::{ModuleDef, RawModuleDefVersion}; use spacetimedb_table::page_pool::PagePool; use std::future::Future; use std::ops::Deref; +use std::sync::atomic::{AtomicU16, Ordering}; use std::sync::Arc; use std::time::Duration; use tokio::sync::{watch, OwnedRwLockReadGuard, OwnedRwLockWriteGuard, RwLock as AsyncRwLock}; @@ -118,6 +119,8 @@ pub struct HostController { db_cores: JobCores, /// The pool of buffers used to build `BsatnRowList`s in subscriptions. pub bsatn_rlb_pool: BsatnRowListBuilderPool, + /// HTTP port used by local IDC delivery. + idc_http_port: Arc, } pub(crate) struct HostRuntimes { @@ -233,6 +236,7 @@ impl HostController { page_pool: PagePool::new(default_config.page_pool_max_size), bsatn_rlb_pool: BsatnRowListBuilderPool::new(), db_cores, + idc_http_port: Arc::new(AtomicU16::new(80)), } } @@ -241,6 +245,10 @@ impl HostController { self.program_storage = ps; } + pub fn set_idc_http_port(&self, port: u16) { + self.idc_http_port.store(port, Ordering::Relaxed); + } + /// Get a [`ModuleHost`] managed by this controller, or launch it from /// persistent state. /// @@ -1117,6 +1125,7 @@ impl Host { replica_ctx.relational_db().clone(), IdcActorConfig { sender_identity: replica_ctx.database_identity, + http_port: host_controller.idc_http_port.load(Ordering::Relaxed), }, module_host.downgrade(), ); diff --git a/crates/core/src/host/idc_actor.rs b/crates/core/src/host/idc_actor.rs index 06ab9c0c357..14d08ab7dd2 100644 --- a/crates/core/src/host/idc_actor.rs +++ b/crates/core/src/host/idc_actor.rs @@ -4,7 +4,7 @@ /// 1. Loads undelivered entries from `st_outbound_msg` on startup, resolving delivery data from outbox tables. /// 2. Accepts immediate notifications via an mpsc channel when new outbox rows are inserted. /// 3. Delivers each message in msg_id order via HTTP POST to -/// `http://localhost:80/v1/database/{target_db}/call-from-database/{reducer}?sender_identity=&msg_id=` +/// `http://localhost:{http_port}/v1/database/{target_db}/call-from-database/{reducer}?sender_identity=&msg_id=` /// 4. On transport errors (network, 5xx, 4xx except 422/402): retries infinitely with exponential /// backoff, blocking only the affected target database (other targets continue unaffected). /// 5. On reducer errors (HTTP 422) or budget exceeded (HTTP 402): calls the configured @@ -16,14 +16,13 @@ use crate::host::FunctionArgs; use spacetimedb_datastore::execution_context::Workload; use spacetimedb_datastore::system_tables::{StOutboundMsgRow, ST_OUTBOUND_MSG_ID}; use spacetimedb_datastore::traits::IsolationLevel; -use spacetimedb_lib::{AlgebraicValue, Identity}; +use spacetimedb_lib::{AlgebraicValue, Identity, ProductValue}; use spacetimedb_primitives::{ColId, TableId}; use std::collections::{HashMap, VecDeque}; use std::sync::Arc; use std::time::{Duration, Instant}; use tokio::sync::mpsc; -const IDC_HTTP_PORT: u16 = 80; const INITIAL_BACKOFF: Duration = Duration::from_millis(100); const MAX_BACKOFF: Duration = Duration::from_secs(30); /// How long to wait before polling again when there is no work. @@ -38,6 +37,7 @@ pub type IdcActorSender = mpsc::UnboundedSender<()>; /// The identity of this (sender) database, set when the IDC actor is started. pub struct IdcActorConfig { pub sender_identity: Identity, + pub http_port: u16, } /// A handle that, when dropped, stops the IDC actor background task. @@ -86,6 +86,7 @@ struct PendingMessage { target_db_identity: Identity, target_reducer: String, args_bsatn: Vec, + request_row: ProductValue, /// From the outbox table's `TableSchema::on_result_reducer`. on_result_reducer: Option, } @@ -171,7 +172,7 @@ async fn run_idc_loop( log::warn!( "idc_actor: transport error delivering msg_id={} to {}: {reason}", msg.msg_id, - hex::encode(msg.target_db_identity.to_byte_array()), + msg.target_db_identity.to_hex(), ); state.record_transport_error(); // Do NOT pop the front — keep retrying this message for this target. @@ -246,19 +247,23 @@ async fn finalize_message( return; }; - // Encode (result_payload: String) as BSATN args. - // The on_result reducer is expected to accept a single String argument. - let args_bytes = match spacetimedb_sats::bsatn::to_vec(&result_payload) { - Ok(b) => b, - Err(e) => { - log::error!( - "idc_actor: failed to encode on_result args for msg_id={}: {e}", - msg.msg_id - ); - delete_message(db, msg.msg_id); - return; - } + // Encode `(request_row: OutboxRow, result: Result<(), String>)` as BSATN args. + let result = if result_payload.is_empty() { + Ok::<(), String>(()) + } else { + Err::<(), String>(result_payload) }; + let mut args_bytes = Vec::new(); + if let Err(e) = spacetimedb_sats::bsatn::to_writer(&mut args_bytes, &msg.request_row) + .and_then(|_| spacetimedb_sats::bsatn::to_writer(&mut args_bytes, &result)) + { + log::error!( + "idc_actor: failed to encode on_result args for msg_id={}: {e}", + msg.msg_id + ); + delete_message(db, msg.msg_id); + return; + } let caller_identity = Identity::ZERO; // system call let result = host @@ -366,12 +371,24 @@ fn load_pending_into_targets(db: &RelationalDB, targets: &mut HashMap Identity::from_u256(**u), + // Col 1: target_db_identity stored as SATS `Identity`, + // i.e. the product wrapper `(__identity__: U256)`. + let target_db_identity: Identity = match pv.elements.get(1) { + Some(AlgebraicValue::Product(identity_pv)) if identity_pv.elements.len() == 1 => { + match &identity_pv.elements[0] { + AlgebraicValue::U256(u) => Identity::from_u256(**u), + other => { + log::error!( + "idc_actor: outbox row col 1 expected Identity inner U256, got {other:?} (msg_id={})", + st_row.msg_id, + ); + continue; + } + } + } other => { log::error!( - "idc_actor: outbox row col 1 expected U256 (Identity), got {other:?} (msg_id={})", + "idc_actor: outbox row col 1 expected Identity wrapper, got {other:?} (msg_id={})", st_row.msg_id, ); continue; @@ -392,6 +409,7 @@ fn load_pending_into_targets(db: &RelationalDB, targets: &mut HashMap DeliveryOutcome { - let target_db_hex = hex::encode(msg.target_db_identity.to_byte_array()); - let sender_hex = hex::encode(config.sender_identity.to_byte_array()); +async fn attempt_delivery(client: &reqwest::Client, config: &IdcActorConfig, msg: &PendingMessage) -> DeliveryOutcome { + let target_db_hex = msg.target_db_identity.to_hex(); + let sender_hex = config.sender_identity.to_hex(); let url = format!( - "http://localhost:{IDC_HTTP_PORT}/v1/database/{target_db_hex}/call-from-database/{}?sender_identity={sender_hex}&msg_id={}", - msg.target_reducer, msg.msg_id, + "http://localhost:{}/v1/database/{target_db_hex}/call-from-database/{}?sender_identity={sender_hex}&msg_id={}", + config.http_port, msg.target_reducer, msg.msg_id, ); let result = client diff --git a/crates/core/src/host/v8/mod.rs b/crates/core/src/host/v8/mod.rs index 993ed9f7d95..f3be86dc2de 100644 --- a/crates/core/src/host/v8/mod.rs +++ b/crates/core/src/host/v8/mod.rs @@ -1188,13 +1188,15 @@ async fn spawn_instance_worker( send_worker_reply("call_reducer", reply_tx, res, trapped); should_exit = trapped; } - JsWorkerRequest::CallReducerDeleteOutboundOnSuccess { reply_tx, params, msg_id } => { - let (res, trapped) = instance_common.call_reducer_with_tx_and_success_action( - None, - params, - &mut inst, - |tx| tx.delete_outbound_msg(msg_id).map_err(anyhow::Error::from), - ); + JsWorkerRequest::CallReducerDeleteOutboundOnSuccess { + reply_tx, + params, + msg_id, + } => { + let (res, trapped) = + instance_common.call_reducer_with_tx_and_success_action(None, params, &mut inst, |tx| { + tx.delete_outbound_msg(msg_id).map_err(anyhow::Error::from) + }); worker_trapped.store(trapped, Ordering::Relaxed); send_worker_reply("call_reducer", reply_tx, res, trapped); should_exit = trapped; diff --git a/crates/core/src/host/wasm_common/module_host_actor.rs b/crates/core/src/host/wasm_common/module_host_actor.rs index ae611b2a7b5..89fa8e151dd 100644 --- a/crates/core/src/host/wasm_common/module_host_actor.rs +++ b/crates/core/src/host/wasm_common/module_host_actor.rs @@ -472,7 +472,11 @@ impl WasmModuleInstance { res } - pub fn call_reducer_delete_outbound_on_success(&mut self, params: CallReducerParams, msg_id: u64) -> ReducerCallResult { + pub fn call_reducer_delete_outbound_on_success( + &mut self, + params: CallReducerParams, + msg_id: u64, + ) -> ReducerCallResult { let (res, trapped) = crate::callgrind_flag::invoke_allowing_callgrind(|| { self.common .call_reducer_with_tx_and_success_action(None, params, &mut self.instance, |tx| { diff --git a/crates/datastore/src/locking_tx_datastore/committed_state.rs b/crates/datastore/src/locking_tx_datastore/committed_state.rs index 352d52f3572..7cc460364a5 100644 --- a/crates/datastore/src/locking_tx_datastore/committed_state.rs +++ b/crates/datastore/src/locking_tx_datastore/committed_state.rs @@ -19,10 +19,10 @@ use crate::{ is_built_in_meta_row, system_tables, table_id_is_reserved, StColumnRow, StConstraintData, StConstraintRow, StFields, StIndexRow, StSequenceFields, StSequenceRow, StTableFields, StTableRow, StViewRow, SystemTable, ST_CLIENT_ID, ST_CLIENT_IDX, ST_COLUMN_ID, ST_COLUMN_IDX, ST_COLUMN_NAME, ST_CONSTRAINT_ID, ST_CONSTRAINT_IDX, - ST_CONSTRAINT_NAME, ST_INBOUND_MSG_ID, ST_INDEX_ID, ST_INDEX_IDX, ST_INDEX_NAME, ST_MODULE_ID, - ST_MODULE_IDX, ST_ROW_LEVEL_SECURITY_ID, ST_ROW_LEVEL_SECURITY_IDX, ST_SCHEDULED_ID, ST_SCHEDULED_IDX, - ST_SEQUENCE_ID, ST_SEQUENCE_IDX, ST_SEQUENCE_NAME, ST_TABLE_ID, ST_TABLE_IDX, ST_VAR_ID, ST_VAR_IDX, - ST_VIEW_ARG_ID, ST_VIEW_ARG_IDX, + ST_CONSTRAINT_NAME, ST_INBOUND_MSG_ID, ST_INDEX_ID, ST_INDEX_IDX, ST_INDEX_NAME, ST_MODULE_ID, ST_MODULE_IDX, + ST_ROW_LEVEL_SECURITY_ID, ST_ROW_LEVEL_SECURITY_IDX, ST_SCHEDULED_ID, ST_SCHEDULED_IDX, ST_SEQUENCE_ID, + ST_SEQUENCE_IDX, ST_SEQUENCE_NAME, ST_TABLE_ID, ST_TABLE_IDX, ST_VAR_ID, ST_VAR_IDX, ST_VIEW_ARG_ID, + ST_VIEW_ARG_IDX, }, traits::{EphemeralTables, TxData}, }; diff --git a/crates/datastore/src/locking_tx_datastore/mut_tx.rs b/crates/datastore/src/locking_tx_datastore/mut_tx.rs index 032b72089a6..04abf66c5b5 100644 --- a/crates/datastore/src/locking_tx_datastore/mut_tx.rs +++ b/crates/datastore/src/locking_tx_datastore/mut_tx.rs @@ -2697,7 +2697,6 @@ impl MutTxId { }) } - pub fn insert_via_serialize_bsatn<'a, T: Serialize>( &'a mut self, table_id: TableId, diff --git a/crates/datastore/src/system_tables.rs b/crates/datastore/src/system_tables.rs index efedacb473a..e2c35c4bebb 100644 --- a/crates/datastore/src/system_tables.rs +++ b/crates/datastore/src/system_tables.rs @@ -710,7 +710,10 @@ fn system_module_def() -> ModuleDef { let st_outbound_msg_type = builder.add_type::(); builder - .build_table(ST_OUTBOUND_MSG_NAME, *st_outbound_msg_type.as_ref().expect("should be ref")) + .build_table( + ST_OUTBOUND_MSG_NAME, + *st_outbound_msg_type.as_ref().expect("should be ref"), + ) .with_type(TableType::System) .with_auto_inc_primary_key(StOutboundMsgFields::MsgId) .with_index_no_accessor_name(btree(StOutboundMsgFields::MsgId)); diff --git a/crates/schema/src/def/validate/v10.rs b/crates/schema/src/def/validate/v10.rs index cdc6e9ef909..3d8fc6b75cf 100644 --- a/crates/schema/src/def/validate/v10.rs +++ b/crates/schema/src/def/validate/v10.rs @@ -885,9 +885,9 @@ fn attach_outboxes_to_tables( .into()); } - // Validate col 1: must be U256 (Identity wire representation). + // Validate col 1: must be Identity (or its raw U256 wire representation). let col1_ty = &table.columns[1].ty; - if *col1_ty != AlgebraicType::U256 { + if *col1_ty != AlgebraicType::U256 && !col1_ty.is_identity() { return Err(ValidationError::OutboxInvalidTargetColumn { table: table.name.clone(), found: col1_ty.clone().into(), @@ -1249,6 +1249,34 @@ mod tests { ); } + #[test] + fn test_outbox_accepts_identity_target_column() { + let mut builder = RawModuleDefV10Builder::new(); + + builder + .build_table_with_new_type( + "outbound_pings", + ProductType::from([ + ("id", AlgebraicType::U64), + ("target", AlgebraicType::identity()), + ("payload", AlgebraicType::String), + ]), + true, + ) + .with_auto_inc_primary_key(0) + .finish(); + + builder.add_reducer("send_ping", ProductType::unit()); + builder.add_reducer("receive_ping", ProductType::from([("payload", AlgebraicType::String)])); + builder.add_outbox("outbound_pings", "receive_ping", Option::<&str>::None); + + let validated: Result = builder.finish().try_into(); + assert!( + validated.is_ok(), + "expected outbox validation to accept Identity target column, got {validated:?}", + ); + } + #[test] fn invalid_product_type_ref() { let mut builder = RawModuleDefV10Builder::new(); diff --git a/crates/smoketests/spacetime.json b/crates/smoketests/spacetime.json new file mode 100644 index 00000000000..fc8abe551ec --- /dev/null +++ b/crates/smoketests/spacetime.json @@ -0,0 +1,3 @@ +{ + "server": "maincloud" +} \ No newline at end of file diff --git a/crates/smoketests/tests/smoketests/idc.rs b/crates/smoketests/tests/smoketests/idc.rs index 1f9be3eef40..c67d9d4871f 100644 --- a/crates/smoketests/tests/smoketests/idc.rs +++ b/crates/smoketests/tests/smoketests/idc.rs @@ -94,10 +94,16 @@ pub fn always_fail(_ctx: &ReducerContext) -> Result<(), String> { assert_eq!(first.0, 422, "first IDC call should surface reducer error: {first:?}"); assert_eq!(first.1.trim(), "boom"); - assert_eq!(second.0, 422, "duplicate IDC call should replay reducer error: {second:?}"); + assert_eq!( + second.0, 422, + "duplicate IDC call should replay reducer error: {second:?}" + ); assert_eq!(second.1.trim(), "boom"); let logs = test.logs(100).unwrap(); - let executions = logs.iter().filter(|line| line.contains("IDC failing reducer executed")).count(); + let executions = logs + .iter() + .filter(|line| line.contains("IDC failing reducer executed")) + .count(); assert_eq!(executions, 1, "duplicate IDC request should not rerun the reducer"); } diff --git a/crates/standalone/src/lib.rs b/crates/standalone/src/lib.rs index 68450a1b283..e83ff07eda3 100644 --- a/crates/standalone/src/lib.rs +++ b/crates/standalone/src/lib.rs @@ -118,6 +118,10 @@ impl StandaloneEnv { pub fn bsatn_rlb_pool(&self) -> &BsatnRowListBuilderPool { &self.host_controller.bsatn_rlb_pool } + + pub fn set_idc_http_port(&self, port: u16) { + self.host_controller.set_idc_http_port(port); + } } #[derive(Debug, thiserror::Error)] diff --git a/crates/standalone/src/subcommands/start.rs b/crates/standalone/src/subcommands/start.rs index 399b7e9a350..c16fab58e49 100644 --- a/crates/standalone/src/subcommands/start.rs +++ b/crates/standalone/src/subcommands/start.rs @@ -249,7 +249,9 @@ pub async fn exec(args: &ArgMatches, db_cores: JobCores) -> anyhow::Result<()> { "failed to bind the SpacetimeDB server to '{listen_addr}', please check that the address is valid and not already in use" ))?; socket2::SockRef::from(&tcp).set_nodelay(true)?; - log::info!("Starting SpacetimeDB listening on {}", tcp.local_addr()?); + let local_addr = tcp.local_addr()?; + ctx.set_idc_http_port(local_addr.port()); + log::info!("Starting SpacetimeDB listening on {local_addr}"); if let Some(pg_port) = pg_port { let server_addr = listen_addr.split(':').next().unwrap(); diff --git a/smoketests/tests/outbox.py b/smoketests/tests/outbox.py new file mode 100644 index 00000000000..9990a98f921 --- /dev/null +++ b/smoketests/tests/outbox.py @@ -0,0 +1,138 @@ +import time + +from .. import Smoketest, random_string + + +class OutboxPingPong(Smoketest): + AUTOPUBLISH = False + + RECEIVER_MODULE = """ +use spacetimedb::{Identity, ReducerContext, SpacetimeType, Table}; + +#[derive(SpacetimeType)] +pub struct Ping { + payload: String, +} + +#[spacetimedb::table(accessor = received_pings, public)] +pub struct ReceivedPing { + #[primary_key] + #[auto_inc] + id: u64, + sender: Identity, + payload: String, +} + +#[spacetimedb::reducer] +pub fn receive_ping(ctx: &ReducerContext, ping: Ping) { + ctx.db.received_pings().insert(ReceivedPing { + id: 0, + sender: ctx.sender(), + payload: ping.payload, + }); +} +""" + + SENDER_MODULE = """ +use spacetimedb::{Identity, ReducerContext, SpacetimeType, Table}; + +#[derive(SpacetimeType)] +pub struct Ping { + payload: String, +} + +#[spacetimedb::table(accessor = outbound_pings, public, outbox(receive_ping, on_result = local_callback))] +pub struct OutboundPing { + #[primary_key] + #[auto_inc] + id: u64, + target: Identity, + ping: Ping, +} + +#[spacetimedb::table(accessor = callback_results, public)] +pub struct CallbackResult { + #[primary_key] + #[auto_inc] + id: u64, + payload: String, + status: String, +} + +#[spacetimedb::reducer] +pub fn send_ping(ctx: &ReducerContext, target_hex: String, payload: String) { + let target = Identity::from_hex(&target_hex).expect("target identity should be valid hex"); + ctx.db.outbound_pings().insert(OutboundPing { + id: 0, + target, + ping: Ping { payload }, + }); +} + + +#[spacetimedb::reducer] +pub fn local_callback(ctx: &ReducerContext, request: OutboundPing, result: Result<(), String>) { + let status = match result { + Ok(()) => "ok".to_string(), + Err(err) => format!("err:{err}"), + }; + ctx.db.callback_results().insert(CallbackResult { + id: 0, + payload: request.ping.payload, + status, + }); +} +""" + + def sql_on(self, database_identity, query): + return self.spacetime("sql", "--anonymous", "--", database_identity, query) + + def test_outbox_ping_from_sender_module_reaches_receiver_module(self): + self.write_module_code(self.RECEIVER_MODULE) + receiver_name = f"outbox-receiver-{random_string()}" + self.publish_module(receiver_name, clear=False) + receiver_identity = self.database_identity + self.addCleanup(lambda db=receiver_identity: self.spacetime("delete", "--yes", db)) + + self.write_module_code(self.SENDER_MODULE) + sender_name = f"outbox-sender-{random_string()}" + self.publish_module(sender_name, clear=False) + sender_identity = self.database_identity + self.addCleanup(lambda db=sender_identity: self.spacetime("delete", "--yes", db)) + + payload = "ping" + self.call("send_ping", receiver_identity, payload) + + deadline = time.time() + 8 + while True: + output = self.sql_on(receiver_identity, "SELECT * FROM received_pings") + if f'"{payload}"' in output: + break + if time.time() >= deadline: + self.fail(f"timed out waiting for ping delivery, last query output:\n{output}") + time.sleep(0.1) + + received = self.sql_on(receiver_identity, "SELECT * FROM received_pings") + self.assertIn(f'"{payload}"', received) + + callback_deadline = time.time() + 8 + while True: + callback_results = self.sql_on(sender_identity, "SELECT payload, status FROM callback_results") + if f'"{payload}"' in callback_results and '"ok"' in callback_results: + break + if time.time() >= callback_deadline: + self.fail( + "timed out waiting for callback result, last query output:\n" + f"{callback_results}" + ) + time.sleep(0.1) + + callback_results = self.sql_on(sender_identity, "SELECT payload, status FROM callback_results") + self.assertEqual(callback_results.count(f'"{payload}"'), 1, callback_results) + self.assertEqual(callback_results.count('"ok"'), 1, callback_results) + + time.sleep(1.0) + + callback_results_after_wait = self.sql_on(sender_identity, "SELECT payload, status FROM callback_results") + self.assertEqual(callback_results_after_wait.count(f'"{payload}"'), 1, callback_results_after_wait) + self.assertEqual(callback_results_after_wait.count('"ok"'), 1, callback_results_after_wait) From 1437e92eab6cf98e35a48bcdc30c8e255b8f80da Mon Sep 17 00:00:00 2001 From: Shubham Mishra Date: Thu, 9 Apr 2026 19:57:42 +0530 Subject: [PATCH 13/23] remove reducer_outcome --- crates/client-api/src/routes/database.rs | 3 +-- crates/core/src/host/host_controller.rs | 7 ++----- crates/core/src/host/module_host.rs | 11 ++++++----- crates/core/src/host/wasm_common/module_host_actor.rs | 11 ++++++++++- 4 files changed, 19 insertions(+), 13 deletions(-) diff --git a/crates/client-api/src/routes/database.rs b/crates/client-api/src/routes/database.rs index 5a5bf9abe0c..03f889df997 100644 --- a/crates/client-api/src/routes/database.rs +++ b/crates/client-api/src/routes/database.rs @@ -311,7 +311,6 @@ pub async fn call_from_database( Ok(rcr) => { let (status, body) = match rcr.outcome { ReducerOutcome::Committed => (StatusCode::OK, "".into()), - ReducerOutcome::Deduplicated => (StatusCode::OK, "deduplicated".into()), // 422 = reducer ran but returned Err; the IDC actor uses this to distinguish // reducer failures from transport errors (which it retries). ReducerOutcome::Failed(errmsg) => (StatusCode::UNPROCESSABLE_ENTITY, *errmsg), @@ -351,7 +350,7 @@ fn reducer_outcome_response( outcome: ReducerOutcome, ) -> (StatusCode, Box) { match outcome { - ReducerOutcome::Committed | ReducerOutcome::Deduplicated => (StatusCode::OK, "".into()), + ReducerOutcome::Committed => (StatusCode::OK, "".into()), ReducerOutcome::Failed(errmsg) => { // TODO: different status code? this is what cloudflare uses, sorta (StatusCode::from_u16(530).unwrap(), *errmsg) diff --git a/crates/core/src/host/host_controller.rs b/crates/core/src/host/host_controller.rs index 610755dad3a..fb9de982bd0 100644 --- a/crates/core/src/host/host_controller.rs +++ b/crates/core/src/host/host_controller.rs @@ -165,22 +165,19 @@ pub enum ReducerOutcome { Committed, Failed(Box>), BudgetExceeded, - /// The call was identified as a duplicate via the `st_databases_tx_offset` dedup index - /// and was discarded without running the reducer. - Deduplicated, } impl ReducerOutcome { pub fn into_result(self) -> anyhow::Result<()> { match self { - Self::Committed | Self::Deduplicated => Ok(()), + Self::Committed => Ok(()), Self::Failed(e) => Err(anyhow::anyhow!(e)), Self::BudgetExceeded => Err(anyhow::anyhow!("reducer ran out of energy")), } } pub fn is_err(&self) -> bool { - !matches!(self, Self::Committed | Self::Deduplicated) + !matches!(self, Self::Committed) } } diff --git a/crates/core/src/host/module_host.rs b/crates/core/src/host/module_host.rs index 4ac99831775..1337e171f8b 100644 --- a/crates/core/src/host/module_host.rs +++ b/crates/core/src/host/module_host.rs @@ -589,7 +589,7 @@ pub fn call_identity_connected( // then insert into `st_client`, // but if we crashed in between, we'd be left in an inconsistent state // where the reducer had run but `st_client` was not yet updated. - ReducerOutcome::Committed | ReducerOutcome::Deduplicated => Ok(()), + ReducerOutcome::Committed => Ok(()), // If the reducer returned an error or couldn't run due to insufficient energy, // abort the connection: the module code has decided it doesn't want this client. @@ -629,7 +629,8 @@ pub struct CallReducerParams { /// The tuple is `(sender_database_identity, sender_msg_id)`. /// Before running the reducer, `st_inbound_msg` is consulted: /// if `sender_msg_id` ≤ the stored last-delivered msg_id for the sender, - /// the call is a duplicate and returns [`ReducerCallError::Deduplicated`]. + /// the call is a duplicate and returns the stored committed or failed result + /// without running the reducer again. /// Otherwise the reducer runs, and the stored msg_id is updated atomically /// within the same transaction. pub dedup_sender: Option<(Identity, u64)>, @@ -1500,9 +1501,9 @@ impl ModuleHost { .. }) => fallback(), - // If it succeeded (or was deduplicated), as mentioned above, `st_client` is already updated. + // If it succeeded, as mentioned above, `st_client` is already updated. Ok(ReducerCallResult { - outcome: ReducerOutcome::Committed | ReducerOutcome::Deduplicated, + outcome: ReducerOutcome::Committed, .. }) => Ok(()), } @@ -1762,7 +1763,7 @@ impl ModuleHost { /// delivery using the `st_inbound_msg` dedup index. /// Before invoking the reducer, the receiver checks whether /// `sender_msg_id` ≤ the last delivered msg_id for `sender_database_identity`. - /// If so, the call is a duplicate and [`ReducerOutcome::Deduplicated`] is returned + /// If so, the call is a duplicate and the stored committed or failed result is returned /// without running the reducer. /// Otherwise the reducer runs, and the dedup index is updated atomically /// within the same transaction. diff --git a/crates/core/src/host/wasm_common/module_host_actor.rs b/crates/core/src/host/wasm_common/module_host_actor.rs index 89fa8e151dd..ecdf8680623 100644 --- a/crates/core/src/host/wasm_common/module_host_actor.rs +++ b/crates/core/src/host/wasm_common/module_host_actor.rs @@ -487,6 +487,7 @@ impl WasmModuleInstance { res } + pub fn clear_all_clients(&self) -> anyhow::Result<()> { self.common.clear_all_clients() } @@ -892,7 +893,15 @@ impl InstanceCommon { s if s == st_inbound_msg_result_status::REDUCER_ERROR => { ReducerOutcome::Failed(Box::new(stored.result_payload.into())) } - _ => ReducerOutcome::Deduplicated, + _ => { + log::warn!( + "IDC: unexpected inbound dedup result_status={} for sender {}, msg_id={}", + stored.result_status, + sender_identity, + sender_msg_id + ); + ReducerOutcome::Committed + } }; return ( ReducerCallResult { From 7745fd785e93819c84ac5032fc3f34c72b92cfdd Mon Sep 17 00:00:00 2001 From: Shubham Mishra Date: Thu, 9 Apr 2026 21:45:49 +0530 Subject: [PATCH 14/23] Preserve IDC reducer return values --- crates/bindings/src/rt.rs | 7 +-- crates/client-api/src/routes/database.rs | 17 ++++-- crates/core/src/host/host_controller.rs | 2 + crates/core/src/host/idc_actor.rs | 52 ++++++++++++------- .../src/host/wasm_common/module_host_actor.rs | 42 ++------------- .../src/locking_tx_datastore/mut_tx.rs | 3 +- crates/datastore/src/system_tables.rs | 7 ++- crates/lib/src/db/raw_def/v10.rs | 2 +- crates/schema/src/def.rs | 2 +- 9 files changed, 66 insertions(+), 68 deletions(-) diff --git a/crates/bindings/src/rt.rs b/crates/bindings/src/rt.rs index 7e24468e8ce..3214d83097c 100644 --- a/crates/bindings/src/rt.rs +++ b/crates/bindings/src/rt.rs @@ -538,11 +538,12 @@ pub const fn assert_outbox_table_primary_key() {} /// Verify at compile time that a function has the correct signature for an outbox `on_result` reducer. /// -/// The reducer must accept `(OutboxRow, Result<(), String>)` as its user-supplied arguments: -/// `fn on_result(ctx: &ReducerContext, request: OutboxRow, result: Result<(), String>)` -pub const fn outbox_typecheck<'de, OutboxRow>(_x: impl Reducer<'de, (OutboxRow, Result<(), String>)>) +/// The reducer must accept `(OutboxRow, Result)` as its user-supplied arguments: +/// `fn on_result(ctx: &ReducerContext, request: OutboxRow, result: Result)` +pub const fn outbox_typecheck<'de, OutboxRow, T>(_x: impl Reducer<'de, (OutboxRow, Result)>) where OutboxRow: spacetimedb_lib::SpacetimeType + Serialize + Deserialize<'de>, + T: spacetimedb_lib::SpacetimeType + Serialize + Deserialize<'de>, { core::mem::forget(_x); } diff --git a/crates/client-api/src/routes/database.rs b/crates/client-api/src/routes/database.rs index 03f889df997..702de8707f3 100644 --- a/crates/client-api/src/routes/database.rs +++ b/crates/client-api/src/routes/database.rs @@ -310,15 +310,26 @@ pub async fn call_from_database( match result { Ok(rcr) => { let (status, body) = match rcr.outcome { - ReducerOutcome::Committed => (StatusCode::OK, "".into()), + ReducerOutcome::Committed => ( + StatusCode::OK, + axum::body::Body::from(rcr.reducer_return_value.unwrap_or_default()), + ), // 422 = reducer ran but returned Err; the IDC actor uses this to distinguish // reducer failures from transport errors (which it retries). - ReducerOutcome::Failed(errmsg) => (StatusCode::UNPROCESSABLE_ENTITY, *errmsg), + ReducerOutcome::Failed(errmsg) => { + ( + StatusCode::UNPROCESSABLE_ENTITY, + axum::body::Body::from(errmsg.to_string()), + ) + } ReducerOutcome::BudgetExceeded => { log::warn!( "Node's energy budget exceeded for identity: {owner_identity} while executing {reducer}" ); - (StatusCode::PAYMENT_REQUIRED, "Module energy budget exhausted.".into()) + ( + StatusCode::PAYMENT_REQUIRED, + axum::body::Body::from("Module energy budget exhausted."), + ) } }; Ok(( diff --git a/crates/core/src/host/host_controller.rs b/crates/core/src/host/host_controller.rs index fb9de982bd0..aaac550ab02 100644 --- a/crates/core/src/host/host_controller.rs +++ b/crates/core/src/host/host_controller.rs @@ -23,6 +23,7 @@ use crate::util::jobs::{AllocatedJobCore, JobCores}; use crate::worker_metrics::WORKER_METRICS; use anyhow::{anyhow, bail, Context}; use async_trait::async_trait; +use bytes::Bytes; use durability::{Durability, EmptyHistory}; use log::{info, trace, warn}; use parking_lot::Mutex; @@ -139,6 +140,7 @@ impl HostRuntimes { #[derive(Debug)] pub struct ReducerCallResult { pub outcome: ReducerOutcome, + pub reducer_return_value: Option, pub energy_used: EnergyQuanta, pub execution_duration: Duration, pub tx_offset: Option, diff --git a/crates/core/src/host/idc_actor.rs b/crates/core/src/host/idc_actor.rs index 14d08ab7dd2..13bde0ed65d 100644 --- a/crates/core/src/host/idc_actor.rs +++ b/crates/core/src/host/idc_actor.rs @@ -13,8 +13,9 @@ use crate::db::relational_db::RelationalDB; use crate::host::module_host::WeakModuleHost; use crate::host::FunctionArgs; +use bytes::Bytes; use spacetimedb_datastore::execution_context::Workload; -use spacetimedb_datastore::system_tables::{StOutboundMsgRow, ST_OUTBOUND_MSG_ID}; +use spacetimedb_datastore::system_tables::{st_inbound_msg_result_status, StOutboundMsgRow, ST_OUTBOUND_MSG_ID}; use spacetimedb_datastore::traits::IsolationLevel; use spacetimedb_lib::{AlgebraicValue, Identity, ProductValue}; use spacetimedb_primitives::{ColId, TableId}; @@ -130,7 +131,7 @@ impl TargetState { /// Outcome of a delivery attempt. enum DeliveryOutcome { /// Reducer succeeded (HTTP 200). - Success, + Success(Bytes), /// Reducer ran but returned Err (HTTP 422). ReducerError(String), /// Budget exceeded (HTTP 402). @@ -211,14 +212,13 @@ async fn run_idc_loop( } /// Decode the delivery outcome into `(result_status, result_payload)` for recording. -fn outcome_to_result(outcome: &DeliveryOutcome) -> (u8, String) { - use spacetimedb_datastore::system_tables::st_inbound_msg_result_status; +fn outcome_to_result(outcome: &DeliveryOutcome) -> (u8, Bytes) { match outcome { - DeliveryOutcome::Success => (st_inbound_msg_result_status::SUCCESS, String::new()), - DeliveryOutcome::ReducerError(msg) => (st_inbound_msg_result_status::REDUCER_ERROR, msg.clone()), + DeliveryOutcome::Success(payload) => (st_inbound_msg_result_status::SUCCESS, payload.clone()), + DeliveryOutcome::ReducerError(msg) => (st_inbound_msg_result_status::REDUCER_ERROR, Bytes::from(msg.clone())), DeliveryOutcome::BudgetExceeded => ( st_inbound_msg_result_status::REDUCER_ERROR, - "budget exceeded".to_string(), + Bytes::from("budget exceeded".to_string()), ), DeliveryOutcome::TransportError(_) => unreachable!("transport errors never finalize"), } @@ -232,8 +232,8 @@ async fn finalize_message( db: &RelationalDB, module_host: &WeakModuleHost, msg: &PendingMessage, - _result_status: u8, - result_payload: String, + result_status: u8, + result_payload: Bytes, ) { // Call the on_result reducer if configured. if let Some(on_result_reducer) = &msg.on_result_reducer { @@ -247,16 +247,8 @@ async fn finalize_message( return; }; - // Encode `(request_row: OutboxRow, result: Result<(), String>)` as BSATN args. - let result = if result_payload.is_empty() { - Ok::<(), String>(()) - } else { - Err::<(), String>(result_payload) - }; let mut args_bytes = Vec::new(); - if let Err(e) = spacetimedb_sats::bsatn::to_writer(&mut args_bytes, &msg.request_row) - .and_then(|_| spacetimedb_sats::bsatn::to_writer(&mut args_bytes, &result)) - { + if let Err(e) = spacetimedb_sats::bsatn::to_writer(&mut args_bytes, &msg.request_row) { log::error!( "idc_actor: failed to encode on_result args for msg_id={}: {e}", msg.msg_id @@ -264,6 +256,28 @@ async fn finalize_message( delete_message(db, msg.msg_id); return; } + match result_status { + st_inbound_msg_result_status::SUCCESS => { + args_bytes.push(0); + args_bytes.extend_from_slice(&result_payload); + } + st_inbound_msg_result_status::REDUCER_ERROR => { + let err = String::from_utf8_lossy(&result_payload).into_owned(); + if let Err(e) = spacetimedb_sats::bsatn::to_writer(&mut args_bytes, &Err::<(), String>(err)) { + log::error!( + "idc_actor: failed to encode on_result error args for msg_id={}: {e}", + msg.msg_id + ); + delete_message(db, msg.msg_id); + return; + } + } + status => { + log::error!("idc_actor: unexpected result status {status} for msg_id={}", msg.msg_id); + delete_message(db, msg.msg_id); + return; + } + } let caller_identity = Identity::ZERO; // system call let result = host @@ -452,7 +466,7 @@ async fn attempt_delivery(client: &reqwest::Client, config: &IdcActorConfig, msg let status = resp.status(); if status.is_success() { // HTTP 200: reducer committed successfully. - DeliveryOutcome::Success + DeliveryOutcome::Success(resp.bytes().await.unwrap_or_default()) } else if status.as_u16() == 422 { // HTTP 422: reducer ran but returned Err(...). let body = resp.text().await.unwrap_or_default(); diff --git a/crates/core/src/host/wasm_common/module_host_actor.rs b/crates/core/src/host/wasm_common/module_host_actor.rs index ecdf8680623..3c1e20cad11 100644 --- a/crates/core/src/host/wasm_common/module_host_actor.rs +++ b/crates/core/src/host/wasm_common/module_host_actor.rs @@ -2,7 +2,7 @@ use super::instrumentation::CallTimes; use super::*; use crate::client::ClientActorId; use crate::database_logger; -use crate::energy::{EnergyMonitor, EnergyQuanta, FunctionBudget, FunctionFingerprint}; +use crate::energy::{EnergyMonitor, FunctionBudget, FunctionFingerprint}; use crate::error::DBError; use crate::host::host_controller::CallProcedureReturn; use crate::host::instance_env::{InstanceEnv, TxSlot}; @@ -880,41 +880,6 @@ impl InstanceCommon { let workload = Workload::Reducer(ReducerContext::from(op.clone())); let tx = tx.unwrap_or_else(|| stdb.begin_mut_tx(IsolationLevel::Serializable, workload)); - // Check the dedup index atomically within this transaction. - // If the incoming msg_id is ≤ the last delivered msg_id for this sender, - // the message is a duplicate; discard it and return the stored result. - if let Some((sender_identity, sender_msg_id)) = dedup_sender { - let stored = tx.get_inbound_msg_row(sender_identity); - if stored.as_ref().is_some_and(|r| sender_msg_id <= r.last_outbound_msg) { - let _ = stdb.rollback_mut_tx(tx); - let stored = stored.unwrap(); - let outcome = match stored.result_status { - s if s == st_inbound_msg_result_status::SUCCESS => ReducerOutcome::Committed, - s if s == st_inbound_msg_result_status::REDUCER_ERROR => { - ReducerOutcome::Failed(Box::new(stored.result_payload.into())) - } - _ => { - log::warn!( - "IDC: unexpected inbound dedup result_status={} for sender {}, msg_id={}", - stored.result_status, - sender_identity, - sender_msg_id - ); - ReducerOutcome::Committed - } - }; - return ( - ReducerCallResult { - outcome, - energy_used: EnergyQuanta::ZERO, - execution_duration: Duration::ZERO, - tx_offset: None, - }, - false, - ); - } - } - let mut tx_slot = inst.tx_slot(); let vm_metrics = self.vm_metrics.get_for_reducer_id(reducer_id); @@ -969,7 +934,7 @@ impl InstanceCommon { sender_identity, sender_msg_id, st_inbound_msg_result_status::SUCCESS, - String::new(), + return_value.clone().unwrap_or_default(), ), None => Ok(()), }; @@ -1049,7 +1014,7 @@ impl InstanceCommon { sender_identity, sender_msg_id, st_inbound_msg_result_status::REDUCER_ERROR, - err_msg, + err_msg.into(), ) { log::error!("IDC: failed to record reducer error in dedup table for sender {sender_identity}: {e}"); let _ = stdb.rollback_mut_tx(dedup_tx); @@ -1060,6 +1025,7 @@ impl InstanceCommon { let res = ReducerCallResult { outcome: ReducerOutcome::from(&event.status), + reducer_return_value: event.reducer_return_value.clone(), energy_used: energy_quanta_used, execution_duration: total_duration, tx_offset: Some(tx_offset), diff --git a/crates/datastore/src/locking_tx_datastore/mut_tx.rs b/crates/datastore/src/locking_tx_datastore/mut_tx.rs index 04abf66c5b5..b3ef124ec64 100644 --- a/crates/datastore/src/locking_tx_datastore/mut_tx.rs +++ b/crates/datastore/src/locking_tx_datastore/mut_tx.rs @@ -35,6 +35,7 @@ use crate::{ locking_tx_datastore::state_view::ScanOrIndex, traits::{InsertFlags, RowTypeForTable, TxData, UpdateFlags}, }; +use bytes::Bytes; use core::{cell::RefCell, iter, mem, ops::RangeBounds}; use itertools::Either; use smallvec::SmallVec; @@ -2636,7 +2637,7 @@ impl MutTxId { sender_identity: Identity, last_outbound_msg: u64, result_status: u8, - result_payload: String, + result_payload: Bytes, ) -> Result<()> { // Delete the existing row if present. self.delete_col_eq( diff --git a/crates/datastore/src/system_tables.rs b/crates/datastore/src/system_tables.rs index e2c35c4bebb..ff071698307 100644 --- a/crates/datastore/src/system_tables.rs +++ b/crates/datastore/src/system_tables.rs @@ -11,6 +11,7 @@ //! - Use [`st_fields_enum`] to define its column enum. //! - Register its schema in [`system_module_def`], making sure to call `validate_system_table` at the end of the function. +use bytes::Bytes; use spacetimedb_data_structures::map::{HashCollectionExt as _, HashMap}; use spacetimedb_lib::db::auth::{StAccess, StTableType}; use spacetimedb_lib::db::raw_def::v9::{btree, RawSql}; @@ -1936,8 +1937,10 @@ pub struct StInboundMsgRow { pub last_outbound_msg: u64, /// See [st_inbound_msg_result_status] for values. pub result_status: u8, - /// Error message if result_status is REDUCER_ERROR, empty otherwise. - pub result_payload: String, + /// Reducer return payload encoded as raw bytes. + /// For `SUCCESS`, this stores the committed reducer return value. + /// For `REDUCER_ERROR`, this stores the error message bytes. + pub result_payload: Bytes, } impl TryFrom> for StInboundMsgRow { diff --git a/crates/lib/src/db/raw_def/v10.rs b/crates/lib/src/db/raw_def/v10.rs index 44fe45a6889..d331a67b0e4 100644 --- a/crates/lib/src/db/raw_def/v10.rs +++ b/crates/lib/src/db/raw_def/v10.rs @@ -280,7 +280,7 @@ pub struct RawColumnDefaultValueV10 { /// /// The `remote_reducer` is the name of the reducer to call on the target database. /// If `on_result_reducer` is set, that local reducer is called when delivery completes, -/// with signature `fn on_result(ctx: &ReducerContext, request: OutboxRow, result: Result<(), String>)`. +/// with signature `fn on_result(ctx: &ReducerContext, request: OutboxRow, result: Result)`. #[derive(Debug, Clone, SpacetimeType)] #[sats(crate = crate)] #[cfg_attr(feature = "test", derive(PartialEq, Eq, PartialOrd, Ord))] diff --git a/crates/schema/src/def.rs b/crates/schema/src/def.rs index 12070c5c0ed..d4ef56b3379 100644 --- a/crates/schema/src/def.rs +++ b/crates/schema/src/def.rs @@ -723,7 +723,7 @@ pub struct OutboxDef { /// The local reducer called with the delivery result, if any. /// - /// Signature: `fn on_result(ctx: &ReducerContext, request: OutboxRow, result: Result<(), String>)`. + /// Signature: `fn on_result(ctx: &ReducerContext, request: OutboxRow, result: Result)`. pub on_result_reducer: Option, } From 3011ffd51abe6d3d8be3cfa36abaefec7eb9600f Mon Sep 17 00:00:00 2001 From: Shubham Mishra Date: Fri, 10 Apr 2026 10:10:39 +0530 Subject: [PATCH 15/23] simplify --- crates/core/src/host/idc_actor.rs | 122 +++++++++++++++++- crates/core/src/host/module_host.rs | 37 ++---- crates/core/src/host/scheduler.rs | 2 +- crates/core/src/host/v8/mod.rs | 67 +++++++++- .../src/host/wasm_common/module_host_actor.rs | 106 +++++++-------- 5 files changed, 242 insertions(+), 92 deletions(-) diff --git a/crates/core/src/host/idc_actor.rs b/crates/core/src/host/idc_actor.rs index 13bde0ed65d..c303c1e07a3 100644 --- a/crates/core/src/host/idc_actor.rs +++ b/crates/core/src/host/idc_actor.rs @@ -11,10 +11,13 @@ /// `on_result_reducer` (read from the outbox table's schema) and deletes the st_outbound_msg row. /// 6. Enforces sequential delivery per target database: msg N+1 is only delivered after N is done. use crate::db::relational_db::RelationalDB; -use crate::host::module_host::WeakModuleHost; -use crate::host::FunctionArgs; +use crate::energy::EnergyQuanta; +use crate::host::module_host::{CallReducerParams, ModuleInfo, WeakModuleHost}; +use crate::host::{FunctionArgs, ReducerCallResult, ReducerOutcome}; +use anyhow::anyhow; use bytes::Bytes; -use spacetimedb_datastore::execution_context::Workload; +use spacetimedb_datastore::execution_context::{ReducerContext, Workload}; +use spacetimedb_datastore::locking_tx_datastore::MutTxId; use spacetimedb_datastore::system_tables::{st_inbound_msg_result_status, StOutboundMsgRow, ST_OUTBOUND_MSG_ID}; use spacetimedb_datastore::traits::IsolationLevel; use spacetimedb_lib::{AlgebraicValue, Identity, ProductValue}; @@ -224,6 +227,119 @@ fn outcome_to_result(outcome: &DeliveryOutcome) -> (u8, Bytes) { } } +pub(crate) type ReducerSuccessAction = Box) -> anyhow::Result<()> + Send>; + +fn reducer_workload(module: &ModuleInfo, params: &CallReducerParams) -> Workload { + let reducer_def = module.module_def.reducer_by_id(params.reducer_id); + Workload::Reducer(ReducerContext { + name: reducer_def.name.clone(), + caller_identity: params.caller_identity, + caller_connection_id: params.caller_connection_id, + timestamp: params.timestamp, + arg_bsatn: params.args.get_bsatn().clone(), + }) +} + +fn duplicate_result_from_row(row: spacetimedb_datastore::system_tables::StInboundMsgRow) -> ReducerCallResult { + let outcome = match row.result_status { + st_inbound_msg_result_status::SUCCESS => ReducerOutcome::Committed, + st_inbound_msg_result_status::REDUCER_ERROR => ReducerOutcome::Failed(Box::new( + String::from_utf8_lossy(&row.result_payload).into_owned().into_boxed_str(), + )), + status => { + log::warn!( + "IDC: unexpected inbound dedup result_status={} for sender {}", + status, + Identity::from(row.database_identity) + ); + ReducerOutcome::Failed(Box::new("unexpected inbound dedup result status".into())) + } + }; + + ReducerCallResult { + outcome, + reducer_return_value: (row.result_status == st_inbound_msg_result_status::SUCCESS).then_some(row.result_payload), + energy_used: EnergyQuanta::ZERO, + execution_duration: Duration::ZERO, + tx_offset: None, + } +} + +fn record_failed_inbound_result(db: &RelationalDB, sender_identity: Identity, sender_msg_id: u64, error: &str) { + let mut dedup_tx = db.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); + if let Err(e) = dedup_tx.upsert_inbound_last_msg( + sender_identity, + sender_msg_id, + st_inbound_msg_result_status::REDUCER_ERROR, + error.to_string().into(), + ) { + log::error!("IDC: failed to record reducer error in dedup table for sender {sender_identity}: {e}"); + let _ = db.rollback_mut_tx(dedup_tx); + } else if let Err(e) = db.commit_tx(dedup_tx) { + log::error!("IDC: failed to commit dedup error record for sender {sender_identity}: {e}"); + } +} + +pub(crate) fn call_reducer_from_database( + module: &ModuleInfo, + db: &RelationalDB, + params: CallReducerParams, + sender_identity: Identity, + sender_msg_id: u64, + call_reducer: F, +) -> (ReducerCallResult, bool) +where + F: FnOnce(Option, CallReducerParams, ReducerSuccessAction) -> (ReducerCallResult, bool), +{ + let tx = db.begin_mut_tx(IsolationLevel::Serializable, reducer_workload(module, ¶ms)); + if let Some(row) = tx.get_inbound_msg_row(sender_identity) + && sender_msg_id <= row.last_outbound_msg + { + let _ = db.rollback_mut_tx(tx); + return (duplicate_result_from_row(row), false); + } + + let (result, trapped) = call_reducer( + Some(tx), + params, + Box::new(move |tx, reducer_return_value| { + tx.upsert_inbound_last_msg( + sender_identity, + sender_msg_id, + st_inbound_msg_result_status::SUCCESS, + reducer_return_value.clone().unwrap_or_default(), + ) + .map_err(anyhow::Error::from) + }), + ); + + if let ReducerOutcome::Failed(err) = &result.outcome { + record_failed_inbound_result(db, sender_identity, sender_msg_id, err); + } + + (result, trapped) +} + +pub(crate) fn call_reducer_delete_outbound_on_success( + module: &ModuleInfo, + db: &RelationalDB, + params: CallReducerParams, + msg_id: u64, + call_reducer: F, +) -> (ReducerCallResult, bool) +where + F: FnOnce(Option, CallReducerParams, ReducerSuccessAction) -> (ReducerCallResult, bool), +{ + let tx = db.begin_mut_tx(IsolationLevel::Serializable, reducer_workload(module, ¶ms)); + call_reducer( + Some(tx), + params, + Box::new(move |tx, _reducer_return_value| { + tx.delete_outbound_msg(msg_id).map_err(|e| anyhow!(e)) + }), + ) +} + /// Finalize a delivered message: call the on_result reducer (if any), then delete from ST_OUTBOUND_MSG. /// /// On the happy path, `on_result_reducer` success and deletion of `st_outbound_msg` diff --git a/crates/core/src/host/module_host.rs b/crates/core/src/host/module_host.rs index 1337e171f8b..918808ac156 100644 --- a/crates/core/src/host/module_host.rs +++ b/crates/core/src/host/module_host.rs @@ -624,16 +624,6 @@ pub struct CallReducerParams { pub timer: Option, pub reducer_id: ReducerId, pub args: ArgsTuple, - /// If set, enables at-most-once delivery semantics for database-to-database calls. - /// - /// The tuple is `(sender_database_identity, sender_msg_id)`. - /// Before running the reducer, `st_inbound_msg` is consulted: - /// if `sender_msg_id` ≤ the stored last-delivered msg_id for the sender, - /// the call is a duplicate and returns the stored committed or failed result - /// without running the reducer again. - /// Otherwise the reducer runs, and the stored msg_id is updated atomically - /// within the same transaction. - pub dedup_sender: Option<(Identity, u64)>, } impl CallReducerParams { @@ -654,7 +644,6 @@ impl CallReducerParams { timer: None, reducer_id, args, - dedup_sender: None, } } } @@ -1578,7 +1567,6 @@ impl ModuleHost { timer, reducer_id, args, - dedup_sender: None, }) } @@ -1606,7 +1594,6 @@ impl ModuleHost { timer, reducer_id, args, - dedup_sender: None, }; self.call( @@ -1643,7 +1630,6 @@ impl ModuleHost { timer, reducer_id, args, - dedup_sender: None, }; self.call( @@ -1793,27 +1779,28 @@ impl ModuleHost { return Err(ReducerCallError::NoSuchReducer); } - let args = args - .into_tuple_for_def(&self.info.module_def, reducer_def) - .map_err(InvalidReducerArguments)?; - let caller_connection_id = caller_connection_id.unwrap_or(ConnectionId::ZERO); - let call_reducer_params = CallReducerParams { - timestamp: Timestamp::now(), + let call_reducer_params = Self::call_reducer_params( + &self.info, caller_identity, caller_connection_id, client, request_id, timer, reducer_id, + reducer_def, args, - dedup_sender: Some((sender_database_identity, sender_msg_id)), - }; + )?; self.call( &reducer_def.name, - call_reducer_params, - async |p, inst| Ok(inst.call_reducer(p)), - async |p, inst| inst.call_reducer(p).await, + (call_reducer_params, sender_database_identity, sender_msg_id), + async |(p, sender_database_identity, sender_msg_id), inst| { + Ok(inst.call_reducer_from_database(p, sender_database_identity, sender_msg_id)) + }, + async |(p, sender_database_identity, sender_msg_id), inst| { + inst.call_reducer_from_database(p, sender_database_identity, sender_msg_id) + .await + }, ) .await? } diff --git a/crates/core/src/host/scheduler.rs b/crates/core/src/host/scheduler.rs index d3b285e9f16..eac5ca47110 100644 --- a/crates/core/src/host/scheduler.rs +++ b/crates/core/src/host/scheduler.rs @@ -443,7 +443,7 @@ pub(super) async fn call_scheduled_function( // print their message and backtrace when they occur, so we don't need to do // anything with the error payload. let result = panic::catch_unwind(panic::AssertUnwindSafe(|| { - inst_common.call_reducer_with_tx(Some(tx), params, inst) + inst_common.call_reducer_with_tx(Some(tx), params, inst, |_tx, _ret| Ok(())) })); let reschedule = delete_scheduled_function_row(module_info, db, id, None, None, inst_common, inst); // Currently, we drop the return value from the function call. In the future, diff --git a/crates/core/src/host/v8/mod.rs b/crates/core/src/host/v8/mod.rs index f3be86dc2de..3dd5e9fa5b4 100644 --- a/crates/core/src/host/v8/mod.rs +++ b/crates/core/src/host/v8/mod.rs @@ -586,6 +586,13 @@ enum JsWorkerRequest { reply_tx: JsReplyTx, params: CallReducerParams, }, + /// See [`JsInstance::call_reducer_from_database`]. + CallReducerFromDatabase { + reply_tx: JsReplyTx, + params: CallReducerParams, + sender_identity: Identity, + sender_msg_id: u64, + }, /// See [`JsInstance::call_reducer_delete_outbound_on_success`]. CallReducerDeleteOutboundOnSuccess { reply_tx: JsReplyTx, @@ -633,7 +640,7 @@ enum JsWorkerRequest { }, } -static_assert_size!(CallReducerParams, 256); +static_assert_size!(CallReducerParams, 192); fn send_worker_reply(ctx: &str, reply_tx: JsReplyTx, value: T, trapped: bool) { if reply_tx.send(JsWorkerReply { value, trapped }).is_err() { @@ -908,6 +915,25 @@ impl JsInstanceLane { .map_err(|_| ReducerCallError::WorkerError(instance_lane_worker_error("call_reducer"))) } + pub async fn call_reducer_from_database( + &self, + params: CallReducerParams, + sender_identity: Identity, + sender_msg_id: u64, + ) -> Result { + self.run_once("call_reducer", |inst: JsInstance| async move { + inst.send_request(|reply_tx| JsWorkerRequest::CallReducerFromDatabase { + reply_tx, + params, + sender_identity, + sender_msg_id, + }) + .await + }) + .await + .map_err(|_| ReducerCallError::WorkerError(instance_lane_worker_error("call_reducer"))) + } + pub async fn call_reducer_delete_outbound_on_success( &self, params: CallReducerParams, @@ -1166,7 +1192,15 @@ async fn spawn_instance_worker( let mut requests_since_heap_check = 0u64; let mut last_heap_check_at = Instant::now(); for request in request_rx.iter() { - let mut call_reducer = |tx, params| instance_common.call_reducer_with_tx(tx, params, &mut inst); + let info_for_idc = instance_common.info(); + let db_for_idc = inst.replica_ctx().relational_db().clone(); + let mut call_reducer_with_success = + |tx, params, on_success: crate::host::idc_actor::ReducerSuccessAction| { + instance_common.call_reducer_with_tx(tx, params, &mut inst, on_success) + }; + let mut call_reducer = |tx, params| { + call_reducer_with_success(tx, params, Box::new(|_tx, _ret| Ok(()))) + }; let mut should_exit = false; core_pinner.pin_if_changed(); @@ -1188,15 +1222,36 @@ async fn spawn_instance_worker( send_worker_reply("call_reducer", reply_tx, res, trapped); should_exit = trapped; } + JsWorkerRequest::CallReducerFromDatabase { + reply_tx, + params, + sender_identity, + sender_msg_id, + } => { + let (res, trapped) = crate::host::idc_actor::call_reducer_from_database( + info_for_idc.as_ref(), + db_for_idc.as_ref(), + params, + sender_identity, + sender_msg_id, + |tx, params, on_success| call_reducer_with_success(tx, params, on_success), + ); + worker_trapped.store(trapped, Ordering::Relaxed); + send_worker_reply("call_reducer", reply_tx, res, trapped); + should_exit = trapped; + } JsWorkerRequest::CallReducerDeleteOutboundOnSuccess { reply_tx, params, msg_id, } => { - let (res, trapped) = - instance_common.call_reducer_with_tx_and_success_action(None, params, &mut inst, |tx| { - tx.delete_outbound_msg(msg_id).map_err(anyhow::Error::from) - }); + let (res, trapped) = crate::host::idc_actor::call_reducer_delete_outbound_on_success( + info_for_idc.as_ref(), + db_for_idc.as_ref(), + params, + msg_id, + |tx, params, on_success| call_reducer_with_success(tx, params, on_success), + ); worker_trapped.store(trapped, Ordering::Relaxed); send_worker_reply("call_reducer", reply_tx, res, trapped); should_exit = trapped; diff --git a/crates/core/src/host/wasm_common/module_host_actor.rs b/crates/core/src/host/wasm_common/module_host_actor.rs index 3c1e20cad11..f946df44175 100644 --- a/crates/core/src/host/wasm_common/module_host_actor.rs +++ b/crates/core/src/host/wasm_common/module_host_actor.rs @@ -37,7 +37,6 @@ use spacetimedb_datastore::db_metrics::DB_METRICS; use spacetimedb_datastore::error::{DatastoreError, ViewError}; use spacetimedb_datastore::execution_context::{self, ReducerContext, Workload}; use spacetimedb_datastore::locking_tx_datastore::{FuncCallType, MutTxId, ViewCallInfo}; -use spacetimedb_datastore::system_tables::st_inbound_msg_result_status; use spacetimedb_datastore::traits::{IsolationLevel, Program}; use spacetimedb_execution::pipelined::PipelinedProject; use spacetimedb_lib::buffer::DecodeError; @@ -467,7 +466,29 @@ impl WasmModuleInstance { } pub fn call_reducer(&mut self, params: CallReducerParams) -> ReducerCallResult { - let (res, trapped) = self.call_reducer_with_tx(None, params); + let (res, trapped) = self.call_reducer_with_tx(None, params, |_tx, _ret| Ok(())); + self.trapped = trapped; + res + } + + pub fn call_reducer_from_database( + &mut self, + params: CallReducerParams, + sender_identity: Identity, + sender_msg_id: u64, + ) -> ReducerCallResult { + let info = self.common.info.clone(); + let db = self.instance.replica_ctx().relational_db().clone(); + let (res, trapped) = crate::callgrind_flag::invoke_allowing_callgrind(|| { + crate::host::idc_actor::call_reducer_from_database( + info.as_ref(), + db.as_ref(), + params, + sender_identity, + sender_msg_id, + |tx, params, on_success| self.call_reducer_with_tx(tx, params, on_success), + ) + }); self.trapped = trapped; res } @@ -477,11 +498,16 @@ impl WasmModuleInstance { params: CallReducerParams, msg_id: u64, ) -> ReducerCallResult { + let info = self.common.info.clone(); + let db = self.instance.replica_ctx().relational_db().clone(); let (res, trapped) = crate::callgrind_flag::invoke_allowing_callgrind(|| { - self.common - .call_reducer_with_tx_and_success_action(None, params, &mut self.instance, |tx| { - tx.delete_outbound_msg(msg_id).map_err(anyhow::Error::from) - }) + crate::host::idc_actor::call_reducer_delete_outbound_on_success( + info.as_ref(), + db.as_ref(), + params, + msg_id, + |tx, params, on_success| self.call_reducer_with_tx(tx, params, on_success), + ) }); self.trapped = trapped; res @@ -498,7 +524,7 @@ impl WasmModuleInstance { caller_connection_id: ConnectionId, ) -> Result<(), ClientConnectedError> { let module = &self.common.info.clone(); - let call_reducer = |tx, params| self.call_reducer_with_tx(tx, params); + let call_reducer = |tx, params| self.call_reducer_with_tx(tx, params, |_tx, _ret| Ok(())); let mut trapped = false; let res = call_identity_connected(caller_auth, caller_connection_id, module, call_reducer, &mut trapped); self.trapped = trapped; @@ -511,7 +537,7 @@ impl WasmModuleInstance { caller_connection_id: ConnectionId, ) -> Result<(), ReducerCallError> { let module = &self.common.info.clone(); - let call_reducer = |tx, params| self.call_reducer_with_tx(tx, params); + let call_reducer = |tx, params| self.call_reducer_with_tx(tx, params, |_tx, _ret| Ok(())); let mut trapped = false; let res = ModuleHost::call_identity_disconnected_inner( caller_identity, @@ -526,7 +552,7 @@ impl WasmModuleInstance { pub fn disconnect_client(&mut self, client_id: ClientActorId) -> Result<(), ReducerCallError> { let module = &self.common.info.clone(); - let call_reducer = |tx, params| self.call_reducer_with_tx(tx, params); + let call_reducer = |tx, params| self.call_reducer_with_tx(tx, params, |_tx, _ret| Ok(())); let mut trapped = false; let res = ModuleHost::disconnect_client_inner(client_id, module, call_reducer, &mut trapped); self.trapped = trapped; @@ -536,7 +562,7 @@ impl WasmModuleInstance { pub fn init_database(&mut self, program: Program) -> anyhow::Result> { let module_def = &self.common.info.clone().module_def; let replica_ctx = &self.instance.replica_ctx().clone(); - let call_reducer = |tx, params| self.call_reducer_with_tx(tx, params); + let call_reducer = |tx, params| self.call_reducer_with_tx(tx, params, |_tx, _ret| Ok(())); let (res, trapped) = init_database(replica_ctx, module_def, program, call_reducer); self.trapped = trapped; res @@ -560,9 +586,17 @@ impl WasmModuleInstance { impl WasmModuleInstance { #[tracing::instrument(level = "trace", skip_all)] - fn call_reducer_with_tx(&mut self, tx: Option, params: CallReducerParams) -> (ReducerCallResult, bool) { + fn call_reducer_with_tx( + &mut self, + tx: Option, + params: CallReducerParams, + on_success: F, + ) -> (ReducerCallResult, bool) + where + F: FnOnce(&mut MutTxId, &Option) -> anyhow::Result<()>, + { crate::callgrind_flag::invoke_allowing_callgrind(|| { - self.common.call_reducer_with_tx(tx, params, &mut self.instance) + self.common.call_reducer_with_tx(tx, params, &mut self.instance, on_success) }) } @@ -826,16 +860,7 @@ impl InstanceCommon { /// /// The `bool` in the return type signifies whether there was an "outer error". /// For WASM, this should be interpreted as a trap occurring. - pub(crate) fn call_reducer_with_tx( - &mut self, - tx: Option, - params: CallReducerParams, - inst: &mut I, - ) -> (ReducerCallResult, bool) { - self.call_reducer_with_tx_and_success_action(tx, params, inst, |_tx| Ok(())) - } - - pub(crate) fn call_reducer_with_tx_and_success_action( + pub(crate) fn call_reducer_with_tx( &mut self, tx: Option, params: CallReducerParams, @@ -843,7 +868,7 @@ impl InstanceCommon { on_success: F, ) -> (ReducerCallResult, bool) where - F: FnOnce(&mut MutTxId) -> anyhow::Result<()>, + F: FnOnce(&mut MutTxId, &Option) -> anyhow::Result<()>, { let CallReducerParams { timestamp, @@ -854,7 +879,6 @@ impl InstanceCommon { reducer_id, args, timer, - dedup_sender, } = params; let caller_connection_id_opt = (caller_connection_id != ConnectionId::ZERO).then_some(caller_connection_id); @@ -927,21 +951,9 @@ impl InstanceCommon { } _ => Ok(()), }; - // Atomically update the dedup index so this sender's tx_offset is recorded - // as delivered. This prevents re-processing if the message is retried. - let dedup_res = match dedup_sender { - Some((sender_identity, sender_msg_id)) => tx.upsert_inbound_last_msg( - sender_identity, - sender_msg_id, - st_inbound_msg_result_status::SUCCESS, - return_value.clone().unwrap_or_default(), - ), - None => Ok(()), - }; let res = lifecycle_res .map_err(anyhow::Error::from) - .and(dedup_res.map_err(anyhow::Error::from)) - .and_then(|_| on_success(&mut tx)); + .and_then(|_| on_success(&mut tx, &return_value)); match res { Ok(()) => (EventStatus::Committed(DatabaseUpdate::default()), return_value), Err(err) => { @@ -1003,26 +1015,6 @@ impl InstanceCommon { let tx_offset = committed.tx_offset; let event = committed.event; - // For IDC (database-to-database) calls: if the reducer failed with a user error, - // record the failure in st_inbound_msg in a separate tx (since the reducer tx - // was rolled back). This allows the sending database to receive the error on dedup - // rather than re-running the reducer. - if let (Some((sender_identity, sender_msg_id)), EventStatus::FailedUser(err)) = (dedup_sender, &event.status) { - let err_msg = err.clone(); - let mut dedup_tx = stdb.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); - if let Err(e) = dedup_tx.upsert_inbound_last_msg( - sender_identity, - sender_msg_id, - st_inbound_msg_result_status::REDUCER_ERROR, - err_msg.into(), - ) { - log::error!("IDC: failed to record reducer error in dedup table for sender {sender_identity}: {e}"); - let _ = stdb.rollback_mut_tx(dedup_tx); - } else if let Err(e) = stdb.commit_tx(dedup_tx) { - log::error!("IDC: failed to commit dedup error record for sender {sender_identity}: {e}"); - } - } - let res = ReducerCallResult { outcome: ReducerOutcome::from(&event.status), reducer_return_value: event.reducer_return_value.clone(), From 37853a22b51962807ea41a13fdef792326de1d5a Mon Sep 17 00:00:00 2001 From: Shubham Mishra Date: Fri, 10 Apr 2026 15:45:25 +0530 Subject: [PATCH 16/23] some refactoring --- crates/client-api/src/routes/database.rs | 3 +- crates/core/src/host/idc_actor.rs | 85 ++++++++++--------- crates/core/src/host/module_host.rs | 26 +++--- crates/core/src/host/v8/mod.rs | 40 +++++---- .../src/host/wasm_common/module_host_actor.rs | 14 +-- .../src/locking_tx_datastore/mut_tx.rs | 5 +- crates/datastore/src/system_tables.rs | 49 ++++++++--- 7 files changed, 132 insertions(+), 90 deletions(-) diff --git a/crates/client-api/src/routes/database.rs b/crates/client-api/src/routes/database.rs index 702de8707f3..33140c8a302 100644 --- a/crates/client-api/src/routes/database.rs +++ b/crates/client-api/src/routes/database.rs @@ -103,6 +103,7 @@ fn map_reducer_error(e: ReducerCallError, reducer: &str) -> (StatusCode, String) log::debug!("Attempt to call {lifecycle:?} lifecycle reducer {reducer}"); StatusCode::BAD_REQUEST } + ReducerCallError::OutOfOrderInboundMessage { .. } => StatusCode::BAD_REQUEST, }; log::debug!("Error while invoking reducer {e:#}"); @@ -244,7 +245,7 @@ pub struct CallFromDatabaseQuery { /// - `msg_id` — the inter-database message ID from the sender's st_outbound_msg. /// /// Before invoking the reducer, the receiver checks `st_inbound_msg`. -/// If the incoming `msg_id` is ≤ the last delivered msg_id for `sender_identity`, +/// If the incoming `msg_id` is the last delivered msg_id for `sender_identity`, /// the call is a duplicate and 200 OK is returned immediately without running the reducer. /// Otherwise the reducer is invoked, the dedup index is updated atomically in the same /// transaction, and an acknowledgment is returned on success. diff --git a/crates/core/src/host/idc_actor.rs b/crates/core/src/host/idc_actor.rs index c303c1e07a3..70ccab5b5a6 100644 --- a/crates/core/src/host/idc_actor.rs +++ b/crates/core/src/host/idc_actor.rs @@ -13,12 +13,14 @@ use crate::db::relational_db::RelationalDB; use crate::energy::EnergyQuanta; use crate::host::module_host::{CallReducerParams, ModuleInfo, WeakModuleHost}; -use crate::host::{FunctionArgs, ReducerCallResult, ReducerOutcome}; +use crate::host::{FunctionArgs, ReducerCallError, ReducerCallResult, ReducerOutcome}; use anyhow::anyhow; use bytes::Bytes; use spacetimedb_datastore::execution_context::{ReducerContext, Workload}; use spacetimedb_datastore::locking_tx_datastore::MutTxId; -use spacetimedb_datastore::system_tables::{st_inbound_msg_result_status, StOutboundMsgRow, ST_OUTBOUND_MSG_ID}; +use spacetimedb_datastore::system_tables::{ + StInboundMsgResultStatus, StOutboundMsgRow, ST_OUTBOUND_MSG_ID, +}; use spacetimedb_datastore::traits::IsolationLevel; use spacetimedb_lib::{AlgebraicValue, Identity, ProductValue}; use spacetimedb_primitives::{ColId, TableId}; @@ -215,12 +217,12 @@ async fn run_idc_loop( } /// Decode the delivery outcome into `(result_status, result_payload)` for recording. -fn outcome_to_result(outcome: &DeliveryOutcome) -> (u8, Bytes) { +fn outcome_to_result(outcome: &DeliveryOutcome) -> (StInboundMsgResultStatus, Bytes) { match outcome { - DeliveryOutcome::Success(payload) => (st_inbound_msg_result_status::SUCCESS, payload.clone()), - DeliveryOutcome::ReducerError(msg) => (st_inbound_msg_result_status::REDUCER_ERROR, Bytes::from(msg.clone())), + DeliveryOutcome::Success(payload) => (StInboundMsgResultStatus::Success, payload.clone()), + DeliveryOutcome::ReducerError(msg) => (StInboundMsgResultStatus::ReducerError, Bytes::from(msg.clone())), DeliveryOutcome::BudgetExceeded => ( - st_inbound_msg_result_status::REDUCER_ERROR, + StInboundMsgResultStatus::ReducerError, Bytes::from("budget exceeded".to_string()), ), DeliveryOutcome::TransportError(_) => unreachable!("transport errors never finalize"), @@ -229,6 +231,11 @@ fn outcome_to_result(outcome: &DeliveryOutcome) -> (u8, Bytes) { pub(crate) type ReducerSuccessAction = Box) -> anyhow::Result<()> + Send>; +#[derive(Debug, Clone, Copy)] +pub enum ReducerSuccessActionKind { + DeleteOutboundMsg(u64), +} + fn reducer_workload(module: &ModuleInfo, params: &CallReducerParams) -> Workload { let reducer_def = module.module_def.reducer_by_id(params.reducer_id); Workload::Reducer(ReducerContext { @@ -242,23 +249,15 @@ fn reducer_workload(module: &ModuleInfo, params: &CallReducerParams) -> Workload fn duplicate_result_from_row(row: spacetimedb_datastore::system_tables::StInboundMsgRow) -> ReducerCallResult { let outcome = match row.result_status { - st_inbound_msg_result_status::SUCCESS => ReducerOutcome::Committed, - st_inbound_msg_result_status::REDUCER_ERROR => ReducerOutcome::Failed(Box::new( + StInboundMsgResultStatus::Success => ReducerOutcome::Committed, + StInboundMsgResultStatus::ReducerError => ReducerOutcome::Failed(Box::new( String::from_utf8_lossy(&row.result_payload).into_owned().into_boxed_str(), )), - status => { - log::warn!( - "IDC: unexpected inbound dedup result_status={} for sender {}", - status, - Identity::from(row.database_identity) - ); - ReducerOutcome::Failed(Box::new("unexpected inbound dedup result status".into())) - } }; ReducerCallResult { outcome, - reducer_return_value: (row.result_status == st_inbound_msg_result_status::SUCCESS).then_some(row.result_payload), + reducer_return_value: (row.result_status == StInboundMsgResultStatus::Success).then_some(row.result_payload), energy_used: EnergyQuanta::ZERO, execution_duration: Duration::ZERO, tx_offset: None, @@ -270,7 +269,7 @@ fn record_failed_inbound_result(db: &RelationalDB, sender_identity: Identity, se if let Err(e) = dedup_tx.upsert_inbound_last_msg( sender_identity, sender_msg_id, - st_inbound_msg_result_status::REDUCER_ERROR, + StInboundMsgResultStatus::ReducerError, error.to_string().into(), ) { log::error!("IDC: failed to record reducer error in dedup table for sender {sender_identity}: {e}"); @@ -287,16 +286,25 @@ pub(crate) fn call_reducer_from_database( sender_identity: Identity, sender_msg_id: u64, call_reducer: F, -) -> (ReducerCallResult, bool) +) -> Result<(ReducerCallResult, bool), ReducerCallError> where F: FnOnce(Option, CallReducerParams, ReducerSuccessAction) -> (ReducerCallResult, bool), { let tx = db.begin_mut_tx(IsolationLevel::Serializable, reducer_workload(module, ¶ms)); - if let Some(row) = tx.get_inbound_msg_row(sender_identity) - && sender_msg_id <= row.last_outbound_msg - { - let _ = db.rollback_mut_tx(tx); - return (duplicate_result_from_row(row), false); + if let Some(row) = tx.get_inbound_msg_row(sender_identity) { + if sender_msg_id == row.last_outbound_msg { + let _ = db.rollback_mut_tx(tx); + return Ok((duplicate_result_from_row(row), false)); + } + if sender_msg_id < row.last_outbound_msg { + let expected = row.last_outbound_msg + 1; + let _ = db.rollback_mut_tx(tx); + return Err(ReducerCallError::OutOfOrderInboundMessage { + sender: sender_identity, + expected, + received: sender_msg_id, + }); + } } let (result, trapped) = call_reducer( @@ -306,7 +314,7 @@ where tx.upsert_inbound_last_msg( sender_identity, sender_msg_id, - st_inbound_msg_result_status::SUCCESS, + StInboundMsgResultStatus::Success, reducer_return_value.clone().unwrap_or_default(), ) .map_err(anyhow::Error::from) @@ -317,14 +325,14 @@ where record_failed_inbound_result(db, sender_identity, sender_msg_id, err); } - (result, trapped) + Ok((result, trapped)) } -pub(crate) fn call_reducer_delete_outbound_on_success( +pub(crate) fn call_reducer_with_success_action( module: &ModuleInfo, db: &RelationalDB, params: CallReducerParams, - msg_id: u64, + action: ReducerSuccessActionKind, call_reducer: F, ) -> (ReducerCallResult, bool) where @@ -335,7 +343,11 @@ where Some(tx), params, Box::new(move |tx, _reducer_return_value| { - tx.delete_outbound_msg(msg_id).map_err(|e| anyhow!(e)) + match action { + ReducerSuccessActionKind::DeleteOutboundMsg(msg_id) => { + tx.delete_outbound_msg(msg_id).map_err(|e| anyhow!(e)) + } + } }), ) } @@ -348,7 +360,7 @@ async fn finalize_message( db: &RelationalDB, module_host: &WeakModuleHost, msg: &PendingMessage, - result_status: u8, + result_status: StInboundMsgResultStatus, result_payload: Bytes, ) { // Call the on_result reducer if configured. @@ -373,11 +385,11 @@ async fn finalize_message( return; } match result_status { - st_inbound_msg_result_status::SUCCESS => { + StInboundMsgResultStatus::Success => { args_bytes.push(0); args_bytes.extend_from_slice(&result_payload); } - st_inbound_msg_result_status::REDUCER_ERROR => { + StInboundMsgResultStatus::ReducerError => { let err = String::from_utf8_lossy(&result_payload).into_owned(); if let Err(e) = spacetimedb_sats::bsatn::to_writer(&mut args_bytes, &Err::<(), String>(err)) { log::error!( @@ -388,16 +400,11 @@ async fn finalize_message( return; } } - status => { - log::error!("idc_actor: unexpected result status {status} for msg_id={}", msg.msg_id); - delete_message(db, msg.msg_id); - return; - } } let caller_identity = Identity::ZERO; // system call let result = host - .call_reducer_delete_outbound_on_success( + .call_reducer_with_success_action( caller_identity, None, // no connection_id None, // no client sender @@ -405,7 +412,7 @@ async fn finalize_message( None, // no timer on_result_reducer, FunctionArgs::Bsatn(bytes::Bytes::from(args_bytes)), - msg.msg_id, + ReducerSuccessActionKind::DeleteOutboundMsg(msg.msg_id), ) .await; diff --git a/crates/core/src/host/module_host.rs b/crates/core/src/host/module_host.rs index 918808ac156..56c74bde00b 100644 --- a/crates/core/src/host/module_host.rs +++ b/crates/core/src/host/module_host.rs @@ -946,6 +946,12 @@ pub enum ReducerCallError { ScheduleReducerNotFound, #[error("can't directly call special {0:?} lifecycle reducer")] LifecycleReducer(Lifecycle), + #[error("out-of-order inbound message id {received} from {sender}; expected {expected}")] + OutOfOrderInboundMessage { + sender: Identity, + expected: u64, + received: u64, + }, } #[derive(Debug, PartialEq, Eq)] @@ -1605,7 +1611,7 @@ impl ModuleHost { .await? } - async fn call_reducer_delete_outbound_on_success_inner( + async fn call_reducer_with_success_action_inner( &self, caller_identity: Identity, caller_connection_id: Option, @@ -1615,7 +1621,7 @@ impl ModuleHost { reducer_id: ReducerId, reducer_def: &ReducerDef, args: FunctionArgs, - msg_id: u64, + action: crate::host::idc_actor::ReducerSuccessActionKind, ) -> Result { let args = args .into_tuple_for_def(&self.info.module_def, reducer_def) @@ -1634,9 +1640,9 @@ impl ModuleHost { self.call( &reducer_def.name, - (call_reducer_params, msg_id), - async |(p, msg_id), inst| Ok(inst.call_reducer_delete_outbound_on_success(p, msg_id)), - async |(p, msg_id), inst| inst.call_reducer_delete_outbound_on_success(p, msg_id).await, + (call_reducer_params, action), + async |(p, action), inst| Ok(inst.call_reducer_with_success_action(p, action)), + async |(p, action), inst| inst.call_reducer_with_success_action(p, action).await, ) .await? } @@ -1691,7 +1697,7 @@ impl ModuleHost { res } - pub async fn call_reducer_delete_outbound_on_success( + pub async fn call_reducer_with_success_action( &self, caller_identity: Identity, caller_connection_id: Option, @@ -1700,7 +1706,7 @@ impl ModuleHost { timer: Option, reducer_name: &str, args: FunctionArgs, - msg_id: u64, + action: crate::host::idc_actor::ReducerSuccessActionKind, ) -> Result { let res = async { let (reducer_id, reducer_def) = self @@ -1716,7 +1722,7 @@ impl ModuleHost { return Err(ReducerCallError::NoSuchReducer); } - self.call_reducer_delete_outbound_on_success_inner( + self.call_reducer_with_success_action_inner( caller_identity, caller_connection_id, client, @@ -1725,7 +1731,7 @@ impl ModuleHost { reducer_id, reducer_def, args, - msg_id, + action, ) .await } @@ -1795,7 +1801,7 @@ impl ModuleHost { &reducer_def.name, (call_reducer_params, sender_database_identity, sender_msg_id), async |(p, sender_database_identity, sender_msg_id), inst| { - Ok(inst.call_reducer_from_database(p, sender_database_identity, sender_msg_id)) + inst.call_reducer_from_database(p, sender_database_identity, sender_msg_id) }, async |(p, sender_database_identity, sender_msg_id), inst| { inst.call_reducer_from_database(p, sender_database_identity, sender_msg_id) diff --git a/crates/core/src/host/v8/mod.rs b/crates/core/src/host/v8/mod.rs index 3dd5e9fa5b4..4b183c00d2b 100644 --- a/crates/core/src/host/v8/mod.rs +++ b/crates/core/src/host/v8/mod.rs @@ -468,15 +468,15 @@ impl JsInstance { .unwrap_or_else(|_| panic!("worker should stay live while calling a reducer")) } - pub async fn call_reducer_delete_outbound_on_success( + pub async fn call_reducer_with_success_action( &self, params: CallReducerParams, - msg_id: u64, + action: crate::host::idc_actor::ReducerSuccessActionKind, ) -> ReducerCallResult { - self.send_request(|reply_tx| JsWorkerRequest::CallReducerDeleteOutboundOnSuccess { + self.send_request(|reply_tx| JsWorkerRequest::CallReducerWithSuccessAction { reply_tx, params, - msg_id, + action, }) .await .unwrap_or_else(|_| panic!("worker should stay live while calling a reducer")) @@ -588,16 +588,16 @@ enum JsWorkerRequest { }, /// See [`JsInstance::call_reducer_from_database`]. CallReducerFromDatabase { - reply_tx: JsReplyTx, + reply_tx: JsReplyTx>, params: CallReducerParams, sender_identity: Identity, sender_msg_id: u64, }, - /// See [`JsInstance::call_reducer_delete_outbound_on_success`]. - CallReducerDeleteOutboundOnSuccess { + /// See [`JsInstance::call_reducer_with_success_action`]. + CallReducerWithSuccessAction { reply_tx: JsReplyTx, params: CallReducerParams, - msg_id: u64, + action: crate::host::idc_actor::ReducerSuccessActionKind, }, /// See [`JsInstance::call_view`]. CallView { @@ -931,19 +931,19 @@ impl JsInstanceLane { .await }) .await - .map_err(|_| ReducerCallError::WorkerError(instance_lane_worker_error("call_reducer"))) + .map_err(|_| ReducerCallError::WorkerError(instance_lane_worker_error("call_reducer")))? } - pub async fn call_reducer_delete_outbound_on_success( + pub async fn call_reducer_with_success_action( &self, params: CallReducerParams, - msg_id: u64, + action: crate::host::idc_actor::ReducerSuccessActionKind, ) -> Result { self.run_once("call_reducer", |inst: JsInstance| async move { - inst.send_request(|reply_tx| JsWorkerRequest::CallReducerDeleteOutboundOnSuccess { + inst.send_request(|reply_tx| JsWorkerRequest::CallReducerWithSuccessAction { reply_tx, params, - msg_id, + action, }) .await }) @@ -1228,7 +1228,7 @@ async fn spawn_instance_worker( sender_identity, sender_msg_id, } => { - let (res, trapped) = crate::host::idc_actor::call_reducer_from_database( + let res = crate::host::idc_actor::call_reducer_from_database( info_for_idc.as_ref(), db_for_idc.as_ref(), params, @@ -1236,20 +1236,24 @@ async fn spawn_instance_worker( sender_msg_id, |tx, params, on_success| call_reducer_with_success(tx, params, on_success), ); + let (res, trapped) = match res { + Ok((rcr, trapped)) => (Ok(rcr), trapped), + Err(err) => (Err(err), false), + }; worker_trapped.store(trapped, Ordering::Relaxed); send_worker_reply("call_reducer", reply_tx, res, trapped); should_exit = trapped; } - JsWorkerRequest::CallReducerDeleteOutboundOnSuccess { + JsWorkerRequest::CallReducerWithSuccessAction { reply_tx, params, - msg_id, + action, } => { - let (res, trapped) = crate::host::idc_actor::call_reducer_delete_outbound_on_success( + let (res, trapped) = crate::host::idc_actor::call_reducer_with_success_action( info_for_idc.as_ref(), db_for_idc.as_ref(), params, - msg_id, + action, |tx, params, on_success| call_reducer_with_success(tx, params, on_success), ); worker_trapped.store(trapped, Ordering::Relaxed); diff --git a/crates/core/src/host/wasm_common/module_host_actor.rs b/crates/core/src/host/wasm_common/module_host_actor.rs index f946df44175..373468060c7 100644 --- a/crates/core/src/host/wasm_common/module_host_actor.rs +++ b/crates/core/src/host/wasm_common/module_host_actor.rs @@ -476,7 +476,7 @@ impl WasmModuleInstance { params: CallReducerParams, sender_identity: Identity, sender_msg_id: u64, - ) -> ReducerCallResult { + ) -> Result { let info = self.common.info.clone(); let db = self.instance.replica_ctx().relational_db().clone(); let (res, trapped) = crate::callgrind_flag::invoke_allowing_callgrind(|| { @@ -488,24 +488,24 @@ impl WasmModuleInstance { sender_msg_id, |tx, params, on_success| self.call_reducer_with_tx(tx, params, on_success), ) - }); + })?; self.trapped = trapped; - res + Ok(res) } - pub fn call_reducer_delete_outbound_on_success( + pub fn call_reducer_with_success_action( &mut self, params: CallReducerParams, - msg_id: u64, + action: crate::host::idc_actor::ReducerSuccessActionKind, ) -> ReducerCallResult { let info = self.common.info.clone(); let db = self.instance.replica_ctx().relational_db().clone(); let (res, trapped) = crate::callgrind_flag::invoke_allowing_callgrind(|| { - crate::host::idc_actor::call_reducer_delete_outbound_on_success( + crate::host::idc_actor::call_reducer_with_success_action( info.as_ref(), db.as_ref(), params, - msg_id, + action, |tx, params, on_success| self.call_reducer_with_tx(tx, params, on_success), ) }); diff --git a/crates/datastore/src/locking_tx_datastore/mut_tx.rs b/crates/datastore/src/locking_tx_datastore/mut_tx.rs index b3ef124ec64..ded51da5d6d 100644 --- a/crates/datastore/src/locking_tx_datastore/mut_tx.rs +++ b/crates/datastore/src/locking_tx_datastore/mut_tx.rs @@ -2630,13 +2630,12 @@ impl MutTxId { /// Update the last delivered msg_id for `sender_identity` in `st_inbound_msg`. /// /// If an entry already exists, it is replaced; otherwise a new entry is inserted. - /// `result_status` and `result_payload` store the outcome of the reducer call - /// (see [st_inbound_msg_result_status]). + /// `result_status` and `result_payload` store the outcome of the reducer call. pub fn upsert_inbound_last_msg( &mut self, sender_identity: Identity, last_outbound_msg: u64, - result_status: u8, + result_status: crate::system_tables::StInboundMsgResultStatus, result_payload: Bytes, ) -> Result<()> { // Delete the existing row if present. diff --git a/crates/datastore/src/system_tables.rs b/crates/datastore/src/system_tables.rs index ff071698307..26fd3830d89 100644 --- a/crates/datastore/src/system_tables.rs +++ b/crates/datastore/src/system_tables.rs @@ -1930,13 +1930,48 @@ impl From for ProductValue { /// System Table [ST_INBOUND_MSG_NAME] /// Tracks the last message id received from each sender database for deduplication. /// Also stores the result of the reducer call for returning on subsequent duplicate requests. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(u8)] +pub enum StInboundMsgResultStatus { + /// Reducer ran and committed successfully. + Success = 1, + /// Reducer ran and returned a user-visible error. + ReducerError = 2, +} + +impl TryFrom for StInboundMsgResultStatus { + type Error = &'static str; + + fn try_from(value: u8) -> Result { + match value { + 1 => Ok(Self::Success), + 2 => Ok(Self::ReducerError), + _ => Err("invalid st_inbound_msg result status"), + } + } +} + +impl From for u8 { + fn from(value: StInboundMsgResultStatus) -> Self { + value as u8 + } +} + +impl_st!([] StInboundMsgResultStatus, AlgebraicType::U8); +impl<'de> Deserialize<'de> for StInboundMsgResultStatus { + fn deserialize>(deserializer: D) -> Result { + let value = u8::deserialize(deserializer)?; + Self::try_from(value).map_err(D::Error::custom) + } +} +impl_serialize!([] StInboundMsgResultStatus, (self, ser) => u8::from(*self).serialize(ser)); + #[derive(Debug, Clone, PartialEq, Eq, SpacetimeType)] #[sats(crate = spacetimedb_lib)] pub struct StInboundMsgRow { pub database_identity: IdentityViaU256, pub last_outbound_msg: u64, - /// See [st_inbound_msg_result_status] for values. - pub result_status: u8, + pub result_status: StInboundMsgResultStatus, /// Reducer return payload encoded as raw bytes. /// For `SUCCESS`, this stores the committed reducer return value. /// For `REDUCER_ERROR`, this stores the error message bytes. @@ -1950,16 +1985,6 @@ impl TryFrom> for StInboundMsgRow { } } -/// Result status values stored in [ST_INBOUND_MSG_NAME] rows. -pub mod st_inbound_msg_result_status { - /// Reducer has been delivered but result not yet received. - pub const NOT_YET_RECEIVED: u8 = 0; - /// Reducer ran and returned Ok(()). - pub const SUCCESS: u8 = 1; - /// Reducer ran and returned Err(...). - pub const REDUCER_ERROR: u8 = 2; -} - /// System Table [ST_OUTBOUND_MSG_NAME] /// Tracks undelivered outbound inter-database messages (outbox pattern). /// A row's presence means the message has not yet been fully processed. From c1d7c9a708847fbccf228ed474f89c746b7c3c0e Mon Sep 17 00:00:00 2001 From: Shubham Mishra Date: Fri, 10 Apr 2026 22:23:19 +0530 Subject: [PATCH 17/23] renaming --- crates/client-api/src/routes/database.rs | 16 ++++--- crates/core/src/host/idc_actor.rs | 42 ++++++++++--------- crates/core/src/host/module_host.rs | 10 ----- .../src/host/wasm_common/module_host_actor.rs | 2 + 4 files changed, 34 insertions(+), 36 deletions(-) diff --git a/crates/client-api/src/routes/database.rs b/crates/client-api/src/routes/database.rs index 33140c8a302..c592d5f01a3 100644 --- a/crates/client-api/src/routes/database.rs +++ b/crates/client-api/src/routes/database.rs @@ -236,6 +236,7 @@ pub struct CallFromDatabaseQuery { msg_id: u64, } + /// Call a reducer on behalf of another database, with deduplication. /// /// Endpoint: `POST /database/:name_or_identity/call-from-database/:reducer` @@ -244,11 +245,12 @@ pub struct CallFromDatabaseQuery { /// - `sender_identity` — hex-encoded identity of the sending database. /// - `msg_id` — the inter-database message ID from the sender's st_outbound_msg. /// -/// Before invoking the reducer, the receiver checks `st_inbound_msg`. -/// If the incoming `msg_id` is the last delivered msg_id for `sender_identity`, -/// the call is a duplicate and 200 OK is returned immediately without running the reducer. -/// Otherwise the reducer is invoked, the dedup index is updated atomically in the same -/// transaction, and an acknowledgment is returned on success. +/// Semantics: +/// - The client **must send strictly increasing `msg_id` values per `sender_identity`.** +/// - If a `msg_id` is **less than the last seen msg_id** for that sender, the request +/// is treated as **invalid and rejected with a bad request error**. +/// - If the incoming `msg_id` is equal to the last delivered msg_id, the call is treated +/// as a duplicate and **200 OK is returned without invoking the reducer**. pub async fn call_from_database( State(worker_ctx): State, Extension(auth): Extension, @@ -295,6 +297,7 @@ pub async fn call_from_database( ) .await; + //Wait for durability before sending response if let Ok(rcr) = result.as_mut() && let Some(tx_offset) = rcr.tx_offset.as_mut() && let Some(mut durable_offset) = module.durable_tx_offset() @@ -316,13 +319,14 @@ pub async fn call_from_database( axum::body::Body::from(rcr.reducer_return_value.unwrap_or_default()), ), // 422 = reducer ran but returned Err; the IDC actor uses this to distinguish - // reducer failures from transport errors (which it retries). + // reducer failures from other errors (which it retries). ReducerOutcome::Failed(errmsg) => { ( StatusCode::UNPROCESSABLE_ENTITY, axum::body::Body::from(errmsg.to_string()), ) } + // This will be retried by IDC acttor ReducerOutcome::BudgetExceeded => { log::warn!( "Node's energy budget exceeded for identity: {owner_identity} while executing {reducer}" diff --git a/crates/core/src/host/idc_actor.rs b/crates/core/src/host/idc_actor.rs index 70ccab5b5a6..a30643f8f81 100644 --- a/crates/core/src/host/idc_actor.rs +++ b/crates/core/src/host/idc_actor.rs @@ -98,7 +98,7 @@ struct PendingMessage { } /// Per-target-database delivery state. -struct TargetState { +struct DatabaseQueue { queue: VecDeque, /// When `Some`, this target is in backoff and should not be retried until this instant. blocked_until: Option, @@ -106,7 +106,7 @@ struct TargetState { backoff: Duration, } -impl TargetState { +impl DatabaseQueue { fn new() -> Self { Self { queue: VecDeque::new(), @@ -141,7 +141,7 @@ enum DeliveryOutcome { ReducerError(String), /// Budget exceeded (HTTP 402). BudgetExceeded, - /// Transport error: network failure, unexpected HTTP status, etc. Caller should retry. + /// Transport error: database/reducer not found, network failure, unexpected HTTP status, etc. Caller should retry. TransportError(String), } @@ -155,21 +155,21 @@ async fn run_idc_loop( let client = reqwest::Client::new(); // Per-target-database delivery state. - let mut targets: HashMap = HashMap::new(); + let mut db_queues: HashMap = HashMap::new(); // On startup, load any pending messages that survived a restart. - load_pending_into_targets(&db, &mut targets); + load_pending_into_targets(&db, &mut db_queues); loop { // Deliver one message per ready target, then re-check. let mut any_delivered = true; while any_delivered { any_delivered = false; - for state in targets.values_mut() { - if !state.is_ready() { + for queue in db_queues.values_mut() { + if !queue.is_ready() { continue; } - let Some(msg) = state.queue.front().cloned() else { + let Some(msg) = queue.queue.front().cloned() else { continue; }; let outcome = attempt_delivery(&client, &config, &msg).await; @@ -180,12 +180,12 @@ async fn run_idc_loop( msg.msg_id, msg.target_db_identity.to_hex(), ); - state.record_transport_error(); + queue.record_transport_error(); // Do NOT pop the front — keep retrying this message for this target. } outcome => { - state.queue.pop_front(); - state.record_success(); + queue.queue.pop_front(); + queue.record_success(); any_delivered = true; let (result_status, result_payload) = outcome_to_result(&outcome); finalize_message(&db, &module_host, &msg, result_status, result_payload).await; @@ -195,7 +195,7 @@ async fn run_idc_loop( } // Compute how long to sleep: min over all blocked targets' unblock times. - let next_unblock = targets + let next_unblock = db_queues .values() .filter_map(|s| s.blocked_until) .min() @@ -204,6 +204,8 @@ async fn run_idc_loop( // Wait for a notification or the next retry time. tokio::select! { + //TODO:(shub) optimise this to send new entry directly instead of calling + //`load_pending_into_targets` _ = notify_rx.recv() => { // Drain all pending notifications (coalesce bursts). while notify_rx.try_recv().is_ok() {} @@ -211,8 +213,8 @@ async fn run_idc_loop( _ = tokio::time::sleep(sleep_duration) => {} } - // Reload pending messages from DB (catches anything missed and handles restart recovery). - load_pending_into_targets(&db, &mut targets); + // Reload pending messages from DB (catches new entries). + load_pending_into_targets(&db, &mut db_queues); } } @@ -247,7 +249,7 @@ fn reducer_workload(module: &ModuleInfo, params: &CallReducerParams) -> Workload }) } -fn duplicate_result_from_row(row: spacetimedb_datastore::system_tables::StInboundMsgRow) -> ReducerCallResult { +fn duplicate_result_from_st_inbound_row(row: spacetimedb_datastore::system_tables::StInboundMsgRow) -> ReducerCallResult { let outcome = match row.result_status { StInboundMsgResultStatus::Success => ReducerOutcome::Committed, StInboundMsgResultStatus::ReducerError => ReducerOutcome::Failed(Box::new( @@ -294,7 +296,7 @@ where if let Some(row) = tx.get_inbound_msg_row(sender_identity) { if sender_msg_id == row.last_outbound_msg { let _ = db.rollback_mut_tx(tx); - return Ok((duplicate_result_from_row(row), false)); + return Ok((duplicate_result_from_st_inbound_row(row), false)); } if sender_msg_id < row.last_outbound_msg { let expected = row.last_outbound_msg + 1; @@ -426,6 +428,8 @@ async fn finalize_message( return; } Err(e) => { + + delete_message(db, msg.msg_id); log::error!( "idc_actor: on_result reducer '{}' failed for msg_id={}: {e:?}", on_result_reducer, @@ -435,8 +439,6 @@ async fn finalize_message( } } - // Delete the row regardless of whether on_result succeeded or failed. - delete_message(db, msg.msg_id); } /// Load all messages from ST_OUTBOUND_MSG into the per-target queues, resolving delivery data @@ -444,7 +446,7 @@ async fn finalize_message( /// /// A row's presence in ST_OUTBOUND_MSG means it has not yet been processed. /// Messages already in a target's queue (by msg_id) are not re-added. -fn load_pending_into_targets(db: &RelationalDB, targets: &mut HashMap) { +fn load_pending_into_targets(db: &RelationalDB, db_queues: &mut HashMap) { let tx = db.begin_tx(Workload::Internal); let st_outbound_msg_rows: Vec = db @@ -557,7 +559,7 @@ fn load_pending_into_targets(db: &RelationalDB, targets: &mut HashMap (EventStatus::Committed(DatabaseUpdate::default()), return_value), From bd2c3ec79c3429cefe4aee6682ad26177903cf3b Mon Sep 17 00:00:00 2001 From: Shubham Mishra Date: Fri, 10 Apr 2026 22:24:03 +0530 Subject: [PATCH 18/23] fmt --- crates/client-api/src/routes/database.rs | 11 +++----- crates/core/src/host/idc_actor.rs | 26 +++++++++---------- crates/core/src/host/v8/mod.rs | 4 +-- .../src/host/wasm_common/module_host_actor.rs | 4 +-- 4 files changed, 20 insertions(+), 25 deletions(-) diff --git a/crates/client-api/src/routes/database.rs b/crates/client-api/src/routes/database.rs index c592d5f01a3..9746f92461d 100644 --- a/crates/client-api/src/routes/database.rs +++ b/crates/client-api/src/routes/database.rs @@ -236,7 +236,6 @@ pub struct CallFromDatabaseQuery { msg_id: u64, } - /// Call a reducer on behalf of another database, with deduplication. /// /// Endpoint: `POST /database/:name_or_identity/call-from-database/:reducer` @@ -320,12 +319,10 @@ pub async fn call_from_database( ), // 422 = reducer ran but returned Err; the IDC actor uses this to distinguish // reducer failures from other errors (which it retries). - ReducerOutcome::Failed(errmsg) => { - ( - StatusCode::UNPROCESSABLE_ENTITY, - axum::body::Body::from(errmsg.to_string()), - ) - } + ReducerOutcome::Failed(errmsg) => ( + StatusCode::UNPROCESSABLE_ENTITY, + axum::body::Body::from(errmsg.to_string()), + ), // This will be retried by IDC acttor ReducerOutcome::BudgetExceeded => { log::warn!( diff --git a/crates/core/src/host/idc_actor.rs b/crates/core/src/host/idc_actor.rs index a30643f8f81..81426883ea9 100644 --- a/crates/core/src/host/idc_actor.rs +++ b/crates/core/src/host/idc_actor.rs @@ -18,9 +18,7 @@ use anyhow::anyhow; use bytes::Bytes; use spacetimedb_datastore::execution_context::{ReducerContext, Workload}; use spacetimedb_datastore::locking_tx_datastore::MutTxId; -use spacetimedb_datastore::system_tables::{ - StInboundMsgResultStatus, StOutboundMsgRow, ST_OUTBOUND_MSG_ID, -}; +use spacetimedb_datastore::system_tables::{StInboundMsgResultStatus, StOutboundMsgRow, ST_OUTBOUND_MSG_ID}; use spacetimedb_datastore::traits::IsolationLevel; use spacetimedb_lib::{AlgebraicValue, Identity, ProductValue}; use spacetimedb_primitives::{ColId, TableId}; @@ -249,11 +247,15 @@ fn reducer_workload(module: &ModuleInfo, params: &CallReducerParams) -> Workload }) } -fn duplicate_result_from_st_inbound_row(row: spacetimedb_datastore::system_tables::StInboundMsgRow) -> ReducerCallResult { +fn duplicate_result_from_st_inbound_row( + row: spacetimedb_datastore::system_tables::StInboundMsgRow, +) -> ReducerCallResult { let outcome = match row.result_status { StInboundMsgResultStatus::Success => ReducerOutcome::Committed, StInboundMsgResultStatus::ReducerError => ReducerOutcome::Failed(Box::new( - String::from_utf8_lossy(&row.result_payload).into_owned().into_boxed_str(), + String::from_utf8_lossy(&row.result_payload) + .into_owned() + .into_boxed_str(), )), }; @@ -344,11 +346,9 @@ where call_reducer( Some(tx), params, - Box::new(move |tx, _reducer_return_value| { - match action { - ReducerSuccessActionKind::DeleteOutboundMsg(msg_id) => { - tx.delete_outbound_msg(msg_id).map_err(|e| anyhow!(e)) - } + Box::new(move |tx, _reducer_return_value| match action { + ReducerSuccessActionKind::DeleteOutboundMsg(msg_id) => { + tx.delete_outbound_msg(msg_id).map_err(|e| anyhow!(e)) } }), ) @@ -428,7 +428,6 @@ async fn finalize_message( return; } Err(e) => { - delete_message(db, msg.msg_id); log::error!( "idc_actor: on_result reducer '{}' failed for msg_id={}: {e:?}", @@ -438,7 +437,6 @@ async fn finalize_message( } } } - } /// Load all messages from ST_OUTBOUND_MSG into the per-target queues, resolving delivery data @@ -559,7 +557,9 @@ fn load_pending_into_targets(db: &RelationalDB, db_queues: &mut HashMap WasmModuleInstance { res } - pub fn clear_all_clients(&self) -> anyhow::Result<()> { self.common.clear_all_clients() } @@ -597,7 +596,8 @@ impl WasmModuleInstance { F: FnOnce(&mut MutTxId, &Option) -> anyhow::Result<()>, { crate::callgrind_flag::invoke_allowing_callgrind(|| { - self.common.call_reducer_with_tx(tx, params, &mut self.instance, on_success) + self.common + .call_reducer_with_tx(tx, params, &mut self.instance, on_success) }) } From 584d13b88b4a42ea7f6b5a8f6a6b764e40b0295a Mon Sep 17 00:00:00 2001 From: Shubham Mishra Date: Fri, 10 Apr 2026 22:30:06 +0530 Subject: [PATCH 19/23] clippy --- crates/bindings/tests/ui/outbox.rs | 9 +++++++++ crates/bindings/tests/ui/outbox.stderr | 5 +++++ crates/core/src/host/idc_actor.rs | 1 - 3 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 crates/bindings/tests/ui/outbox.rs create mode 100644 crates/bindings/tests/ui/outbox.stderr diff --git a/crates/bindings/tests/ui/outbox.rs b/crates/bindings/tests/ui/outbox.rs new file mode 100644 index 00000000000..d7a0c78edd6 --- /dev/null +++ b/crates/bindings/tests/ui/outbox.rs @@ -0,0 +1,9 @@ +#[spacetimedb::table(accessor = messages, outbox(send_message))] +struct Message { + #[primary_key] + #[auto_inc] + row_id: u64, + body: String, +} + +fn main() {} diff --git a/crates/bindings/tests/ui/outbox.stderr b/crates/bindings/tests/ui/outbox.stderr new file mode 100644 index 00000000000..11813a08976 --- /dev/null +++ b/crates/bindings/tests/ui/outbox.stderr @@ -0,0 +1,5 @@ +error: outbox requires the `spacetimedb` `stable-idc` feature + --> tests/ui/outbox.rs:1:43 + | +1 | #[spacetimedb::table(accessor = messages, outbox(send_message))] + | ^^^^^^ diff --git a/crates/core/src/host/idc_actor.rs b/crates/core/src/host/idc_actor.rs index 81426883ea9..7160ec24a6e 100644 --- a/crates/core/src/host/idc_actor.rs +++ b/crates/core/src/host/idc_actor.rs @@ -425,7 +425,6 @@ async fn finalize_message( on_result_reducer, msg.msg_id, ); - return; } Err(e) => { delete_message(db, msg.msg_id); From 246d01af7a82ba55a4ff49f185f68b0494560365 Mon Sep 17 00:00:00 2001 From: Shubham Mishra Date: Fri, 10 Apr 2026 23:51:13 +0530 Subject: [PATCH 20/23] fix merge conflict --- crates/core/src/host/v8/mod.rs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/crates/core/src/host/v8/mod.rs b/crates/core/src/host/v8/mod.rs index 5cc1a0d4ca3..f11cb039a1a 100644 --- a/crates/core/src/host/v8/mod.rs +++ b/crates/core/src/host/v8/mod.rs @@ -1519,8 +1519,11 @@ async fn spawn_instance_worker( Ok((rcr, trapped)) => (Ok(rcr), trapped), Err(err) => (Err(err), false), }; - worker_trapped.store(trapped, Ordering::Relaxed); - send_worker_reply("call_reducer", reply_tx, res, trapped); + if trapped { + worker_state_in_thread.mark_trapped(); + } + + send_worker_reply("call_reducer", reply_tx, res); should_exit = trapped; } JsWorkerRequest::CallReducerWithSuccessAction { @@ -1535,8 +1538,11 @@ async fn spawn_instance_worker( action, |tx, params, on_success| call_reducer_with_success(tx, params, on_success), ); - worker_trapped.store(trapped, Ordering::Relaxed); - send_worker_reply("call_reducer", reply_tx, res, trapped); + if trapped { + worker_state_in_thread.mark_trapped(); + } + + send_worker_reply("call_reducer", reply_tx, res); should_exit = trapped; } JsWorkerRequest::CallView { reply_tx, cmd } => { From 9849ca852bced9c4307c5e7f5d733ff0df2def0e Mon Sep 17 00:00:00 2001 From: Shubham Mishra Date: Fri, 10 Apr 2026 23:54:14 +0530 Subject: [PATCH 21/23] deleted redundtant ui test --- crates/bindings/tests/ui/outbox.rs | 9 --------- crates/bindings/tests/ui/outbox.stderr | 5 ----- 2 files changed, 14 deletions(-) delete mode 100644 crates/bindings/tests/ui/outbox.rs delete mode 100644 crates/bindings/tests/ui/outbox.stderr diff --git a/crates/bindings/tests/ui/outbox.rs b/crates/bindings/tests/ui/outbox.rs deleted file mode 100644 index d7a0c78edd6..00000000000 --- a/crates/bindings/tests/ui/outbox.rs +++ /dev/null @@ -1,9 +0,0 @@ -#[spacetimedb::table(accessor = messages, outbox(send_message))] -struct Message { - #[primary_key] - #[auto_inc] - row_id: u64, - body: String, -} - -fn main() {} diff --git a/crates/bindings/tests/ui/outbox.stderr b/crates/bindings/tests/ui/outbox.stderr deleted file mode 100644 index 11813a08976..00000000000 --- a/crates/bindings/tests/ui/outbox.stderr +++ /dev/null @@ -1,5 +0,0 @@ -error: outbox requires the `spacetimedb` `stable-idc` feature - --> tests/ui/outbox.rs:1:43 - | -1 | #[spacetimedb::table(accessor = messages, outbox(send_message))] - | ^^^^^^ From f034f116ec6f96f6ea72f805ae3afb06343e036d Mon Sep 17 00:00:00 2001 From: Shubham Mishra Date: Sat, 11 Apr 2026 07:40:48 +0530 Subject: [PATCH 22/23] h2 client --- crates/core/src/host/idc_actor.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/crates/core/src/host/idc_actor.rs b/crates/core/src/host/idc_actor.rs index 7160ec24a6e..bc287b707ea 100644 --- a/crates/core/src/host/idc_actor.rs +++ b/crates/core/src/host/idc_actor.rs @@ -150,7 +150,19 @@ async fn run_idc_loop( module_host: WeakModuleHost, mut notify_rx: mpsc::UnboundedReceiver<()>, ) { - let client = reqwest::Client::new(); + let client = reqwest::Client::builder() + .http2_prior_knowledge() + .http2_keep_alive_interval(Some(Duration::from_millis(200))) + .http2_keep_alive_timeout(Duration::from_secs(5)) + .http2_keep_alive_while_idle(true) + // Both stram and connection window sizes are 64KB by default, + // increasing the connection window to handle single stream blocking the entire connection. + .http2_initial_connection_window_size(Some(2 * 64 * 1024)) + .http2_initial_stream_window_size(Some(64 * 1024)) + .timeout(Duration::from_secs(5)) + .build() + .unwrap(); + // Per-target-database delivery state. let mut db_queues: HashMap = HashMap::new(); From 1efb1208313b8577cb315cd0793cc029eca0ffef Mon Sep 17 00:00:00 2001 From: Shubham Mishra Date: Sat, 11 Apr 2026 08:23:34 +0530 Subject: [PATCH 23/23] comment --- crates/client-api/src/routes/database.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/client-api/src/routes/database.rs b/crates/client-api/src/routes/database.rs index 9746f92461d..60a074fc249 100644 --- a/crates/client-api/src/routes/database.rs +++ b/crates/client-api/src/routes/database.rs @@ -319,6 +319,8 @@ pub async fn call_from_database( ), // 422 = reducer ran but returned Err; the IDC actor uses this to distinguish // reducer failures from other errors (which it retries). + // This is inconsistent with `call` endpoint, which returns 523 status code if + // reducer fails ReducerOutcome::Failed(errmsg) => ( StatusCode::UNPROCESSABLE_ENTITY, axum::body::Body::from(errmsg.to_string()),