fix(db): bound runtime Postgres statements and lock waits

.env.example

@@ -38,6 +38,12 @@ REDIS_URL=redis://localhost:6379
 # READ_DATABASE_URL is set, reader (default 50).
 # BUZZ_DB_POOL_SIZE=50
 
+# Postgres statement_timeout and lock_timeout applied to every runtime
+# connection. Accepts an integer (milliseconds) with an optional us/ms/s/min/h/d
+# unit; `0` disables the limit. Schema migrations always run with both lifted.
+# BUZZ_DB_STATEMENT_TIMEOUT=30s
+# BUZZ_DB_LOCK_TIMEOUT=5s
+
 # -----------------------------------------------------------------------------
 # Typesense (search)
 # -----------------------------------------------------------------------------

crates/buzz-db/src/lib.rs

@@ -65,6 +65,32 @@ use uuid::Uuid;
 
 use buzz_core::{CommunityId, StoredEvent};
 
+/// Default maximum time a runtime query may execute before Postgres cancels it.
+pub const RUNTIME_STATEMENT_TIMEOUT: &str = "30s";
+/// Default maximum time a runtime query may wait to acquire a lock.
+pub const RUNTIME_LOCK_TIMEOUT: &str = "5s";
+/// Postgres spelling of "no limit", used for schema migrations.
+pub const TIMEOUT_DISABLED: &str = "0";
+
+/// Apply the runtime safety limits shared by writer, reader, audit, and search
+/// pools. Values are Postgres interval strings (`"30s"`, `"500ms"`), with
+/// [`TIMEOUT_DISABLED`] lifting a limit entirely.
+pub async fn apply_runtime_connection_timeouts(
+    connection: &mut PgConnection,
+    statement_timeout: &str,
+    lock_timeout: &str,
+) -> std::result::Result<(), sqlx::Error> {
+    sqlx::query(
+        "SELECT set_config('statement_timeout', $1, false), \
+                set_config('lock_timeout', $2, false)",
+    )
+    .bind(statement_timeout)
+    .bind(lock_timeout)
+    .execute(connection)
+    .await?;
+    Ok(())
+}
+
 fn event_replacement_lock_key(
     community_id: CommunityId,
     kind: i32,
@@ -527,6 +553,14 @@ pub struct DbConfig {
     /// than the staleness gate never routes anyway, so a larger budget
     /// would only misrepresent the config.
     pub replica_read_max_age_ms: u64,
+    /// Postgres `statement_timeout` applied to every runtime connection. An
+    /// operator running a backfill or working an incident can widen this without
+    /// a code change; [`TIMEOUT_DISABLED`] removes the cap.
+    pub statement_timeout: String,
+    /// Postgres `lock_timeout` applied to every runtime connection. Bounds
+    /// heavyweight and row lock waits only — advisory-lock waits are bounded by
+    /// [`Self::statement_timeout`] instead.
+    pub lock_timeout: String,
 }
 
 impl Default for DbConfig {
@@ -544,6 +578,8 @@ impl Default for DbConfig {
             max_lifetime_secs: 1800,
             idle_timeout_secs: 600,
             replica_read_max_age_ms: 0,
+            statement_timeout: RUNTIME_STATEMENT_TIMEOUT.to_string(),
+            lock_timeout: RUNTIME_LOCK_TIMEOUT.to_string(),
         }
     }
 }
@@ -682,18 +718,23 @@ impl Db {
             .acquire_timeout(Duration::from_secs(config.acquire_timeout_secs))
             .max_lifetime(Duration::from_secs(config.max_lifetime_secs))
             .idle_timeout(Duration::from_secs(config.idle_timeout_secs));
-        if arm_floor_guard {
-            options = options.after_connect(|conn, _meta| {
-                Box::pin(async move {
+        let statement_timeout = config.statement_timeout.clone();
+        let lock_timeout = config.lock_timeout.clone();
+        options = options.after_connect(move |conn, _meta| {
+            let statement_timeout = statement_timeout.clone();
+            let lock_timeout = lock_timeout.clone();
+            Box::pin(async move {
+                apply_runtime_connection_timeouts(conn, &statement_timeout, &lock_timeout).await?;
+                if arm_floor_guard {
                     // `SET` cannot take bind parameters; `set_config` can.
                     sqlx::query("SELECT set_config('buzz.created_at_floor', $1, false)")
                         .bind(replica_fence::CREATED_AT_FLOOR_SECS.to_string())
                         .execute(conn)
                         .await?;
-                    Ok(())
-                })
-            });
-        }
+                }
+                Ok(())
+            })
+        });
         Ok(options.connect(url).await?)
     }
 
@@ -721,12 +762,22 @@ impl Db {
     /// No floor guard: replica sessions are read-only, the trigger never
     /// fires there (see [`Db::connect_pool`]).
     fn connect_read_pool(config: &DbConfig, url: &str, max_connections: u32) -> Result<PgPool> {
+        let statement_timeout = config.statement_timeout.clone();
+        let lock_timeout = config.lock_timeout.clone();
         Ok(PgPoolOptions::new()
             .max_connections(max_connections)
             .min_connections(0)
             .acquire_timeout(Self::READER_ACQUIRE_TIMEOUT)
             .max_lifetime(Duration::from_secs(config.max_lifetime_secs))
             .idle_timeout(Duration::from_secs(config.idle_timeout_secs))
+            .after_connect(move |connection, _meta| {
+                let statement_timeout = statement_timeout.clone();
+                let lock_timeout = lock_timeout.clone();
+                Box::pin(async move {
+                    apply_runtime_connection_timeouts(connection, &statement_timeout, &lock_timeout)
+                        .await
+                })
+            })
             .connect_lazy(url)?)
     }
 
@@ -6511,6 +6562,65 @@ mod tests {
         .await;
     }
 
+    /// Migrations must outlive the runtime caps — an index build or an
+    /// `ACCESS EXCLUSIVE` wait routinely exceeds them, and startup treats a
+    /// migration failure as fatal — and the relaxed session must not survive
+    /// into the pool afterwards.
+    #[tokio::test]
+    #[ignore = "requires Postgres"]
+    async fn migrations_ignore_runtime_timeouts_and_leak_no_relaxed_session() {
+        const TIGHT: &str = "50ms";
+
+        let admin = PgPool::connect(&admin_url().await)
+            .await
+            .expect("connect admin");
+        let name = format!("migration_timeouts_{}", Uuid::new_v4().simple());
+        sqlx::query(sqlx::AssertSqlSafe(format!("CREATE DATABASE {name}")))
+            .execute(&admin)
+            .await
+            .expect("create scratch db");
+        let base = admin_url().await;
+        let idx = base.rfind('/').expect("db url has a path segment");
+        let scratch_url = format!("{}/{}", &base[..idx], name);
+
+        // Far shorter than the migration suite needs, ample for a pooled query.
+        let db = Db::new(&DbConfig {
+            database_url: scratch_url,
+            max_connections: 2,
+            min_connections: 2,
+            statement_timeout: TIGHT.to_string(),
+            lock_timeout: TIGHT.to_string(),
+            ..DbConfig::default()
+        })
+        .await
+        .expect("connect Db against the unmigrated scratch db");
+
+        db.migrate()
+            .await
+            .expect("migrations must not inherit the runtime caps");
+
+        // Hold every connection at once so a leaked relaxed session cannot hide
+        // behind a freshly dialed one.
+        let mut held = Vec::new();
+        for _ in 0..2 {
+            let mut connection = db.pool.acquire().await.expect("acquire pooled connection");
+            let statement_timeout: String = sqlx::query_scalar("SHOW statement_timeout")
+                .fetch_one(&mut *connection)
+                .await
+                .expect("SHOW statement_timeout");
+            let lock_timeout: String = sqlx::query_scalar("SHOW lock_timeout")
+                .fetch_one(&mut *connection)
+                .await
+                .expect("SHOW lock_timeout");
+            assert_eq!(statement_timeout, TIGHT);
+            assert_eq!(lock_timeout, TIGHT);
+            held.push(connection);
+        }
+        drop(held);
+
+        drop_scratch_db(&admin, db.pool.clone(), &name).await;
+    }
+
     /// Insert identical community + channel rows into a database so the same
     /// (community, channel) ids resolve in both writer and replica.
     async fn seed_community_channel(
@@ -8317,15 +8427,39 @@ mod tests {
         let idx = base.rfind('/').expect("db url has a path segment");
         let scratch_url = format!("{}/{}", &base[..idx], name);
         let db = Db::new(&DbConfig {
-            database_url: scratch_url,
+            database_url: scratch_url.clone(),
+            read_database_url: Some(scratch_url),
             max_connections: 2,
             ..DbConfig::default()
         })
         .await
         .expect("connect armed Db");
         let cid = CommunityId::from_uuid(community);
 
-        // Perci nit: assert the effective session value, not the intent.
+        // Assert the effective session values, not only pool-builder intent.
+        let statement_timeout: String = sqlx::query_scalar("SHOW statement_timeout")
+            .fetch_one(&db.pool)
+            .await
+            .expect("SHOW statement_timeout");
+        let lock_timeout: String = sqlx::query_scalar("SHOW lock_timeout")
+            .fetch_one(&db.pool)
+            .await
+            .expect("SHOW lock_timeout");
+        assert_eq!(statement_timeout, RUNTIME_STATEMENT_TIMEOUT);
+        assert_eq!(lock_timeout, RUNTIME_LOCK_TIMEOUT);
+
+        let read_pool = db.read_pool.as_ref().expect("read pool configured");
+        let reader_statement_timeout: String = sqlx::query_scalar("SHOW statement_timeout")
+            .fetch_one(read_pool)
+            .await
+            .expect("SHOW reader statement_timeout");
+        let reader_lock_timeout: String = sqlx::query_scalar("SHOW lock_timeout")
+            .fetch_one(read_pool)
+            .await
+            .expect("SHOW reader lock_timeout");
+        assert_eq!(reader_statement_timeout, RUNTIME_STATEMENT_TIMEOUT);
+        assert_eq!(reader_lock_timeout, RUNTIME_LOCK_TIMEOUT);
+
         let effective: String = sqlx::query_scalar("SHOW buzz.created_at_floor")
             .fetch_one(&db.pool)
             .await

crates/buzz-db/src/migration.rs

@@ -4,16 +4,27 @@
 //! multi-tenant rewrite owns a clean consolidated `0001`; legacy single-tenant
 //! cutover/backfill is a separate operator script, not startup migration state.
 
-use sqlx::PgPool;
+use sqlx::{Connection, PgPool};
 
 use crate::Result;
 
 static MIGRATOR: sqlx::migrate::Migrator = sqlx::migrate!("../../migrations");
 
 /// Run all pending Buzz database migrations.
+///
+/// DDL runs with the runtime `statement_timeout` and `lock_timeout` lifted. An
+/// index build on a populated table, or an `ACCESS EXCLUSIVE` wait behind live
+/// traffic, routinely outlasts the runtime caps — and because startup treats a
+/// migration failure as fatal, inheriting them would turn a slow migration into
+/// a relay that cannot boot. sqlx also takes its migration advisory lock as a
+/// single waiting statement, so a second replica rolling out would be canceled
+/// mid-wait rather than queueing behind the first.
+///
+/// The connection is closed instead of returned to the pool: its session still
+/// carries the lifted limits and must never serve traffic.
 pub async fn run_migrations(pool: &PgPool) -> Result<()> {
     reject_legacy_nip_rs_cardinality_ambiguity(pool).await?;
-    MIGRATOR.run(pool).await?;
+    run_migrator_without_runtime_timeouts(pool).await?;
     // The replica-fence proof (see `replica_fence`) requires the commit-time
     // `created_at` floor trigger from migration 0021 — correctly shaped — on
     // the `events` parent and every partition. `CREATE TABLE .. PARTITION OF`
@@ -25,6 +36,36 @@ pub async fn run_migrations(pool: &PgPool) -> Result<()> {
     Ok(())
 }
 
+async fn run_migrator_without_runtime_timeouts(pool: &PgPool) -> Result<()> {
+    let mut connection = pool.acquire().await?;
+    lift_runtime_timeouts(&mut connection).await?;
+    let migrated = MIGRATOR.run(&mut *connection).await;
+    // Retire the connection either way; report the migration outcome first so a
+    // close failure cannot mask it.
+    let retired = retire_connection(connection).await;
+    migrated?;
+    retired?;
+    Ok(())
+}
+
+/// Remove both runtime limits from one connection's session.
+async fn lift_runtime_timeouts(connection: &mut sqlx::PgConnection) -> Result<()> {
+    crate::apply_runtime_connection_timeouts(
+        connection,
+        crate::TIMEOUT_DISABLED,
+        crate::TIMEOUT_DISABLED,
+    )
+    .await?;
+    Ok(())
+}
+
+/// Close a connection instead of returning it to the pool, so a session that
+/// carries lifted limits can never serve traffic.
+async fn retire_connection(connection: sqlx::pool::PoolConnection<sqlx::Postgres>) -> Result<()> {
+    connection.detach().close().await?;
+    Ok(())
+}
+
 /// Migration 0007 is checksum-frozen and predates exact NIP-RS tag-cardinality
 /// enforcement. A populated database still on 0001-0006 must not let 0007
 /// irreversibly purge duplicate-tag history. Fail before sqlx starts its
@@ -1094,6 +1135,69 @@ mod tests {
         .expect("read applied migrations")
     }
 
+    /// The migration connection must have both limits lifted, and it must not
+    /// come back to the pool afterwards — a session with no statement timeout
+    /// serving traffic is the failure this exemption trades against.
+    #[tokio::test]
+    #[ignore = "requires Postgres"]
+    async fn migration_connection_is_unbounded_and_is_retired_not_reused() {
+        const TIGHT: &str = "50ms";
+
+        let database_url = std::env::var("BUZZ_TEST_DATABASE_URL")
+            .or_else(|_| std::env::var("DATABASE_URL"))
+            .unwrap_or_else(|_| TEST_DB_URL.to_owned());
+        // One slot: a reused connection would be handed straight back below.
+        let pool = sqlx::postgres::PgPoolOptions::new()
+            .max_connections(1)
+            .after_connect(|connection, _meta| {
+                Box::pin(crate::apply_runtime_connection_timeouts(
+                    connection, TIGHT, TIGHT,
+                ))
+            })
+            .connect(&database_url)
+            .await
+            .expect("connect to test DB");
+
+        let mut connection = pool.acquire().await.expect("acquire");
+        assert_eq!(
+            show_timeout(&mut connection, "statement_timeout").await,
+            TIGHT
+        );
+
+        lift_runtime_timeouts(&mut connection)
+            .await
+            .expect("lift runtime timeouts");
+        for setting in ["statement_timeout", "lock_timeout"] {
+            assert_eq!(
+                show_timeout(&mut connection, setting).await,
+                "0",
+                "{setting} must be lifted for the migrator"
+            );
+        }
+
+        retire_connection(connection)
+            .await
+            .expect("retire migration connection");
+
+        let mut fresh = pool.acquire().await.expect("re-acquire");
+        for setting in ["statement_timeout", "lock_timeout"] {
+            assert_eq!(
+                show_timeout(&mut fresh, setting).await,
+                TIGHT,
+                "the pool must not hand out the relaxed migration session"
+            );
+        }
+        drop(fresh);
+        pool.close().await;
+    }
+
+    async fn show_timeout(connection: &mut sqlx::PgConnection, setting: &str) -> String {
+        sqlx::query_scalar(sqlx::AssertSqlSafe(format!("SHOW {setting}")))
+            .fetch_one(connection)
+            .await
+            .unwrap_or_else(|error| panic!("SHOW {setting}: {error}"))
+    }
+
     #[tokio::test]
     #[ignore = "requires Postgres"]
     async fn pre_0007_ambiguous_nip_rs_data_blocks_without_mutation_and_allows_retry() {