From 65ba1374af2526f8fe9d950d1a946056fc7bdb9d Mon Sep 17 00:00:00 2001 From: t Date: Thu, 9 Jul 2026 21:43:38 -0500 Subject: [PATCH 01/58] feat(node,git): cap concurrent served git ops with a 503 load-shed (#62) PR3 of the #62 served-git hardening stack (timeout #165 and teardown wiring #150 are merged). A bounded semaphore caps how many upload-pack / receive-pack / info-refs operations run at once; past the cap a request is shed with a clean 503 + Retry-After before spawning another git subprocess, instead of exhausting the PID/thread table. A permit is acquired at the top of each of the three handlers and held for the whole op, releasing on return. The cap is a portable backstop: the compose pids_limit is absent on Fly, whose 500-connection cap is a different axis. Size --max-concurrent-git-ops (GITLAWB_MAX_CONCURRENT_GIT_OPS, default 128) below the process budget. Range 1..=1_048_576 so 0 (shed everything) and an oversized value that would panic tokio's Semaphore at boot are clean CLI errors. Known gap, tracked separately: info/refs and the withheld-blob (upload_pack_excluding) path are not duration-bounded and do not reap their git child on client disconnect, so a hung git on those two paths holds its slot until it exits and live git can briefly exceed the cap. The main pack path (run_git_service) tears its group down on drop. Tests: Overloaded maps to 503 + Retry-After; the config knob defaults and rejects out-of-range; git_permit sheds at capacity and releases; and each of the three endpoints sheds with 503 when the semaphore is exhausted (load-bearing: drop the permit line and the endpoint test goes red). --- crates/gitlawb-node/src/api/repos.rs | 37 ++++++++ crates/gitlawb-node/src/auth/mod.rs | 1 + crates/gitlawb-node/src/config.rs | 61 ++++++++++++ crates/gitlawb-node/src/error.rs | 31 +++++- crates/gitlawb-node/src/main.rs | 1 + crates/gitlawb-node/src/state.rs | 4 + crates/gitlawb-node/src/test_support.rs | 119 ++++++++++++++++++++++++ 7 files changed, 253 insertions(+), 1 deletion(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index b9cbc352..4bc1f737 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -507,6 +507,9 @@ pub async fn git_info_refs( headers: axum::http::HeaderMap, auth: Option>, ) -> Result { + // Shed with a 503 before spawning git when the concurrency cap is saturated; + // held for the whole op (incl. the smart_http call), released on return. + let _permit = git_permit(&state.git_semaphore)?; let name = smart_http_repo_name(&repo)?; tracing::info!(owner = %owner, repo = %name, "info/refs request"); let record = state @@ -589,6 +592,22 @@ pub async fn git_info_refs( }) } +/// Acquire a permit from the served-git concurrency semaphore, or shed the +/// request with a 503 + Retry-After when every slot is in use. Bind the returned +/// permit to a named local so it is held for the whole git op (it releases on +/// drop); a bare `_` would release it immediately. +fn git_permit( + sem: &std::sync::Arc, +) -> Result { + sem.clone().try_acquire_owned().map_err(|_| { + // Surface the shed so operators can see the cap engaging, mirroring the + // receive-pack rate-limit warn above. A silent 503 makes a saturated or + // misconfigured cap look like a client problem instead of a capacity one. + tracing::warn!("served-git concurrency cap reached; shedding request with 503"); + AppError::Overloaded("git service at capacity, retry shortly".into()) + }) +} + /// Map an error from a `smart_http` git service call to the right `AppError`: /// [`smart_http::GitServiceTimeout`] to 504, a malformed client request to 400, /// anything else to a 500 git error. Pure (no logging) so it is unit-testable; @@ -616,6 +635,9 @@ pub async fn git_upload_pack( auth: Option>, body: Bytes, ) -> Result { + // Shed with a 503 before spawning git when the concurrency cap is saturated; + // held for the whole op (incl. the smart_http call), released on return. + let _permit = git_permit(&state.git_semaphore)?; let name = smart_http_repo_name(&repo)?; let record = state .db @@ -853,6 +875,10 @@ pub async fn git_receive_pack( Extension(auth): Extension, body: Bytes, ) -> Result { + // Shed with a 503 before spawning git when the concurrency cap is saturated. + // Acquired at the very top so it wraps the write-guard below; held for the + // whole op (incl. the smart_http call), released on return. + let _permit = git_permit(&state.git_semaphore)?; let name = smart_http_repo_name(&repo)?; tracing::info!(owner = %owner, repo = %name, "receive-pack request"); let record = state @@ -1862,6 +1888,17 @@ mod tests { assert!(matches!(git_service_app_error(&other), AppError::Git(_))); } + #[test] + fn git_permit_sheds_at_capacity_and_releases() { + let sem = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + let p1 = git_permit(&sem).expect("first acquire succeeds"); + // At capacity the next request is shed with Overloaded (-> 503), not queued. + assert!(matches!(git_permit(&sem), Err(AppError::Overloaded(_)))); + // Releasing the permit frees the slot for the next request. + drop(p1); + assert!(git_permit(&sem).is_ok()); + } + fn repo_owned_by(owner_did: &str) -> crate::db::RepoRecord { let now = chrono::Utc::now(); crate::db::RepoRecord { diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index 720fb3ae..b541b57b 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -520,6 +520,7 @@ mod tests { sync_trigger_rate_limiter: RateLimiter::new(60, Duration::from_secs(3600)), peer_write_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), shutdown_tx: tokio::sync::watch::channel(false).0, + git_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), } } diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index fc2247d9..688e99f4 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -234,6 +234,40 @@ pub struct Config { value_parser = clap::value_parser!(u64).range(1..) )] pub db_retry_max_secs: u64, + + /// Maximum number of served git operations (upload-pack / receive-pack / + /// info-refs) allowed to run concurrently. Beyond this the node sheds the + /// request with a clean 503 + Retry-After instead of spawning another git + /// subprocess and risking PID/thread exhaustion. Portable backstop: the + /// compose `pids_limit` is not present on Fly, whose connection-concurrency + /// cap is a different axis (500 connections each fan out to git + + /// pack-objects + threads). Size below the process budget with headroom. + /// + /// A permit is held for the whole op. upload-pack/receive-pack are + /// duration-bounded by `git_service_timeout_secs`, but the `info/refs` + /// advertisement and the withheld-blob (`upload_pack_excluding`) path are + /// not, so a hung git on those two paths holds its slot until it exits, so a + /// stuck advertisement permanently costs one unit of capacity. Bounding + /// those paths' duration is tracked as separate follow-up. + /// + /// The same two paths have the mirror gap on cancellation: the permit frees + /// when the handler future drops (client disconnect), but neither reaps its + /// git child — `info/refs` runs a bare `Command`, and `upload_pack_excluding` + /// a blocking `spawn_blocking` a dropped future cannot cancel — so the child + /// runs to completion after its slot is freed and live git can briefly exceed + /// the cap. The main pack path (`run_git_service`) tears its process group + /// down on drop, so this is confined to the same two follow-up paths. + /// + /// Default: 128. Must be between 1 and 1_048_576; the ceiling keeps the value + /// well under tokio's `Semaphore` permit limit so an oversized value is a + /// clean CLI error rather than a boot-time panic. + #[arg( + long, + env = "GITLAWB_MAX_CONCURRENT_GIT_OPS", + default_value_t = 128, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub max_concurrent_git_ops: usize, } impl Config { @@ -272,4 +306,31 @@ mod tests { Config::try_parse_from(["gitlawb-node", "--git-service-timeout-secs", "0"]).is_err() ); } + + #[test] + fn max_concurrent_git_ops_defaults_and_rejects_out_of_range() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).max_concurrent_git_ops, + 128 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--max-concurrent-git-ops", "8"]) + .max_concurrent_git_ops, + 8 + ); + // 0 permits would shed every served-git request with a 503; clap must reject it. + assert!(Config::try_parse_from(["gitlawb-node", "--max-concurrent-git-ops", "0"]).is_err()); + // Above the ceiling would panic tokio's Semaphore::new at boot (permits > + // usize::MAX >> 3); clap must reject it as a clean CLI error instead. + assert!( + Config::try_parse_from(["gitlawb-node", "--max-concurrent-git-ops", "1048577"]) + .is_err() + ); + // The ceiling itself is accepted. + assert_eq!( + Config::parse_from(["gitlawb-node", "--max-concurrent-git-ops", "1048576"]) + .max_concurrent_git_ops, + 1_048_576 + ); + } } diff --git a/crates/gitlawb-node/src/error.rs b/crates/gitlawb-node/src/error.rs index bdd5aec9..a07235fa 100644 --- a/crates/gitlawb-node/src/error.rs +++ b/crates/gitlawb-node/src/error.rs @@ -47,6 +47,9 @@ pub enum AppError { #[error("git service timed out: {0}")] Timeout(String), + #[error("server overloaded: {0}")] + Overloaded(String), + #[error("database error: {0}")] Db(#[from] sqlx::Error), @@ -147,6 +150,12 @@ impl IntoResponse for AppError { DB_UNAVAILABLE_CODE, DB_UNAVAILABLE_MESSAGE.into(), ), + // 503 with a Retry-After (attached after this match — the shared tail + // can't carry per-variant headers). This is the single place Overloaded + // becomes a response, so it can never ship a 503 without the retry hint. + AppError::Overloaded(msg) => { + (StatusCode::SERVICE_UNAVAILABLE, "overloaded", msg.clone()) + } AppError::Db(e) => (StatusCode::INTERNAL_SERVER_ERROR, "db_error", e.to_string()), AppError::Internal(e) => ( StatusCode::INTERNAL_SERVER_ERROR, @@ -160,7 +169,17 @@ impl IntoResponse for AppError { "message": message, })); - (status, body).into_response() + let mut resp = (status, body).into_response(); + // Overloaded advertises when to retry. It rides the shared tail above for + // its body/status, so the header is attached here rather than in a bespoke + // early return — keeping the variant handled in exactly one place. + if matches!(self, AppError::Overloaded(_)) { + resp.headers_mut().insert( + axum::http::header::RETRY_AFTER, + axum::http::HeaderValue::from_static("1"), + ); + } + resp } } @@ -182,4 +201,14 @@ mod tests { StatusCode::INTERNAL_SERVER_ERROR ); } + + #[test] + fn overloaded_maps_to_503_with_retry_after() { + let resp = AppError::Overloaded("x".into()).into_response(); + assert_eq!(resp.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!( + resp.headers().get("retry-after").unwrap().to_str().unwrap(), + "1" + ); + } } diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index aa0483db..ee36c783 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -378,6 +378,7 @@ async fn main() -> Result<()> { sync_trigger_rate_limiter, peer_write_rate_limiter, shutdown_tx: shutdown_tx.clone(), + git_semaphore: Arc::new(tokio::sync::Semaphore::new(config.max_concurrent_git_ops)), }; // Periodic peer-count poll for the metrics gauge. If p2p is disabled diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 6a84b3c2..e6e69ea6 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -88,6 +88,10 @@ pub struct AppState { /// * the libp2p swarm task /// * the gossip, sync, operator heartbeat, and rate-limit cleanup loops pub shutdown_tx: tokio::sync::watch::Sender, + /// Bounds concurrent served git operations. A handler acquires a permit + /// before spawning git and holds it for the op; when none are free the + /// request is shed with a 503 rather than exhausting the PID/thread table. + pub git_semaphore: Arc, } impl AppState { diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 4fe52f64..cc322a67 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -82,6 +82,8 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { sync_trigger_rate_limiter: RateLimiter::new(60, Duration::from_secs(3600)), peer_write_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), shutdown_tx: tokio::sync::watch::channel(false).0, + // Generous — no test drives the handler-level shed (git_permit is unit-tested). + git_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), } } @@ -187,6 +189,123 @@ mod tests { ); } + /// PR3 (#62): the served-git concurrency cap sheds at the HTTP layer. One test + /// per git handler drives the `let _permit = git_permit(...)` wiring end to end + /// (this one plus the git_upload_pack / git_receive_pack siblings below); the + /// git_permit unit test covers the helper in isolation. DB-free: an exhausted + /// semaphore sheds before any DB/disk access, so a lazy state works. Remove the + /// permit line from git_info_refs and this goes red (the request falls through + /// to the DB and returns something other than 503). + #[tokio::test] + async fn git_info_refs_sheds_with_503_when_semaphore_exhausted() { + let mut state = test_state_lazy(); + state.git_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + + let router = Router::new() + .route( + "/{owner}/{repo}/info/refs", + axum::routing::get(crate::api::repos::git_info_refs), + ) + .with_state(state); + let resp = router + .oneshot(anon_get( + "/alice/repo.git/info/refs?service=git-upload-pack", + )) + .await + .unwrap(); + + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an exhausted git semaphore must shed info/refs with 503 before touching the DB" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the 503 shed must carry Retry-After" + ); + } + + /// PR3 (#62) sibling of the info/refs shed test: git-upload-pack also acquires a + /// permit at the top, so an exhausted semaphore must shed it with a 503 before + /// any DB/disk work. Anonymous-reachable, so no auth injection is needed. Remove + /// the permit line from git_upload_pack and this goes red. + #[tokio::test] + async fn git_upload_pack_sheds_with_503_when_semaphore_exhausted() { + let mut state = test_state_lazy(); + state.git_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + + let router = Router::new() + .route( + "/{owner}/{repo}/git-upload-pack", + axum::routing::post(crate::api::repos::git_upload_pack), + ) + .with_state(state); + let req = Request::builder() + .method(Method::POST) + .uri("/alice/repo.git/git-upload-pack") + .body(Body::from(&b"0000"[..])) + .unwrap(); + let resp = router.oneshot(req).await.unwrap(); + + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an exhausted git semaphore must shed git-upload-pack with 503 before touching the DB" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the 503 shed must carry Retry-After" + ); + } + + /// PR3 (#62) sibling for the push path: git-receive-pack requires an + /// AuthenticatedDid extension (production: require_signature injects it), so the + /// request carries one via signed_request_as — without it the Extension + /// extractor 500s before the handler body reaches git_permit. The permit is the + /// first statement, so an exhausted semaphore still sheds 503 before any DB + /// work. Remove the permit line from git_receive_pack and this goes red. + #[tokio::test] + async fn git_receive_pack_sheds_with_503_when_semaphore_exhausted() { + let mut state = test_state_lazy(); + state.git_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + + let router = Router::new() + .route( + "/{owner}/{repo}/git-receive-pack", + axum::routing::post(crate::api::repos::git_receive_pack), + ) + .with_state(state); + let owner = "did:key:zRECVSHEDOWNERAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let resp = router + .oneshot(signed_request_as( + owner, + Method::POST, + "/alice/repo.git/git-receive-pack", + Body::from(&b"0000"[..]), + )) + .await + .unwrap(); + + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an exhausted git semaphore must shed git-receive-pack with 503 before touching the DB" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the 503 shed must carry Retry-After" + ); + } + /// N7: merge_pr is owner-only. A non-owner is rejected by require_repo_owner /// before any git work (so no on-disk repo is needed for the rejection). #[sqlx::test] From db4de844add8e1eeb61f2e5b5118751a677ca6f3 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 10 Jul 2026 12:18:07 -0500 Subject: [PATCH 02/58] feat(node,config): add write-pool and per-caller read-cap knobs (#62) Add max_concurrent_git_pushes (default 32) and max_concurrent_reads_per_caller (default 16), both clap range(1..=1_048_576) so an oversized value is a clean CLI error, not a Semaphore::new boot panic. The per-caller knob documents that per-source-IP keying is only as granular as GITLAWB_TRUSTED_PROXY. Wiring lands in the following commits; these are the config surface for the #174 concurrency-fairness fix. Resolves jatmn P1a/P1b groundwork on #174. --- .env.example | 15 +++++ crates/gitlawb-node/src/config.rs | 96 +++++++++++++++++++++++++++++++ 2 files changed, 111 insertions(+) diff --git a/.env.example b/.env.example index bbd9a342..d7c6a30b 100644 --- a/.env.example +++ b/.env.example @@ -113,6 +113,21 @@ GITLAWB_MAX_PACK_BYTES=2147483648 # advertisement or the withheld-blob path, which remain unbounded. Default 600. GITLAWB_GIT_SERVICE_TIMEOUT_SECS=600 +# Max concurrent git-receive-pack (push) operations, in a pool separate from the +# read pool (GITLAWB_MAX_CONCURRENT_GIT_OPS) so anonymous reads cannot shed an +# authenticated push at admission. Over-cap sheds a 503 + Retry-After. Default 32. +GITLAWB_MAX_CONCURRENT_GIT_PUSHES=32 + +# Max concurrent read ops (upload-pack + info/refs advertisements) a single caller +# may hold, so one caller cannot monopolize the read pool. Keyed per-DID when +# signed, else per-source-IP. The per-IP key is only as granular as +# GITLAWB_TRUSTED_PROXY below: left unset, a node behind an edge/NAT keys all +# anonymous callers on the edge IP and this collapses to one global anonymous cap. +# Set GITLAWB_TRUSTED_PROXY for per-client keying; a high-fanout caller (CI behind +# one NAT) should authenticate for a per-DID budget or the operator raises this. +# Default 16. +GITLAWB_MAX_CONCURRENT_READS_PER_CALLER=16 + # ── Push rate limiting (git-receive-pack flood brake) ───────────────────── # Max receive-pack requests (info/refs advertisement + push POST) per client # IP per hour. 0 disables. Default 600. diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index 688e99f4..ff2c2556 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -268,6 +268,42 @@ pub struct Config { value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) )] pub max_concurrent_git_ops: usize, + + /// Maximum number of concurrent `git-receive-pack` (push) operations. Pushes + /// draw from this dedicated pool, separate from `max_concurrent_git_ops` + /// (reads), so a flood of anonymous reads cannot shed an authenticated push at + /// admission (#174). Beyond this a push sheds a clean 503 + Retry-After. + /// + /// Default: 32. Must be between 1 and 1_048_576 (the ceiling keeps the value + /// under tokio's `Semaphore` permit limit so an oversized value is a clean CLI + /// error rather than a boot-time panic). + #[arg( + long, + env = "GITLAWB_MAX_CONCURRENT_GIT_PUSHES", + default_value_t = 32, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub max_concurrent_git_pushes: usize, + + /// Maximum concurrent read operations (`upload-pack` and the `info/refs` + /// advertisements) a single caller may hold at once, so one caller cannot + /// monopolize the `max_concurrent_git_ops` read pool (#174). Callers are keyed + /// per-DID when signed, else per-source-IP. IMPORTANT: the per-source-IP key is + /// only as granular as `GITLAWB_TRUSTED_PROXY`. Left unset (the default), a node + /// behind an edge/NAT keys all anonymous callers on the edge IP, so this cap + /// collapses to a single global anonymous cap rather than per-client. Set + /// `GITLAWB_TRUSTED_PROXY` to key on the real client; a known high-fanout caller + /// (a CI fleet behind one NAT) should authenticate for a per-DID budget or the + /// operator raises this. Over-cap for a caller sheds a clean 503 + Retry-After. + /// + /// Default: 16. Must be between 1 and 1_048_576. + #[arg( + long, + env = "GITLAWB_MAX_CONCURRENT_READS_PER_CALLER", + default_value_t = 16, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub max_concurrent_reads_per_caller: usize, } impl Config { @@ -333,4 +369,64 @@ mod tests { 1_048_576 ); } + + #[test] + fn max_concurrent_git_pushes_defaults_and_rejects_out_of_range() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).max_concurrent_git_pushes, + 32 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--max-concurrent-git-pushes", "8"]) + .max_concurrent_git_pushes, + 8 + ); + // 0 permits would shed every push with a 503; clap must reject it. + assert!( + Config::try_parse_from(["gitlawb-node", "--max-concurrent-git-pushes", "0"]).is_err() + ); + // Above the ceiling would panic tokio's Semaphore::new at boot; clap rejects it. + assert!( + Config::try_parse_from(["gitlawb-node", "--max-concurrent-git-pushes", "1048577"]) + .is_err() + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--max-concurrent-git-pushes", "1048576"]) + .max_concurrent_git_pushes, + 1_048_576 + ); + } + + #[test] + fn max_concurrent_reads_per_caller_defaults_and_rejects_out_of_range() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).max_concurrent_reads_per_caller, + 16 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--max-concurrent-reads-per-caller", "4"]) + .max_concurrent_reads_per_caller, + 4 + ); + // 0 would shed every read from a keyed caller; clap must reject it. + assert!( + Config::try_parse_from(["gitlawb-node", "--max-concurrent-reads-per-caller", "0"]) + .is_err() + ); + assert!(Config::try_parse_from([ + "gitlawb-node", + "--max-concurrent-reads-per-caller", + "1048577" + ]) + .is_err()); + assert_eq!( + Config::parse_from([ + "gitlawb-node", + "--max-concurrent-reads-per-caller", + "1048576" + ]) + .max_concurrent_reads_per_caller, + 1_048_576 + ); + } } From 2434dde42f1ac776da08df8186b7f4a3b54073ae Mon Sep 17 00:00:00 2001 From: t Date: Fri, 10 Jul 2026 12:29:30 -0500 Subject: [PATCH 03/58] fix(node,git): isolate authed pushes in a dedicated write pool (#62) git-receive-pack now draws from a separate git_write_semaphore (max_concurrent_git_pushes) instead of the shared pool, so a flood of anonymous reads can no longer shed an authenticated push at admission (jatmn P1a). The shared field is renamed git_read_semaphore and continues to gate upload-pack and both info/refs advertisements. The write permit stays above acquire_write so it precedes the Tigris fresh-acquire (INV-10). Handler-layer tests: write-pool shed (503), and a cross-boundary proof that an exhausted read pool does NOT shed a push; both mutation-checked (routing receive-pack back to the read pool flips each RED). 497 tests pass. Part of #174. --- crates/gitlawb-node/src/api/repos.rs | 12 ++++--- crates/gitlawb-node/src/auth/mod.rs | 3 +- crates/gitlawb-node/src/main.rs | 5 ++- crates/gitlawb-node/src/state.rs | 14 +++++--- crates/gitlawb-node/src/test_support.rs | 47 ++++++++++++++++++++++--- 5 files changed, 65 insertions(+), 16 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 4bc1f737..dd030a36 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -509,7 +509,7 @@ pub async fn git_info_refs( ) -> Result { // Shed with a 503 before spawning git when the concurrency cap is saturated; // held for the whole op (incl. the smart_http call), released on return. - let _permit = git_permit(&state.git_semaphore)?; + let _permit = git_permit(&state.git_read_semaphore)?; let name = smart_http_repo_name(&repo)?; tracing::info!(owner = %owner, repo = %name, "info/refs request"); let record = state @@ -637,7 +637,7 @@ pub async fn git_upload_pack( ) -> Result { // Shed with a 503 before spawning git when the concurrency cap is saturated; // held for the whole op (incl. the smart_http call), released on return. - let _permit = git_permit(&state.git_semaphore)?; + let _permit = git_permit(&state.git_read_semaphore)?; let name = smart_http_repo_name(&repo)?; let record = state .db @@ -876,9 +876,11 @@ pub async fn git_receive_pack( body: Bytes, ) -> Result { // Shed with a 503 before spawning git when the concurrency cap is saturated. - // Acquired at the very top so it wraps the write-guard below; held for the - // whole op (incl. the smart_http call), released on return. - let _permit = git_permit(&state.git_semaphore)?; + // Pushes draw from the dedicated WRITE pool, separate from reads, so a flood of + // anonymous reads cannot shed an authenticated push (#174). Acquired at the very + // top so it wraps the write-guard below (and precedes the Tigris acquire_write, + // bounding concurrent fresh acquires — INV-10); held for the whole op. + let _permit = git_permit(&state.git_write_semaphore)?; let name = smart_http_repo_name(&repo)?; tracing::info!(owner = %owner, repo = %name, "receive-pack request"); let record = state diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index b541b57b..1d169e1c 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -520,7 +520,8 @@ mod tests { sync_trigger_rate_limiter: RateLimiter::new(60, Duration::from_secs(3600)), peer_write_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), shutdown_tx: tokio::sync::watch::channel(false).0, - git_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), } } diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index ee36c783..b8f687b1 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -378,7 +378,10 @@ async fn main() -> Result<()> { sync_trigger_rate_limiter, peer_write_rate_limiter, shutdown_tx: shutdown_tx.clone(), - git_semaphore: Arc::new(tokio::sync::Semaphore::new(config.max_concurrent_git_ops)), + git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(config.max_concurrent_git_ops)), + git_write_semaphore: Arc::new(tokio::sync::Semaphore::new( + config.max_concurrent_git_pushes, + )), }; // Periodic peer-count poll for the metrics gauge. If p2p is disabled diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index e6e69ea6..b43f4de2 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -88,10 +88,16 @@ pub struct AppState { /// * the libp2p swarm task /// * the gossip, sync, operator heartbeat, and rate-limit cleanup loops pub shutdown_tx: tokio::sync::watch::Sender, - /// Bounds concurrent served git operations. A handler acquires a permit - /// before spawning git and holds it for the op; when none are free the - /// request is shed with a 503 rather than exhausting the PID/thread table. - pub git_semaphore: Arc, + /// Bounds concurrent served git READ operations (upload-pack + both info/refs + /// advertisements). A read handler acquires a permit before spawning git and + /// holds it for the op; when none are free the request is shed with a 503. + /// Writes draw from `git_write_semaphore` so a read flood cannot shed an + /// authenticated push at admission (#174). + pub git_read_semaphore: Arc, + /// Bounds concurrent `git-receive-pack` (push) operations, a pool separate + /// from `git_read_semaphore` so anonymous reads can never shed an authenticated + /// push (#174). Sized by `max_concurrent_git_pushes`. + pub git_write_semaphore: Arc, } impl AppState { diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index cc322a67..fd68c8e6 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -83,7 +83,8 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { peer_write_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), shutdown_tx: tokio::sync::watch::channel(false).0, // Generous — no test drives the handler-level shed (git_permit is unit-tested). - git_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), } } @@ -199,7 +200,7 @@ mod tests { #[tokio::test] async fn git_info_refs_sheds_with_503_when_semaphore_exhausted() { let mut state = test_state_lazy(); - state.git_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + state.git_read_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); let router = Router::new() .route( @@ -235,7 +236,7 @@ mod tests { #[tokio::test] async fn git_upload_pack_sheds_with_503_when_semaphore_exhausted() { let mut state = test_state_lazy(); - state.git_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + state.git_read_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); let router = Router::new() .route( @@ -273,7 +274,7 @@ mod tests { #[tokio::test] async fn git_receive_pack_sheds_with_503_when_semaphore_exhausted() { let mut state = test_state_lazy(); - state.git_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + state.git_write_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); let router = Router::new() .route( @@ -295,7 +296,7 @@ mod tests { assert_eq!( resp.status(), StatusCode::SERVICE_UNAVAILABLE, - "an exhausted git semaphore must shed git-receive-pack with 503 before touching the DB" + "an exhausted write pool must shed git-receive-pack with 503 before touching the DB" ); assert_eq!( resp.headers() @@ -306,6 +307,42 @@ mod tests { ); } + /// #174 (SC1, load-bearing): a saturated READ pool must NOT shed an + /// authenticated push — the write pool is a separate budget. Read pool at zero, + /// write pool with capacity: the push proceeds PAST admission (it then errors on + /// the placeholder DB, but crucially it is not a 503). Route git-receive-pack + /// back to the read pool and this goes red — that is the isolation proof. + #[tokio::test] + async fn git_receive_pack_not_shed_by_exhausted_read_pool() { + let mut state = test_state_lazy(); + // Read pool exhausted as if a flood of anonymous clones held every slot. + state.git_read_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + // Write pool keeps its default capacity from test_state_lazy. + + let router = Router::new() + .route( + "/{owner}/{repo}/git-receive-pack", + axum::routing::post(crate::api::repos::git_receive_pack), + ) + .with_state(state); + let owner = "did:key:zRECVCROSSBOUNDARYAAAAAAAAAAAAAAAAAAAAA"; + let resp = router + .oneshot(signed_request_as( + owner, + Method::POST, + "/alice/repo.git/git-receive-pack", + Body::from(&b"0000"[..]), + )) + .await + .unwrap(); + + assert_ne!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an exhausted READ pool must not shed a push — the write pool is a separate budget (#174)" + ); + } + /// N7: merge_pr is owner-only. A non-owner is rejected by require_repo_owner /// before any git work (so no on-disk repo is needed for the rejection). #[sqlx::test] From fc4c206c3a1ccf38a433b2a4d8255b83910f4357 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 10 Jul 2026 12:50:46 -0500 Subject: [PATCH 04/58] fix(node,git): per-caller concurrency sub-cap on the read pool (#62) Adds PerCallerConcurrency, a bounded-keyed in-flight limiter (distinct from the request-rate RateLimiter) so no single caller monopolizes the served-git read pool. Each caller (per-DID when signed via optional_signature, else per-source-IP via client_key) may hold at most max_concurrent_reads_per_caller concurrent reads; over that it sheds 503. The key map is self-bounding (a key is dropped when its in-flight count hits zero) with a reject-before-insert max_keys backstop so a key farm can't grow it (INV-15). Applied in git_upload_pack and both info/refs advertisements, acquired after the visibility gate so a denied request never consumes a slot (KTD7). Primitive unit-tested (cap + self-bounding + reject-before-insert) and mutation-checked. Handler-layer SC2: same-caller sheds while a different caller passes, proven on BOTH git_info_refs and git_upload_pack with independent mutation probes; plus a None-key bypass test. Per-source-IP keying is trust-config dependent, documented on the config knob. 502 tests pass. Part of #174. --- crates/gitlawb-node/src/api/repos.rs | 210 ++++++++++++++++++++++++ crates/gitlawb-node/src/auth/mod.rs | 1 + crates/gitlawb-node/src/main.rs | 3 + crates/gitlawb-node/src/rate_limit.rs | 142 ++++++++++++++++ crates/gitlawb-node/src/state.rs | 5 + crates/gitlawb-node/src/test_support.rs | 1 + 6 files changed, 362 insertions(+) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index dd030a36..e51b62e6 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -566,6 +566,28 @@ pub async fn git_info_refs( } } + // Per-caller read sub-cap (#174): acquired AFTER the visibility + push-rate + // gates (KTD7) so a denied or rate-limited request never consumes a caller's + // scarce read slot. Applies to BOTH advertisements. Held for the whole op. + let caller_key = read_caller_key( + auth.as_ref().map(|e| e.0 .0.as_str()), + &headers, + peer, + state.push_limiter_trust, + ); + let _caller_permit = match &caller_key { + Some(k) => match state.git_read_per_caller.try_acquire(k) { + Some(p) => Some(p), + None => { + tracing::warn!(repo = %name, caller = %k, "per-caller read cap reached; shedding info/refs with 503"); + return Err(AppError::Overloaded( + "git read service at capacity for this caller, retry shortly".into(), + )); + } + }, + None => None, + }; + // For receive-pack (push), download the latest from Tigris so the client // sees the same refs that acquire_write() will operate on. let disk_path = if service == "git-receive-pack" { @@ -608,6 +630,23 @@ fn git_permit( }) } +/// Resolve the per-caller key for the read sub-cap (#174): the authenticated DID +/// when the caller signed (via `optional_signature`), else the trusted client IP +/// (`client_key`). `None` when neither is available — such a request is bounded by +/// the global read pool only, never a 500. The per-source-IP key is only as +/// granular as `trust`; see the `max_concurrent_reads_per_caller` config doc. +fn read_caller_key( + caller_did: Option<&str>, + headers: &axum::http::HeaderMap, + peer: Option, + trust: crate::rate_limit::TrustedProxy, +) -> Option { + match caller_did { + Some(did) => Some(did.to_string()), + None => crate::rate_limit::client_key(headers, peer, trust), + } +} + /// Map an error from a `smart_http` git service call to the right `AppError`: /// [`smart_http::GitServiceTimeout`] to 504, a malformed client request to 400, /// anything else to a 500 git error. Pure (no logging) so it is unit-testable; @@ -633,6 +672,8 @@ pub async fn git_upload_pack( State(state): State, Path((owner, repo)): Path<(String, String)>, auth: Option>, + crate::rate_limit::PeerAddr(peer): crate::rate_limit::PeerAddr, + headers: axum::http::HeaderMap, body: Bytes, ) -> Result { // Shed with a 503 before spawning git when the concurrency cap is saturated; @@ -658,6 +699,23 @@ pub async fn git_upload_pack( return Err(AppError::RepoNotFound(format!("{owner}/{name}"))); } + // Per-caller read sub-cap (#174): after the visibility gate (KTD7) so a + // visibility-denied caller never consumes a scarce read slot. Keyed per-DID + // when signed, else per-source-IP; no resolvable key -> global read pool only. + let caller_key = read_caller_key(caller, &headers, peer, state.push_limiter_trust); + let _caller_permit = match &caller_key { + Some(k) => match state.git_read_per_caller.try_acquire(k) { + Some(p) => Some(p), + None => { + tracing::warn!(repo = %name, caller = %k, "per-caller read cap reached; shedding upload-pack with 503"); + return Err(AppError::Overloaded( + "git read service at capacity for this caller, retry shortly".into(), + )); + } + }, + None => None, + }; + let disk_path = state .repo_store .acquire(&record.owner_did, &record.name) @@ -2618,6 +2676,158 @@ mod tests { ); } + /// #174 SC2 (info_refs probe): the per-caller read sub-cap sheds a caller that + /// is already at its concurrency budget on the upload-pack advertisement, while + /// a DIFFERENT caller still enters. Remove the sub-cap from `git_info_refs` and + /// the same-caller assertion goes green-not-503 — this is the info_refs half of + /// the two-handler mutation probe. + #[sqlx::test] + async fn info_refs_per_caller_cap_sheds_one_caller_not_others(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6pcadv", "pc", "/tmp/pc-nonexistent", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.31:5000".parse().unwrap(); + // Fill this caller's single read slot (a clone shares the Arc-backed map). + let _slot = state + .git_read_per_caller + .try_acquire(&peer.ip().to_string()) + .expect("first slot for this caller"); + + // Same caller (IP) at its cap -> shed 503 before the git/Tigris work. + let router = crate::server::build_router(state.clone()); + let mut req = Request::builder() + .method(Method::GET) + .uri("/z6pcadv/pc/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + assert_eq!( + router.oneshot(req).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a caller already at its per-caller read cap must shed the advertisement with 503" + ); + + // A DIFFERENT caller (IP) has its own budget -> not shed by the per-caller cap. + let other: SocketAddr = "203.0.113.32:5000".parse().unwrap(); + let router2 = crate::server::build_router(state.clone()); + let mut req2 = Request::builder() + .method(Method::GET) + .uri("/z6pcadv/pc/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + req2.extensions_mut().insert(ConnectInfo(other)); + assert_ne!( + router2.oneshot(req2).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a different caller must not be shed by another caller's saturated budget" + ); + } + + /// #174 SC2 (upload_pack probe): the same per-caller shed on the POST + /// upload-pack path. Remove the sub-cap from `git_upload_pack` and this goes + /// green-not-503 — the upload_pack half of the two-handler mutation probe. + #[sqlx::test] + async fn upload_pack_per_caller_cap_sheds_one_caller_not_others(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6pcupl", "pc", "/tmp/pc-nonexistent", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.41:5000".parse().unwrap(); + let _slot = state + .git_read_per_caller + .try_acquire(&peer.ip().to_string()) + .expect("first slot for this caller"); + + let router = crate::server::build_router(state.clone()); + let mut req = Request::builder() + .method(Method::POST) + .uri("/z6pcupl/pc/git-upload-pack") + .body(Body::from(&b"0000"[..])) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + assert_eq!( + router.oneshot(req).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a caller already at its per-caller read cap must shed upload-pack with 503" + ); + + let other: SocketAddr = "203.0.113.42:5000".parse().unwrap(); + let router2 = crate::server::build_router(state.clone()); + let mut req2 = Request::builder() + .method(Method::POST) + .uri("/z6pcupl/pc/git-upload-pack") + .body(Body::from(&b"0000"[..])) + .unwrap(); + req2.extensions_mut().insert(ConnectInfo(other)); + assert_ne!( + router2.oneshot(req2).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a different caller must not be shed by another caller's saturated budget" + ); + } + + /// #174 SC2 (None-key): a request with no resolvable caller key (no ConnectInfo, + /// no trusted header) must NOT be shed by the per-caller cap even when another + /// caller's budget is full — it is bounded by the global read pool only. A None + /// key never keys into the map, so it never 503s from the per-caller sub-cap. + #[sqlx::test] + async fn info_refs_none_key_bypasses_per_caller_cap(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::http::{Method, Request, StatusCode}; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6pcnone", "pc", "/tmp/pc-nonexistent", None, false) + .await + .unwrap(); + // Saturate an unrelated caller's budget; the None-key request must be + // unaffected because it never keys into the per-caller map. + let _slot = state + .git_read_per_caller + .try_acquire("203.0.113.99") + .expect("hold an unrelated caller's slot"); + + // No ConnectInfo inserted -> PeerAddr is None -> no per-caller key. + let router = crate::server::build_router(state.clone()); + let req = Request::builder() + .method(Method::GET) + .uri("/z6pcnone/pc/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + assert_ne!( + router.oneshot(req).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a request with no resolvable caller key must not be shed by the per-caller cap" + ); + } + /// Repo creation must be throttled by the per-IP creation limiter BEFORE /// signature verification — otherwise a DID farm (one throwaway did:key per /// repo, each carrying a valid but machine-solved iCaptcha proof) walks past diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index 1d169e1c..253b3210 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -522,6 +522,7 @@ mod tests { shutdown_tx: tokio::sync::watch::channel(false).0, git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), } } diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index b8f687b1..c617b434 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -382,6 +382,9 @@ async fn main() -> Result<()> { git_write_semaphore: Arc::new(tokio::sync::Semaphore::new( config.max_concurrent_git_pushes, )), + git_read_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( + config.max_concurrent_reads_per_caller, + ), }; // Periodic peer-count poll for the metrics gauge. If p2p is disabled diff --git a/crates/gitlawb-node/src/rate_limit.rs b/crates/gitlawb-node/src/rate_limit.rs index d40e2691..d505f1bb 100644 --- a/crates/gitlawb-node/src/rate_limit.rs +++ b/crates/gitlawb-node/src/rate_limit.rs @@ -132,6 +132,91 @@ impl RateLimiter { } } +/// A bounded per-caller CONCURRENCY limiter — distinct from [`RateLimiter`], which +/// caps request RATE. Each caller key may hold at most `per_caller` in-flight +/// permits at once; beyond that [`try_acquire`](Self::try_acquire) returns `None` +/// and the caller sheds. Used to stop one caller (a single anonymous source-IP or +/// DID) monopolizing the served-git read pool (#174). +/// +/// The key map is self-bounding: a key is removed the instant its in-flight count +/// reaches zero, so it never holds more keys than there are concurrently-active +/// callers (itself bounded by the read semaphore). A `max_keys` reject-before-insert +/// backstop guarantees a key farm can never grow the map even if that invariant +/// weakened — a NEW key at the cap is rejected WITHOUT allocating an entry (INV-15). +/// +/// Uses a `std::sync::Mutex` (not the file's `tokio::sync::Mutex`) because the +/// permit's `Drop` must release synchronously; the critical section holds no await. +#[derive(Clone)] +pub struct PerCallerConcurrency { + state: Arc>>, + per_caller: usize, + max_keys: usize, +} + +/// RAII permit from [`PerCallerConcurrency::try_acquire`]. On drop it decrements +/// the caller's in-flight count and removes the key when it reaches zero. +pub struct PerCallerPermit { + state: Arc>>, + key: String, +} + +impl PerCallerConcurrency { + pub fn new(per_caller: usize, max_keys: usize) -> Self { + Self { + state: Arc::new(std::sync::Mutex::new(HashMap::new())), + per_caller: per_caller.max(1), + max_keys: max_keys.max(1), + } + } + + /// Convenience constructor with the default key bound. + pub fn with_default_max_keys(per_caller: usize) -> Self { + Self::new(per_caller, DEFAULT_MAX_KEYS) + } + + /// `Some(permit)` when the caller is under its cap and the map has room; + /// `None` (shed) otherwise. Reject-before-insert: a new key at `max_keys` is + /// rejected without allocating. + pub fn try_acquire(&self, key: &str) -> Option { + let mut map = self.state.lock().expect("per-caller mutex poisoned"); + match map.get_mut(key) { + Some(count) => { + if *count >= self.per_caller { + return None; + } + *count += 1; + } + None => { + if map.len() >= self.max_keys { + return None; + } + map.insert(key.to_string(), 1); + } + } + Some(PerCallerPermit { + state: self.state.clone(), + key: key.to_string(), + }) + } + + #[cfg(test)] + pub fn tracked_keys(&self) -> usize { + self.state.lock().unwrap().len() + } +} + +impl Drop for PerCallerPermit { + fn drop(&mut self) { + let mut map = self.state.lock().expect("per-caller mutex poisoned"); + if let Some(count) = map.get_mut(&self.key) { + *count -= 1; + if *count == 0 { + map.remove(&self.key); + } + } + } +} + pub async fn rate_limit_by_did(request: Request, next: Next) -> Response { let limiter = request.extensions().get::().cloned(); @@ -288,6 +373,63 @@ pub async fn rate_limit_by_ip(request: Request, next: Next) -> Response { mod tests { use super::*; + #[test] + fn per_caller_concurrency_caps_one_caller_and_frees_on_drop() { + let lim = PerCallerConcurrency::new(2, 100); + let p1 = lim.try_acquire("did:key:zA").expect("first under cap"); + let p2 = lim.try_acquire("did:key:zA").expect("second under cap"); + assert!( + lim.try_acquire("did:key:zA").is_none(), + "a third in-flight op for the same caller sheds (over the per-caller cap)" + ); + // A DIFFERENT caller is unaffected — the cap is per-caller, not global. + let _other = lim + .try_acquire("did:key:zB") + .expect("a different caller has its own budget"); + drop(p1); + assert!( + lim.try_acquire("did:key:zA").is_some(), + "freeing one in-flight slot lets the same caller back in" + ); + drop(p2); + } + + #[test] + fn per_caller_concurrency_map_is_self_bounding_and_reject_before_insert() { + // Self-bounding: acquire+drop many distinct keys — the map never grows + // because a key is removed the instant its in-flight count hits zero. + let lim = PerCallerConcurrency::new(4, 3); + for i in 0..50 { + let _p = lim.try_acquire(&format!("k{i}")); + } + assert_eq!( + lim.tracked_keys(), + 0, + "keys with zero in-flight ops are removed, so an acquire+drop flood leaves the map empty" + ); + // Reject-before-insert: HOLD max_keys distinct keys, then a new key sheds + // WITHOUT growing the map past the cap (INV-15 — a rejected request never + // allocates an entry). + let held: Vec<_> = (0..3) + .map(|i| lim.try_acquire(&format!("h{i}")).unwrap()) + .collect(); + assert_eq!( + lim.tracked_keys(), + 3, + "three distinct callers held concurrently" + ); + assert!( + lim.try_acquire("h3").is_none(), + "a new key at max_keys is rejected" + ); + assert_eq!( + lim.tracked_keys(), + 3, + "the rejected new key did not allocate an entry (reject-before-insert)" + ); + drop(held); + } + #[tokio::test] async fn allows_within_limit() { let limiter = RateLimiter::new(3, Duration::from_secs(60)); diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index b43f4de2..01007ac4 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -98,6 +98,11 @@ pub struct AppState { /// from `git_read_semaphore` so anonymous reads can never shed an authenticated /// push (#174). Sized by `max_concurrent_git_pushes`. pub git_write_semaphore: Arc, + /// Per-caller concurrency sub-cap on the read pool: each caller (per-DID when + /// signed, else per-source-IP) may hold at most `max_concurrent_reads_per_caller` + /// in-flight read ops, so one caller cannot monopolize `git_read_semaphore` + /// (#174). Applied by `git_upload_pack` and both `info/refs` advertisements. + pub git_read_per_caller: crate::rate_limit::PerCallerConcurrency, } impl AppState { diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index fd68c8e6..7745d233 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -85,6 +85,7 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { // Generous — no test drives the handler-level shed (git_permit is unit-tested). git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), } } From 2c264e08e391f5675cbfd3edeff0238434e9bb15 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 10 Jul 2026 13:09:34 -0500 Subject: [PATCH 05/58] fix(node,git): bound and reap the info/refs advertisement (#62) info_refs ran a bare Command::output() with no timeout and no process-group teardown, so a hung git pinned its concurrency slot indefinitely and a client disconnect orphaned the child (jatmn P1b). Extract the timeout + process_group(0) + KillGroupOnDrop core from run_git_service into a shared drive_git_child, and route info_refs through it with an injectable git_bin. A hung advertisement now aborts with GitServiceTimeout (mapped to 504); disconnect reaps the group. run_git_service's teardown tests all pass through the shared core (proving the group teardown info_refs inherits), the real-git filter tests cover the advertisement happy path, and a new watchdog-bounded test proves a hung advertisement times out. 503 tests pass. Part of #174. --- crates/gitlawb-node/src/api/repos.rs | 15 +++- crates/gitlawb-node/src/git/smart_http.rs | 87 +++++++++++++++++++---- 2 files changed, 85 insertions(+), 17 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index e51b62e6..c42a6989 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -606,11 +606,20 @@ pub async fn git_info_refs( AppError::Git(e.to_string()) })?; - smart_http::info_refs(&disk_path, &service) + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + smart_http::info_refs("git", &service, &disk_path, git_timeout) .await .map_err(|e| { - tracing::error!(repo = %name, service = %service, err = %e, "info_refs git failed"); - AppError::Git(e.to_string()) + let app = git_service_app_error(&e); + match &app { + AppError::Timeout(_) => { + tracing::warn!(repo = %name, service = %service, "info/refs advertisement timed out") + } + _ => { + tracing::error!(repo = %name, service = %service, err = %e, "info_refs git failed") + } + } + app }) } diff --git a/crates/gitlawb-node/src/git/smart_http.rs b/crates/gitlawb-node/src/git/smart_http.rs index eeb35b99..b4e05a01 100644 --- a/crates/gitlawb-node/src/git/smart_http.rs +++ b/crates/gitlawb-node/src/git/smart_http.rs @@ -14,21 +14,29 @@ use tokio::process::Command; /// or `?service=git-receive-pack` /// /// This is the ref advertisement — the first step of a clone or push. -pub async fn info_refs(repo_path: &Path, service: &str) -> Result { +/// +/// `git_bin` is injectable purely so the process-group teardown can be driven by a +/// fake `git` in tests (production passes `"git"`). `timeout` bounds the whole child +/// interaction: previously the advertisement ran a bare `Command::output()` with no +/// deadline and no teardown, so a hung git pinned its concurrency slot indefinitely +/// and a client disconnect orphaned the child (#174). It now shares +/// [`drive_git_child`]'s timeout + `process_group(0)` + [`KillGroupOnDrop`] teardown. +pub async fn info_refs( + git_bin: &str, + service: &str, + repo_path: &Path, + timeout: Duration, +) -> Result { validate_service(service)?; - let output = Command::new("git") + let mut command = Command::new(git_bin); + command .arg(service_to_command(service)) .arg("--stateless-rpc") .arg("--advertise-refs") - .arg(repo_path) - .output() - .await?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - bail!("git {service} --advertise-refs failed: {stderr}"); - } + .arg(repo_path); + // No request body — advertise-refs does not read stdin. + let stdout = drive_git_child(command, Bytes::new(), timeout, "advertise-refs").await?; let content_type = format!("application/x-{service}-advertisement"); @@ -38,7 +46,7 @@ pub async fn info_refs(repo_path: &Path, service: &str) -> Result { let mut body = Vec::new(); body.extend_from_slice(&pkt_service); body.extend_from_slice(flush); - body.extend_from_slice(&output.stdout); + body.extend_from_slice(&stdout); Ok(Response::builder() .status(StatusCode::OK) @@ -207,7 +215,24 @@ async fn run_git_service( command .arg(service_to_command(service)) .arg("--stateless-rpc") - .arg(repo_path) + .arg(repo_path); + drive_git_child(command, input, timeout, service).await +} + +/// Drive a spawned git child under `timeout` with process-group teardown, returning +/// its stdout. Shared core for [`run_git_service`] and [`info_refs`]: the caller +/// passes a `Command` with its args set; this adds piped stdio and `process_group(0)`. +/// On the deadline the whole group is torn down and reaped before returning +/// [`GitServiceTimeout`]; on a dropped future (client disconnect) the +/// [`KillGroupOnDrop`] guard fires. `input` is written to the child's stdin (empty +/// for the advertise-refs path, which has no request body); `what` labels errors. +async fn drive_git_child( + mut command: Command, + input: Bytes, + timeout: Duration, + what: &str, +) -> Result> { + command .stdin(Stdio::piped()) .stdout(Stdio::piped()) .stderr(Stdio::piped()); @@ -294,7 +319,7 @@ async fn run_git_service( // the real cause. if !status.success() { let stderr = String::from_utf8_lossy(&err); - bail!("{service} failed: {stderr}"); + bail!("{what} failed: {stderr}"); } write_result.context("failed to write to git stdin")?; @@ -650,7 +675,9 @@ mod tests { axum::extract::Query(q): axum::extract::Query>, ) -> Response { let service = q.get("service").cloned().unwrap_or_default(); - info_refs(&st.repo, &service).await.unwrap() + info_refs("git", &service, &st.repo, Duration::from_secs(30)) + .await + .unwrap() } async fn pack_handler( @@ -1225,6 +1252,38 @@ mod tests { ); } + // #174 (SC3): the info/refs advertisement is now duration-bounded — previously + // it ran a bare `Command::output()` with no deadline, so a hung git pinned its + // concurrency slot forever. A hung fake git must abort with GitServiceTimeout + // (which the handler maps to 504). The outer watchdog turns a missing timeout + // into a loud failure instead of hanging the suite. info_refs shares + // drive_git_child, so its disconnect/group-teardown is the same code proven by + // run_git_service_tears_down_group_when_future_dropped above. + #[cfg(unix)] + #[tokio::test] + async fn info_refs_times_out_a_hung_advertisement() { + let tmp = tempfile::TempDir::new().unwrap(); + // A fake git that hangs forever instead of advertising refs. + let git_bin = write_fake_git(tmp.path(), "#!/bin/sh\nsleep 300\n"); + + let result = tokio::time::timeout( + Duration::from_secs(10), + info_refs( + git_bin.to_str().unwrap(), + "git-upload-pack", + tmp.path(), + Duration::from_millis(200), + ), + ) + .await + .expect("info_refs must return within the watchdog — its own timeout must fire"); + let err = result.expect_err("a hung advertisement must return an error, not hang"); + assert!( + err.downcast_ref::().is_some(), + "a hung advertisement must abort with GitServiceTimeout (-> 504), got: {err}" + ); + } + // A request that runs to completion must DISARM the guard after reaping, so // no stray group SIGTERM fires. The fake exits non-zero (surfacing as Err) // but leaves a grandchild alive; the grandchild must survive. Goes RED if the From 57c6e01c26ac1e3032cb06b57ebc640e18582a85 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 10 Jul 2026 13:16:41 -0500 Subject: [PATCH 06/58] fix(node,git): bound and reap the withheld-blob pack build (#62) The filtered-pack path ran the whole rev-list + pack-objects build inside a spawn_blocking, so an outer tokio timeout could not cancel the blocking thread and a client disconnect orphaned the git child while the permit freed (jatmn P1b, the second gap path). Split it: rev-list enumeration stays blocking off the runtime (rev_list_keep), but the streaming pack-objects stage now runs under the shared drive_git_child on the async side, so it is duration-bounded (GitServiceTimeout -> 504) and its process group is reaped on disconnect. build_filtered_pack becomes async and takes a git_bin seam + timeout; upload_pack_excluding threads the git_service_timeout through. A watchdog-bounded test proves a hung pack-objects times out (rev-list fast, pack-objects hangs). The refactor's happy path is covered by the existing filtered-pack correctness and real-git partial-clone/fetch tests, all still green; disconnect/group-teardown is the shared drive_git_child code proven by the run_git_service tests. 504 tests pass. Part of #174. --- crates/gitlawb-node/src/api/repos.rs | 2 +- crates/gitlawb-node/src/git/smart_http.rs | 145 ++++++++++++++-------- 2 files changed, 95 insertions(+), 52 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index c42a6989..dba8ea71 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -765,7 +765,7 @@ pub async fn git_upload_pack( smart_http::upload_pack(&disk_path, body, git_timeout).await } else { tracing::info!(repo = %name, caller = ?caller, withheld = withheld.len(), "serving filtered pack"); - smart_http::upload_pack_excluding(&disk_path, body, &withheld).await + smart_http::upload_pack_excluding(&disk_path, body, &withheld, git_timeout).await } } .map_err(|e| { diff --git a/crates/gitlawb-node/src/git/smart_http.rs b/crates/gitlawb-node/src/git/smart_http.rs index b4e05a01..5586a2c7 100644 --- a/crates/gitlawb-node/src/git/smart_http.rs +++ b/crates/gitlawb-node/src/git/smart_http.rs @@ -349,12 +349,17 @@ fn pkt_line(data: &str) -> Vec { format!("{len:04x}{data}").into_bytes() } -/// Build a packfile containing every object reachable from all refs EXCEPT the -/// given blob OIDs. Commits and trees are always included, so SHAs stay intact; -/// only the named blobs are dropped. -pub fn build_filtered_pack(repo_path: &Path, withheld: &HashSet) -> Result> { - // All reachable objects as "oid [path]" lines. - let rev = std::process::Command::new("git") +/// Run `rev-list --objects --all` and return the reachable object ids minus the +/// withheld blobs. Blocking (shells out to git); the caller runs it off the async +/// runtime. Kept separate from the pack-objects stage so that stage can live on the +/// async side under [`drive_git_child`] (`git_bin` is injectable for the same +/// fake-git testing reason as `run_git_service`). +fn rev_list_keep( + git_bin: &str, + repo_path: &Path, + withheld: &HashSet, +) -> Result> { + let rev = std::process::Command::new(git_bin) .args(["rev-list", "--objects", "--all"]) .current_dir(repo_path) .output()?; @@ -372,39 +377,39 @@ pub fn build_filtered_pack(repo_path: &Path, withheld: &HashSet) -> Resu } keep.push(oid.to_string()); } - let mut child = std::process::Command::new("git") - .args(["pack-objects", "--stdout"]) - .current_dir(repo_path) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()?; - // Feed the object ids on stdin, but always reap the child afterward even if - // the write fails or stdin is missing, so an error can't drop the Child - // unwaited and leak a zombie (#53). - let write_result: std::io::Result<()> = { - use std::io::Write as _; - match child.stdin.take() { - Some(mut stdin) => { - let mut data = keep.join("\n").into_bytes(); - data.push(b'\n'); - stdin.write_all(&data) - } - None => Err(std::io::Error::new( - std::io::ErrorKind::BrokenPipe, - "git pack-objects stdin unavailable", - )), - } + Ok(keep) +} + +/// Build a packfile containing every object reachable from all refs EXCEPT the +/// given blob OIDs. Commits and trees are always included, so SHAs stay intact; +/// only the named blobs are dropped. +/// +/// The rev-list enumeration runs blocking off the async runtime; the streaming +/// `pack-objects` stage runs under [`drive_git_child`], so a hung/slow pack build is +/// duration-bounded and its process group is reaped on client disconnect. An outer +/// `tokio::time::timeout` around a `spawn_blocking` cannot cancel the blocking +/// thread, so the streaming stage MUST live on the async side (#174, KTD5). +pub async fn build_filtered_pack( + git_bin: &str, + repo_path: &Path, + withheld: &HashSet, + timeout: Duration, +) -> Result> { + let keep = { + let git_bin = git_bin.to_string(); + let repo_path = repo_path.to_path_buf(); + let withheld = withheld.clone(); + tokio::task::spawn_blocking(move || rev_list_keep(&git_bin, &repo_path, &withheld)) + .await + .context("rev-list task panicked")?? }; - let out = child.wait_with_output()?; - write_result.context("failed to write object ids to git pack-objects stdin")?; - if !out.status.success() { - bail!( - "git pack-objects failed: {}", - String::from_utf8_lossy(&out.stderr) - ); - } - Ok(out.stdout) + let mut data = keep.join("\n").into_bytes(); + data.push(b'\n'); + let mut command = Command::new(git_bin); + command + .args(["pack-objects", "--stdout"]) + .current_dir(repo_path); + drive_git_child(command, Bytes::from(data), timeout, "pack-objects").await } /// Serve a clone/fetch with the withheld blobs removed from the response pack. @@ -434,17 +439,13 @@ pub async fn upload_pack_excluding( repo_path: &Path, request_body: Bytes, withheld: &HashSet, + timeout: Duration, ) -> Result { - // build_filtered_pack shells out to git (rev-list, pack-objects) with - // blocking std::process I/O; run it off the async worker so a large repo's - // pack build does not stall the tokio runtime. - let pack = { - let repo_path = repo_path.to_path_buf(); - let withheld = withheld.clone(); - tokio::task::spawn_blocking(move || build_filtered_pack(&repo_path, &withheld)) - .await - .context("filtered-pack build task panicked")?? - }; + // The rev-list enumeration runs blocking off the runtime; the streaming + // pack-objects stage is duration-bounded and its process group is reaped on + // disconnect via drive_git_child (#174), so a hung build no longer pins its + // concurrency slot and a client disconnect no longer orphans the git child. + let pack = build_filtered_pack("git", repo_path, withheld, timeout).await?; // The client lists its capabilities on the first `want` line. Honor // side-band-64k when offered (every modern smart-HTTP client offers it); @@ -563,7 +564,9 @@ mod tests { let mut withheld = std::collections::HashSet::new(); withheld.insert(secret.clone()); - let pack = build_filtered_pack(&bare, &withheld).unwrap(); + let pack = build_filtered_pack("git", &bare, &withheld, Duration::from_secs(30)) + .await + .unwrap(); let ids = pack_object_ids(&pack); assert!(ids.contains(&public), "public blob must be in the pack"); assert!( @@ -625,7 +628,9 @@ mod tests { b"0098want 0000000000000000000000000000000000000000 \ side-band-64k ofs-delta agent=git/2\n00000009done\n", ); - let resp = upload_pack_excluding(&bare, req, &withheld).await.unwrap(); + let resp = upload_pack_excluding(&bare, req, &withheld, Duration::from_secs(30)) + .await + .unwrap(); let body = to_bytes(resp.into_body(), usize::MAX).await.unwrap(); let ids = pack_object_ids(&extract_pack(&body)); assert!( @@ -684,7 +689,7 @@ mod tests { axum::extract::State(st): axum::extract::State>, body: Bytes, ) -> Response { - upload_pack_excluding(&st.repo, body, &st.withheld) + upload_pack_excluding(&st.repo, body, &st.withheld, Duration::from_secs(30)) .await .unwrap() } @@ -1284,6 +1289,44 @@ mod tests { ); } + // #174 (SC3, KTD5): the filtered-pack build's streaming pack-objects stage is + // duration-bounded on the ASYNC side. The old build ran the whole thing in a + // spawn_blocking, so an outer tokio timeout could not cancel the blocking thread + // and a disconnect orphaned the git child. A fake git that returns objects fast + // on rev-list but hangs on pack-objects must now abort with GitServiceTimeout; + // the watchdog turns a missing bound into a loud failure. The stage runs under + // drive_git_child, so its disconnect/group-teardown is the same code proven by + // run_git_service_tears_down_group_when_future_dropped. + #[cfg(unix)] + #[tokio::test] + async fn build_filtered_pack_times_out_a_hung_pack_objects() { + let tmp = tempfile::TempDir::new().unwrap(); + // rev-list returns one oid fast; pack-objects hangs forever. + let body = "#!/bin/sh\ncase \"$1\" in\n rev-list) echo deadbeefdeadbeefdeadbeefdeadbeefdeadbeef ;;\n pack-objects) sleep 300 ;;\n *) exit 1 ;;\nesac\n"; + let git_bin = write_fake_git(tmp.path(), body); + let withheld = HashSet::new(); + + let result = tokio::time::timeout( + Duration::from_secs(10), + build_filtered_pack( + git_bin.to_str().unwrap(), + tmp.path(), + &withheld, + Duration::from_millis(200), + ), + ) + .await + .expect( + "build_filtered_pack must return within the watchdog — the pack-objects \ + stage must be timeout-bounded, not an uncancellable spawn_blocking", + ); + let err = result.expect_err("a hung pack-objects must return an error, not hang"); + assert!( + err.downcast_ref::().is_some(), + "a hung pack-objects must abort with GitServiceTimeout, got: {err}" + ); + } + // A request that runs to completion must DISARM the guard after reaping, so // no stray group SIGTERM fires. The fake exits non-zero (surfacing as Err) // but leaves a grandchild alive; the grandchild must survive. Goes RED if the From 6336657efcdaf3babc8234224e78e5867d2ebf25 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 10 Jul 2026 13:20:18 -0500 Subject: [PATCH 07/58] docs(node,config): reconcile the concurrency-cap comments with the fix (#62) The max_concurrent_git_ops and git_service_timeout_secs doc-comments (and .env.example) described the info/refs and withheld-blob paths as unbounded follow-up gaps. Both are now closed (#174): the comments reflect the read/write pool split, the per-caller sub-cap, and that every capped path is duration-bounded with process-group teardown. Verified the pattern-doc pre-ship checklist: every git_permit / write-permit / per-caller site holds only a timeout+teardown git path, and all three size knobs are range(1..=1_048_576). Closes the #174 work. No behavior change. --- .env.example | 4 ++-- crates/gitlawb-node/src/config.rs | 32 +++++++++++++++---------------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.env.example b/.env.example index d7c6a30b..69865920 100644 --- a/.env.example +++ b/.env.example @@ -109,8 +109,8 @@ GITLAWB_MAX_PACK_BYTES=2147483648 # Max seconds a served git upload-pack / receive-pack (clone / push) may run # before it is aborted with a 504. Bounds a hung git that would otherwise pin a -# worker and, on push, the repo write lock. Does NOT cover the info/refs -# advertisement or the withheld-blob path, which remain unbounded. Default 600. +# worker and, on push, the repo write lock. Also bounds the info/refs +# advertisement and the withheld-blob pack build (#174). Default 600. GITLAWB_GIT_SERVICE_TIMEOUT_SECS=600 # Max concurrent git-receive-pack (push) operations, in a pool separate from the diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index ff2c2556..93c49449 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -172,9 +172,9 @@ pub struct Config { /// its process group torn down, in seconds. Bounds a git that neither /// finishes nor disconnects. Must be positive; set it very large to /// effectively disable the bound. Default: 600s (10 min), generous for large - /// clones. Does not cover the ref advertisement (`info/refs`) or the - /// withheld-blob fetch path (`upload_pack_excluding`, a blocking - /// `spawn_blocking` a tokio timeout cannot cancel); both remain unbounded. + /// clones. Also bounds the ref advertisement (`info/refs`) and the withheld-blob + /// pack build (`upload_pack_excluding`'s pack-objects stage), which now share the + /// same timeout + process-group teardown (#174). #[arg( long, env = "GITLAWB_GIT_SERVICE_TIMEOUT_SECS", @@ -243,20 +243,20 @@ pub struct Config { /// cap is a different axis (500 connections each fan out to git + /// pack-objects + threads). Size below the process budget with headroom. /// - /// A permit is held for the whole op. upload-pack/receive-pack are - /// duration-bounded by `git_service_timeout_secs`, but the `info/refs` - /// advertisement and the withheld-blob (`upload_pack_excluding`) path are - /// not, so a hung git on those two paths holds its slot until it exits, so a - /// stuck advertisement permanently costs one unit of capacity. Bounding - /// those paths' duration is tracked as separate follow-up. + /// This is the READ pool (`git_read_semaphore`): upload-pack and both info/refs + /// advertisements. Authenticated pushes draw from a separate write pool + /// (`max_concurrent_git_pushes`) and each caller is additionally bounded by + /// `max_concurrent_reads_per_caller`, so an anonymous flood can neither shed a + /// push nor monopolize reads (#174). /// - /// The same two paths have the mirror gap on cancellation: the permit frees - /// when the handler future drops (client disconnect), but neither reaps its - /// git child — `info/refs` runs a bare `Command`, and `upload_pack_excluding` - /// a blocking `spawn_blocking` a dropped future cannot cancel — so the child - /// runs to completion after its slot is freed and live git can briefly exceed - /// the cap. The main pack path (`run_git_service`) tears its process group - /// down on drop, so this is confined to the same two follow-up paths. + /// A permit is held for the whole op, and every capped path is now + /// duration-bounded and reaps its process group on disconnect: upload-pack, + /// receive-pack, and both info/refs advertisements run under + /// `git_service_timeout_secs` with `process_group(0)` teardown, and the + /// withheld-blob (`upload_pack_excluding`) pack-objects stage runs on the async + /// side under the same teardown. So a hung git can no longer pin a slot and a + /// client disconnect no longer orphans its git child (#174 closed the two + /// duration/cancellation gaps this comment previously tracked as follow-up). /// /// Default: 128. Must be between 1 and 1_048_576; the ceiling keeps the value /// well under tokio's `Semaphore` permit limit so an oversized value is a From 5069cd13ca18eeaf7f6be59ace288fa30ca35472 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 10 Jul 2026 13:45:38 -0500 Subject: [PATCH 08/58] fix(node,concurrency): resolve #174 code-review findings Review of the served-git concurrency cap found no P0/P1; these are the verified P2 follow-ups. config: the max_concurrent_git_ops doc overclaimed that "every capped path is duration-bounded." The rev-list object enumeration in the withheld-blob path still runs in an uncancellable spawn_blocking, so a stuck rev-list can hold its slot until git exits. Scope the guarantee to the streaming stages and name the residual. Also tighten the fairness claim: the receive-pack advertisement shares the read pool (a shed advertisement is a cheap retryable GET); only the push POST is on the isolated write pool. api/repos: add info_refs_per_caller_cap_keys_on_did_not_ip, the missing handler proof that a signed caller is keyed by its DID, not its source IP. Filling the DID slot sheds a request from a free IP; collapsing read_caller_key to its IP arm turns the assertion green-not-503 (mutation-verified RED). api/repos: extract acquire_read_caller_permit so both read handlers share one shed path instead of a duplicated match block. rate_limit: recover from a poisoned PerCallerConcurrency mutex instead of panicking. The critical section is pure counter arithmetic and cannot poison the lock, but a panic there would brick the limiter for every caller. 505 tests pass; clippy -D warnings and fmt clean. Part of #174. --- crates/gitlawb-node/src/api/repos.rs | 127 +++++++++++++++++++++----- crates/gitlawb-node/src/config.rs | 20 ++-- crates/gitlawb-node/src/rate_limit.rs | 9 +- 3 files changed, 122 insertions(+), 34 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index dba8ea71..f739c04b 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -575,18 +575,12 @@ pub async fn git_info_refs( peer, state.push_limiter_trust, ); - let _caller_permit = match &caller_key { - Some(k) => match state.git_read_per_caller.try_acquire(k) { - Some(p) => Some(p), - None => { - tracing::warn!(repo = %name, caller = %k, "per-caller read cap reached; shedding info/refs with 503"); - return Err(AppError::Overloaded( - "git read service at capacity for this caller, retry shortly".into(), - )); - } - }, - None => None, - }; + let _caller_permit = acquire_read_caller_permit( + &state.git_read_per_caller, + caller_key.as_deref(), + name, + "info/refs", + )?; // For receive-pack (push), download the latest from Tigris so the client // sees the same refs that acquire_write() will operate on. @@ -656,6 +650,30 @@ fn read_caller_key( } } +/// Acquire the per-caller read sub-cap permit (#174), or shed with a 503. `key` is +/// `None` when no caller key resolves — that request is bounded by the global read +/// pool only and is never shed here (returns `Ok(None)`). `handler` labels the shed +/// log line. Shared by both read handlers so the two acquire sites cannot drift. +fn acquire_read_caller_permit( + limiter: &crate::rate_limit::PerCallerConcurrency, + key: Option<&str>, + repo: &str, + handler: &str, +) -> Result> { + match key { + Some(k) => match limiter.try_acquire(k) { + Some(p) => Ok(Some(p)), + None => { + tracing::warn!(repo = %repo, caller = %k, handler, "per-caller read cap reached; shedding with 503"); + Err(AppError::Overloaded( + "git read service at capacity for this caller, retry shortly".into(), + )) + } + }, + None => Ok(None), + } +} + /// Map an error from a `smart_http` git service call to the right `AppError`: /// [`smart_http::GitServiceTimeout`] to 504, a malformed client request to 400, /// anything else to a 500 git error. Pure (no logging) so it is unit-testable; @@ -712,18 +730,12 @@ pub async fn git_upload_pack( // visibility-denied caller never consumes a scarce read slot. Keyed per-DID // when signed, else per-source-IP; no resolvable key -> global read pool only. let caller_key = read_caller_key(caller, &headers, peer, state.push_limiter_trust); - let _caller_permit = match &caller_key { - Some(k) => match state.git_read_per_caller.try_acquire(k) { - Some(p) => Some(p), - None => { - tracing::warn!(repo = %name, caller = %k, "per-caller read cap reached; shedding upload-pack with 503"); - return Err(AppError::Overloaded( - "git read service at capacity for this caller, retry shortly".into(), - )); - } - }, - None => None, - }; + let _caller_permit = acquire_read_caller_permit( + &state.git_read_per_caller, + caller_key.as_deref(), + name, + "upload-pack", + )?; let disk_path = state .repo_store @@ -2798,6 +2810,73 @@ mod tests { ); } + /// #174 SC2 (per-DID key): a SIGNED caller is keyed by its DID, not its source + /// IP. Fill the DID's single read slot, then drive a request carrying that + /// `AuthenticatedDid` from an IP whose own slot is free — it must shed 503, + /// proving the key is the DID. A second request from the SAME IP but a DIFFERENT + /// DID is not shed. Collapse `read_caller_key` to its IP arm and the first + /// assertion goes green-not-503 (the IP slot is free) — this is the per-DID half + /// of the keying proof that the per-IP probes above do not cover. + #[sqlx::test] + async fn info_refs_per_caller_cap_keys_on_did_not_ip(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6pcdid", "pc", "/tmp/pc-nonexistent", None, false) + .await + .unwrap(); + + let did_a = "did:key:z6MkPerCallerKeyingProofDidAAAAAAAAAAAAAAAA"; + let did_b = "did:key:z6MkPerCallerKeyingProofDidBBBBBBBBBBBBBBBB"; + let peer: SocketAddr = "203.0.113.51:5000".parse().unwrap(); + + // Fill DID_A's single read slot; the IP's own slot is left untouched. + let _slot = state + .git_read_per_caller + .try_acquire(did_a) + .expect("first slot for this DID"); + + // Signed as DID_A, from `peer` (whose IP slot is free): keyed by DID -> shed. + let router = crate::server::build_router(state.clone()); + let mut req = Request::builder() + .method(Method::GET) + .uri("/z6pcdid/pc/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + req.extensions_mut() + .insert(crate::auth::AuthenticatedDid(did_a.to_string())); + assert_eq!( + router.oneshot(req).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a signed caller at its per-DID read cap must shed with 503 (keyed by DID, not the free IP slot)" + ); + + // Same IP, DIFFERENT DID -> its own budget -> not shed by DID_A's saturation. + let router2 = crate::server::build_router(state.clone()); + let mut req2 = Request::builder() + .method(Method::GET) + .uri("/z6pcdid/pc/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + req2.extensions_mut().insert(ConnectInfo(peer)); + req2.extensions_mut() + .insert(crate::auth::AuthenticatedDid(did_b.to_string())); + assert_ne!( + router2.oneshot(req2).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a different DID from the same IP must not be shed by another DID's saturated budget" + ); + } + /// #174 SC2 (None-key): a request with no resolvable caller key (no ConnectInfo, /// no trusted header) must NOT be shed by the per-caller cap even when another /// caller's budget is full — it is bounded by the global read pool only. A None diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index 93c49449..90a061e0 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -244,19 +244,23 @@ pub struct Config { /// pack-objects + threads). Size below the process budget with headroom. /// /// This is the READ pool (`git_read_semaphore`): upload-pack and both info/refs - /// advertisements. Authenticated pushes draw from a separate write pool - /// (`max_concurrent_git_pushes`) and each caller is additionally bounded by - /// `max_concurrent_reads_per_caller`, so an anonymous flood can neither shed a - /// push nor monopolize reads (#174). + /// advertisements. The authenticated push POST draws from a separate write pool + /// (`max_concurrent_git_pushes`) that anonymous reads can never reach, and each + /// caller is additionally bounded by `max_concurrent_reads_per_caller`, so an + /// anonymous flood cannot shed the actual push nor monopolize reads (#174). (The + /// receive-pack advertisement itself shares the read pool; a shed advertisement + /// is a cheap retryable GET, and the write POST it precedes always has capacity.) /// - /// A permit is held for the whole op, and every capped path is now + /// A permit is held for the whole op. Every git subprocess that STREAMS is /// duration-bounded and reaps its process group on disconnect: upload-pack, /// receive-pack, and both info/refs advertisements run under /// `git_service_timeout_secs` with `process_group(0)` teardown, and the /// withheld-blob (`upload_pack_excluding`) pack-objects stage runs on the async - /// side under the same teardown. So a hung git can no longer pin a slot and a - /// client disconnect no longer orphans its git child (#174 closed the two - /// duration/cancellation gaps this comment previously tracked as follow-up). + /// side under the same teardown (#174 closed the two duration/cancellation gaps + /// this comment previously tracked). The one remaining blocking stage is the + /// `rev-list` object enumeration in the withheld-blob path — a bounded walk that + /// terminates, run off the async runtime; it is not process-group-reaped on + /// disconnect, so a stuck rev-list can still hold its slot until git exits. /// /// Default: 128. Must be between 1 and 1_048_576; the ceiling keeps the value /// well under tokio's `Semaphore` permit limit so an oversized value is a diff --git a/crates/gitlawb-node/src/rate_limit.rs b/crates/gitlawb-node/src/rate_limit.rs index d505f1bb..22281691 100644 --- a/crates/gitlawb-node/src/rate_limit.rs +++ b/crates/gitlawb-node/src/rate_limit.rs @@ -178,7 +178,12 @@ impl PerCallerConcurrency { /// `None` (shed) otherwise. Reject-before-insert: a new key at `max_keys` is /// rejected without allocating. pub fn try_acquire(&self, key: &str) -> Option { - let mut map = self.state.lock().expect("per-caller mutex poisoned"); + // Recover from a poisoned lock rather than panicking: the critical section + // is pure counter arithmetic and cannot itself panic, so a poisoned mutex + // would only ever come from an unrelated abort, and a slightly-off count + // self-heals as permits drop. A panic here would instead brick the limiter + // for every caller (each subsequent lock re-panics). + let mut map = self.state.lock().unwrap_or_else(|e| e.into_inner()); match map.get_mut(key) { Some(count) => { if *count >= self.per_caller { @@ -207,7 +212,7 @@ impl PerCallerConcurrency { impl Drop for PerCallerPermit { fn drop(&mut self) { - let mut map = self.state.lock().expect("per-caller mutex poisoned"); + let mut map = self.state.lock().unwrap_or_else(|e| e.into_inner()); if let Some(count) = map.get_mut(&self.key) { *count -= 1; if *count == 0 { From 3b1fa54b73c2af9280b89042946cb4c7b26892e4 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 10 Jul 2026 16:05:29 -0500 Subject: [PATCH 09/58] docs(config): add GITLAWB_MAX_CONCURRENT_GIT_OPS to .env.example (#174) The read-pool knob was referenced by the push and per-caller entries' comments but had no example line of its own, so operators couldn't discover it from the template. Add it with the config default (128). --- .env.example | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.env.example b/.env.example index 69865920..cbf3db3a 100644 --- a/.env.example +++ b/.env.example @@ -113,6 +113,13 @@ GITLAWB_MAX_PACK_BYTES=2147483648 # advertisement and the withheld-blob pack build (#174). Default 600. GITLAWB_GIT_SERVICE_TIMEOUT_SECS=600 +# Max concurrent git read ops (upload-pack + info/refs advertisements) served at +# once, a global pool separate from the push pool below. Over-cap sheds a clean +# 503 + Retry-After. Anonymous reads draw from here, so pair it with +# GITLAWB_MAX_CONCURRENT_READS_PER_CALLER (below) so one caller cannot monopolize +# the pool. Default 128. +GITLAWB_MAX_CONCURRENT_GIT_OPS=128 + # Max concurrent git-receive-pack (push) operations, in a pool separate from the # read pool (GITLAWB_MAX_CONCURRENT_GIT_OPS) so anonymous reads cannot shed an # authenticated push at admission. Over-cap sheds a 503 + Retry-After. Default 32. From 362ee3462e84015913c3f57012e29a972e6c1d8d Mon Sep 17 00:00:00 2001 From: t Date: Sat, 11 Jul 2026 13:02:24 -0500 Subject: [PATCH 10/58] fix(node,git): bound the rev-list stage under drive_git_child so a hung enumeration can't pin a git permit (#174) --- crates/gitlawb-node/src/git/smart_http.rs | 108 +++++++++++++++------- 1 file changed, 77 insertions(+), 31 deletions(-) diff --git a/crates/gitlawb-node/src/git/smart_http.rs b/crates/gitlawb-node/src/git/smart_http.rs index 5586a2c7..206e7b27 100644 --- a/crates/gitlawb-node/src/git/smart_http.rs +++ b/crates/gitlawb-node/src/git/smart_http.rs @@ -6,7 +6,7 @@ use bytes::Bytes; use std::collections::HashSet; use std::path::Path; use std::process::Stdio; -use std::time::Duration; +use std::time::{Duration, Instant}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::process::Command; @@ -349,28 +349,27 @@ fn pkt_line(data: &str) -> Vec { format!("{len:04x}{data}").into_bytes() } -/// Run `rev-list --objects --all` and return the reachable object ids minus the -/// withheld blobs. Blocking (shells out to git); the caller runs it off the async -/// runtime. Kept separate from the pack-objects stage so that stage can live on the -/// async side under [`drive_git_child`] (`git_bin` is injectable for the same -/// fake-git testing reason as `run_git_service`). -fn rev_list_keep( +/// Run `rev-list --objects --all` under `timeout` and return the reachable object +/// ids minus the withheld blobs. Runs under [`drive_git_child`] (async, with the +/// `tokio::time::timeout` + `process_group(0)` + [`KillGroupOnDrop`] teardown), so a +/// hung/slow enumeration is duration-bounded and reaped on client disconnect just +/// like the pack-objects stage. A bare blocking `Command::output()` inside a +/// `spawn_blocking` is uncancellable, so a hung rev-list would pin the endpoint's +/// concurrency permit for the whole hang (#174). `git_bin` is injectable for the +/// same fake-git testing reason as `run_git_service`. +async fn rev_list_keep( git_bin: &str, repo_path: &Path, withheld: &HashSet, + timeout: Duration, ) -> Result> { - let rev = std::process::Command::new(git_bin) + let mut command = Command::new(git_bin); + command .args(["rev-list", "--objects", "--all"]) - .current_dir(repo_path) - .output()?; - if !rev.status.success() { - bail!( - "git rev-list failed: {}", - String::from_utf8_lossy(&rev.stderr) - ); - } + .current_dir(repo_path); + let stdout = drive_git_child(command, Bytes::new(), timeout, "rev-list").await?; let mut keep = Vec::new(); - for line in String::from_utf8_lossy(&rev.stdout).lines() { + for line in String::from_utf8_lossy(&stdout).lines() { let oid = line.split_whitespace().next().unwrap_or(""); if oid.is_empty() || withheld.contains(oid) { continue; @@ -384,32 +383,41 @@ fn rev_list_keep( /// given blob OIDs. Commits and trees are always included, so SHAs stay intact; /// only the named blobs are dropped. /// -/// The rev-list enumeration runs blocking off the async runtime; the streaming -/// `pack-objects` stage runs under [`drive_git_child`], so a hung/slow pack build is -/// duration-bounded and its process group is reaped on client disconnect. An outer -/// `tokio::time::timeout` around a `spawn_blocking` cannot cancel the blocking -/// thread, so the streaming stage MUST live on the async side (#174, KTD5). +/// Both git stages — the `rev-list` enumeration and the streaming `pack-objects` +/// build — run under [`drive_git_child`] sharing one deadline, so a hung/slow git at +/// either stage is duration-bounded and its process group is reaped on client +/// disconnect. An outer `tokio::time::timeout` around a `spawn_blocking` cannot +/// cancel the blocking thread, so neither stage may live off the async side +/// (#174, KTD5). pub async fn build_filtered_pack( git_bin: &str, repo_path: &Path, withheld: &HashSet, timeout: Duration, ) -> Result> { - let keep = { - let git_bin = git_bin.to_string(); - let repo_path = repo_path.to_path_buf(); - let withheld = withheld.clone(); - tokio::task::spawn_blocking(move || rev_list_keep(&git_bin, &repo_path, &withheld)) - .await - .context("rev-list task panicked")?? - }; + // One deadline spans both git stages so a slow rev-list eats into the pack + // budget rather than granting each stage a fresh `timeout` (2x the permit hold). + let deadline = Instant::now() + timeout; + let keep = rev_list_keep( + git_bin, + repo_path, + withheld, + deadline.saturating_duration_since(Instant::now()), + ) + .await?; let mut data = keep.join("\n").into_bytes(); data.push(b'\n'); let mut command = Command::new(git_bin); command .args(["pack-objects", "--stdout"]) .current_dir(repo_path); - drive_git_child(command, Bytes::from(data), timeout, "pack-objects").await + drive_git_child( + command, + Bytes::from(data), + deadline.saturating_duration_since(Instant::now()), + "pack-objects", + ) + .await } /// Serve a clone/fetch with the withheld blobs removed from the response pack. @@ -1327,6 +1335,44 @@ mod tests { ); } + // #174 (SC3, KTD5): the filtered-pack build's rev-list ENUMERATION stage is now + // duration-bounded on the ASYNC side too. It previously ran inside a + // spawn_blocking via a bare `Command::output()`, which an outer tokio timeout + // cannot cancel, so a hung/slow rev-list pinned the endpoint's concurrency permit + // for the whole hang. A fake git that hangs on rev-list must now abort with + // GitServiceTimeout; the watchdog turns a missing bound into a loud failure. + // rev-list runs under drive_git_child, so its disconnect/group-teardown is the + // same code proven by run_git_service_tears_down_group_when_future_dropped. + #[cfg(unix)] + #[tokio::test] + async fn build_filtered_pack_times_out_a_hung_rev_list() { + let tmp = tempfile::TempDir::new().unwrap(); + // rev-list hangs forever; pack-objects would return fast if it were reached. + let body = "#!/bin/sh\ncase \"$1\" in\n rev-list) sleep 300 ;;\n pack-objects) printf '' ;;\n *) exit 1 ;;\nesac\n"; + let git_bin = write_fake_git(tmp.path(), body); + let withheld = HashSet::new(); + + let result = tokio::time::timeout( + Duration::from_secs(10), + build_filtered_pack( + git_bin.to_str().unwrap(), + tmp.path(), + &withheld, + Duration::from_millis(200), + ), + ) + .await + .expect( + "build_filtered_pack must return within the watchdog — the rev-list \ + stage must be timeout-bounded, not an uncancellable spawn_blocking", + ); + let err = result.expect_err("a hung rev-list must return an error, not hang"); + assert!( + err.downcast_ref::().is_some(), + "a hung rev-list must abort with GitServiceTimeout, got: {err}" + ); + } + // A request that runs to completion must DISARM the guard after reaping, so // no stray group SIGTERM fires. The fake exits non-zero (surfacing as Err) // but leaves a grandchild alive; the grandchild must survive. Goes RED if the From 4983b854d1a25c43354d08f420490555e4bce728 Mon Sep 17 00:00:00 2001 From: t Date: Sat, 11 Jul 2026 23:44:13 -0500 Subject: [PATCH 11/58] fix(node): key the per-caller read cap on source IP, not the signed DID (#174) read_caller_key returned the authenticated DID when a caller signed, dropping the source-IP key. Public read routes accept any valid did:key via optional_signature with no admission step, so one host could mint N disposable DIDs and hold max_concurrent_reads_per_caller slots under each, multiplying its budget N-fold and filling the global read pool, while the same host unsigned was capped on its IP. Key the read sub-cap on the resolved source IP for every caller, signed or not, mirroring the push path's IpRateLimiter which already throttles on source IP for this exact DID-farm reason. Drops the now-unused caller_did parameter at both call sites (git_info_refs, git_upload_pack). Inverts info_refs_per_caller_cap_keys_on_did_not_ip into info_refs_per_caller_cap_keys_on_ip_not_did: fill one source IP's slot, then two requests signed under different DIDs from that same IP both shed 503 (farm defeated), while a signed request from a different IP keeps its own budget. RED on the DID-keyed tree, GREEN after. --- crates/gitlawb-node/src/api/repos.rs | 91 ++++++++++++++++------------ 1 file changed, 53 insertions(+), 38 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index f739c04b..e95910ef 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -569,12 +569,7 @@ pub async fn git_info_refs( // Per-caller read sub-cap (#174): acquired AFTER the visibility + push-rate // gates (KTD7) so a denied or rate-limited request never consumes a caller's // scarce read slot. Applies to BOTH advertisements. Held for the whole op. - let caller_key = read_caller_key( - auth.as_ref().map(|e| e.0 .0.as_str()), - &headers, - peer, - state.push_limiter_trust, - ); + let caller_key = read_caller_key(&headers, peer, state.push_limiter_trust); let _caller_permit = acquire_read_caller_permit( &state.git_read_per_caller, caller_key.as_deref(), @@ -633,21 +628,21 @@ fn git_permit( }) } -/// Resolve the per-caller key for the read sub-cap (#174): the authenticated DID -/// when the caller signed (via `optional_signature`), else the trusted client IP -/// (`client_key`). `None` when neither is available — such a request is bounded by -/// the global read pool only, never a 500. The per-source-IP key is only as -/// granular as `trust`; see the `max_concurrent_reads_per_caller` config doc. +/// Resolve the per-caller key for the read sub-cap (#174): always the resolved +/// source IP (`client_key`), never the signed DID. Public read routes accept any +/// valid `did:key` via `optional_signature` with no admission step, so keying on +/// the DID would let one host mint disposable DIDs to multiply its per-source +/// budget; the push path already throttles on the resolved source IP for exactly +/// this DID-farm reason (`rate_limit.rs`, `IpRateLimiter`). `None` when no key +/// resolves (no trusted header and no peer): such a request is bounded by the +/// global read pool only, never a 500. The per-source-IP key is only as granular +/// as `trust`; see the `max_concurrent_reads_per_caller` config doc. fn read_caller_key( - caller_did: Option<&str>, headers: &axum::http::HeaderMap, peer: Option, trust: crate::rate_limit::TrustedProxy, ) -> Option { - match caller_did { - Some(did) => Some(did.to_string()), - None => crate::rate_limit::client_key(headers, peer, trust), - } + crate::rate_limit::client_key(headers, peer, trust) } /// Acquire the per-caller read sub-cap permit (#174), or shed with a 503. `key` is @@ -727,9 +722,10 @@ pub async fn git_upload_pack( } // Per-caller read sub-cap (#174): after the visibility gate (KTD7) so a - // visibility-denied caller never consumes a scarce read slot. Keyed per-DID - // when signed, else per-source-IP; no resolvable key -> global read pool only. - let caller_key = read_caller_key(caller, &headers, peer, state.push_limiter_trust); + // visibility-denied caller never consumes a scarce read slot. Keyed on the + // resolved source IP (never the signed DID, #174 U1); no resolvable key -> + // global read pool only. + let caller_key = read_caller_key(&headers, peer, state.push_limiter_trust); let _caller_permit = acquire_read_caller_permit( &state.git_read_per_caller, caller_key.as_deref(), @@ -2810,15 +2806,16 @@ mod tests { ); } - /// #174 SC2 (per-DID key): a SIGNED caller is keyed by its DID, not its source - /// IP. Fill the DID's single read slot, then drive a request carrying that - /// `AuthenticatedDid` from an IP whose own slot is free — it must shed 503, - /// proving the key is the DID. A second request from the SAME IP but a DIFFERENT - /// DID is not shed. Collapse `read_caller_key` to its IP arm and the first - /// assertion goes green-not-503 (the IP slot is free) — this is the per-DID half - /// of the keying proof that the per-IP probes above do not cover. + /// #174 SC2 (per-source key, U1): the per-caller read sub-cap keys on the + /// resolved source IP, NOT the signed DID, so a disposable-DID farm cannot + /// multiply its budget. Fill the source IP's single read slot, then drive two + /// requests signed under DIFFERENT DIDs from that SAME IP: both must shed 503 + /// (keyed by the saturated IP, not their own free DID slots). A signed request + /// from a DIFFERENT source IP keeps its own budget. Revert `read_caller_key` to + /// prefer the DID and the same-IP assertions go green-not-503 (each fresh DID + /// gets a free slot) -- the farm-defeat mutation probe. #[sqlx::test] - async fn info_refs_per_caller_cap_keys_on_did_not_ip(pool: sqlx::PgPool) { + async fn info_refs_per_caller_cap_keys_on_ip_not_did(pool: sqlx::PgPool) { use axum::body::Body; use axum::extract::ConnectInfo; use axum::http::{Method, Request, StatusCode}; @@ -2830,7 +2827,7 @@ mod tests { state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; state .db - .upsert_mirror_repo("z6pcdid", "pc", "/tmp/pc-nonexistent", None, false) + .upsert_mirror_repo("z6pcip", "pc", "/tmp/pc-nonexistent", None, false) .await .unwrap(); @@ -2838,17 +2835,17 @@ mod tests { let did_b = "did:key:z6MkPerCallerKeyingProofDidBBBBBBBBBBBBBBBB"; let peer: SocketAddr = "203.0.113.51:5000".parse().unwrap(); - // Fill DID_A's single read slot; the IP's own slot is left untouched. + // Fill the SOURCE IP's single read slot; both DIDs' own slots stay free. let _slot = state .git_read_per_caller - .try_acquire(did_a) - .expect("first slot for this DID"); + .try_acquire(&peer.ip().to_string()) + .expect("first slot for this source IP"); - // Signed as DID_A, from `peer` (whose IP slot is free): keyed by DID -> shed. + // Signed as DID_A from `peer`: keyed by the saturated source IP -> shed 503. let router = crate::server::build_router(state.clone()); let mut req = Request::builder() .method(Method::GET) - .uri("/z6pcdid/pc/info/refs?service=git-upload-pack") + .uri("/z6pcip/pc/info/refs?service=git-upload-pack") .body(Body::empty()) .unwrap(); req.extensions_mut().insert(ConnectInfo(peer)); @@ -2857,23 +2854,41 @@ mod tests { assert_eq!( router.oneshot(req).await.unwrap().status(), StatusCode::SERVICE_UNAVAILABLE, - "a signed caller at its per-DID read cap must shed with 503 (keyed by DID, not the free IP slot)" + "a signed caller must be keyed by its source IP, not its DID: the saturated IP must shed it 503" ); - // Same IP, DIFFERENT DID -> its own budget -> not shed by DID_A's saturation. + // Same IP, a DIFFERENT DID: still keyed by the same saturated IP -> also shed. + // The farm defeat: minting a fresh DID buys no fresh per-source budget. let router2 = crate::server::build_router(state.clone()); let mut req2 = Request::builder() .method(Method::GET) - .uri("/z6pcdid/pc/info/refs?service=git-upload-pack") + .uri("/z6pcip/pc/info/refs?service=git-upload-pack") .body(Body::empty()) .unwrap(); req2.extensions_mut().insert(ConnectInfo(peer)); req2.extensions_mut() .insert(crate::auth::AuthenticatedDid(did_b.to_string())); - assert_ne!( + assert_eq!( router2.oneshot(req2).await.unwrap().status(), StatusCode::SERVICE_UNAVAILABLE, - "a different DID from the same IP must not be shed by another DID's saturated budget" + "a second DID from the same source IP must also shed 503: a DID farm cannot multiply the per-source budget" + ); + + // A signed caller from a DIFFERENT source IP keeps its own budget -> not shed. + let other: SocketAddr = "203.0.113.52:5000".parse().unwrap(); + let router3 = crate::server::build_router(state.clone()); + let mut req3 = Request::builder() + .method(Method::GET) + .uri("/z6pcip/pc/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + req3.extensions_mut().insert(ConnectInfo(other)); + req3.extensions_mut() + .insert(crate::auth::AuthenticatedDid(did_a.to_string())); + assert_ne!( + router3.oneshot(req3).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a signed caller from a different source IP must keep its own per-source budget" ); } From e444bc0ae5f3813c93c7e57b147c6ff974f5f497 Mon Sep 17 00:00:00 2001 From: t Date: Sat, 11 Jul 2026 23:54:21 -0500 Subject: [PATCH 12/58] fix(node): draw the receive-pack advertisement from the write pool (#174) git_info_refs acquired git_read_semaphore for BOTH services, so the push handshake (GET /info/refs?service=git-receive-pack) competed in the global read pool. An anonymous clone flood could exhaust that pool and shed a legitimate push with 503 during its required advertisement phase, before it ever reached git_write_semaphore on the POST. The write pool exists precisely so anonymous reads cannot shed an authenticated push, but only the POST drew from it. Select the pool by service: the receive-pack advertisement (phase one of a push) now draws from git_write_semaphore, like the git-receive-pack POST, so a saturated read pool cannot starve it. The per-IP push_rate_limiter that already brakes the advertisement stays as the anti-flood control, and the advertisement stays reader-visible with no new auth requirement. Because the receive-pack branch is now a write-path op, it no longer consumes a read per-caller slot. Handler-layer proofs: with the read pool at zero the receive-pack advertisement survives while the upload-pack advertisement sheds; with the write pool at zero the receive-pack advertisement sheds while upload-pack is unaffected; and a receive-pack advertisement from an IP whose read per-caller budget is full still gets through (mutation-checked, RED when the skip is neutralized). --- crates/gitlawb-node/src/api/repos.rs | 164 ++++++++++++++++++++++++--- 1 file changed, 148 insertions(+), 16 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index e95910ef..3be94bfc 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -507,10 +507,21 @@ pub async fn git_info_refs( headers: axum::http::HeaderMap, auth: Option>, ) -> Result { - // Shed with a 503 before spawning git when the concurrency cap is saturated; - // held for the whole op (incl. the smart_http call), released on return. - let _permit = git_permit(&state.git_read_semaphore)?; let name = smart_http_repo_name(&repo)?; + let service = query + .service + .ok_or_else(|| AppError::BadRequest("missing ?service= parameter".into()))?; + // Shed with a 503 before spawning git when the concurrency cap is saturated; + // held for the whole op (incl. the smart_http call), released on return. The + // receive-pack advertisement is phase one of a push, so it draws from the WRITE + // pool (like the git-receive-pack POST), not the global read pool an anonymous + // clone flood can exhaust and thereby starve the push handshake (#174 U2). The + // upload-pack advertisement stays on the read pool with its per-caller sub-cap. + let _permit = if service == "git-receive-pack" { + git_permit(&state.git_write_semaphore)? + } else { + git_permit(&state.git_read_semaphore)? + }; tracing::info!(owner = %owner, repo = %name, "info/refs request"); let record = state .db @@ -524,9 +535,6 @@ pub async fn git_info_refs( return Err(AppError::RepoNotFound(format!("{owner}/{name}"))); } - let service = query - .service - .ok_or_else(|| AppError::BadRequest("missing ?service= parameter".into()))?; tracing::debug!(service = %service, repo = %name, "info/refs service"); // Enforce read visibility on the ref advertisement, for BOTH services. The @@ -566,16 +574,23 @@ pub async fn git_info_refs( } } - // Per-caller read sub-cap (#174): acquired AFTER the visibility + push-rate - // gates (KTD7) so a denied or rate-limited request never consumes a caller's - // scarce read slot. Applies to BOTH advertisements. Held for the whole op. - let caller_key = read_caller_key(&headers, peer, state.push_limiter_trust); - let _caller_permit = acquire_read_caller_permit( - &state.git_read_per_caller, - caller_key.as_deref(), - name, - "info/refs", - )?; + // Per-caller read sub-cap (#174): upload-pack advertisement only. The + // receive-pack advertisement is a write-path op (it drew from git_write_semaphore + // above and is braked per-IP by push_rate_limiter), so it must NOT consume a + // read caller's scarce slot (#174 U2). Acquired AFTER the visibility + push-rate + // gates (KTD7) so a denied or rate-limited request never consumes a slot. Held + // for the whole op. + let _caller_permit = if service == "git-receive-pack" { + None + } else { + let caller_key = read_caller_key(&headers, peer, state.push_limiter_trust); + acquire_read_caller_permit( + &state.git_read_per_caller, + caller_key.as_deref(), + name, + "info/refs", + )? + }; // For receive-pack (push), download the latest from Tigris so the client // sees the same refs that acquire_write() will operate on. @@ -2693,6 +2708,123 @@ mod tests { ); } + /// #174 U2: the receive-pack advertisement (`GET info/refs?service=git-receive-pack`) + /// is phase one of a push, so it draws from the WRITE pool, not the global read + /// pool an anonymous clone flood can exhaust. Proven at the handler by saturating + /// each pool to zero and checking who shares it (INV-10). Revert the branch to the + /// read pool and the read-saturated receive-pack assertion goes 503 (the exact + /// push-handshake starvation jatmn flagged). + #[sqlx::test] + async fn receive_pack_advertisement_draws_from_write_pool(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + // Build a fresh state with the two pools sized independently, then drive one + // info/refs advertisement for `service` and return its handler status. + async fn advert_status( + pool: &sqlx::PgPool, + read_permits: usize, + write_permits: usize, + service: &str, + ) -> StatusCode { + let mut state = crate::test_support::test_state(pool.clone()).await; + state.git_read_semaphore = Arc::new(Semaphore::new(read_permits)); + state.git_write_semaphore = Arc::new(Semaphore::new(write_permits)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6wpadv", "wp", "/tmp/wp-nonexistent", None, false) + .await + .unwrap(); + let peer: SocketAddr = "203.0.113.61:6000".parse().unwrap(); + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/z6wpadv/wp/info/refs?service={service}")) + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + router.oneshot(req).await.unwrap().status() + } + + // Read pool saturated, write pool free: the push handshake SURVIVES. + assert_ne!( + advert_status(&pool, 0, 8, "git-receive-pack").await, + StatusCode::SERVICE_UNAVAILABLE, + "receive-pack advertisement must draw from the write pool: a saturated read pool must not shed it" + ); + // Write pool saturated, read pool free: the receive-pack advertisement SHEDS, + // proving it now consumes the write pool (not the read pool). + assert_eq!( + advert_status(&pool, 8, 0, "git-receive-pack").await, + StatusCode::SERVICE_UNAVAILABLE, + "receive-pack advertisement draws from the write pool, so a saturated write pool sheds it 503" + ); + // Read pool saturated: the upload-pack advertisement still SHEDS (unchanged). + assert_eq!( + advert_status(&pool, 0, 8, "git-upload-pack").await, + StatusCode::SERVICE_UNAVAILABLE, + "upload-pack advertisement stays on the read pool: a saturated read pool sheds it 503" + ); + // Write pool saturated, read pool free: the upload-pack advertisement is + // UNAFFECTED, proving reads never touch the write pool in either direction. + assert_ne!( + advert_status(&pool, 8, 0, "git-upload-pack").await, + StatusCode::SERVICE_UNAVAILABLE, + "upload-pack advertisement never touches the write pool" + ); + } + + /// #174 U2: the receive-pack advertisement is a write-path op, so it must not be + /// shed by the READ per-caller sub-cap even when the caller's source IP has + /// exhausted its read budget (e.g. concurrent clones from the same host). Fill + /// the IP's read per-caller slot, then the receive-pack advertisement from that + /// same IP must still get through. Restore the unconditional read-cap acquire on + /// the receive-pack branch and this goes 503. + #[sqlx::test] + async fn receive_pack_advertisement_ignores_read_per_caller_cap(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6wpc", "wp", "/tmp/wp-nonexistent", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.71:6000".parse().unwrap(); + // Exhaust the source IP's single READ per-caller slot, as concurrent clones + // from the same host would. + let _slot = state + .git_read_per_caller + .try_acquire(&peer.ip().to_string()) + .expect("fill the IP's read per-caller slot"); + + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::GET) + .uri("/z6wpc/wp/info/refs?service=git-receive-pack") + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + assert_ne!( + router.oneshot(req).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "receive-pack advertisement must not be shed by the read per-caller cap: it is a write-path op" + ); + } + /// #174 SC2 (info_refs probe): the per-caller read sub-cap sheds a caller that /// is already at its concurrency budget on the upload-pack advertisement, while /// a DIFFERENT caller still enters. Remove the sub-cap from `git_info_refs` and From 71389e621a233a30c97f959b9d5ccc5c9f1bd57b Mon Sep 17 00:00:00 2001 From: t Date: Sun, 12 Jul 2026 00:14:39 -0500 Subject: [PATCH 13/58] fix(node): bound the withheld-blob walk at the shared blob_paths seam (#174) The withheld-blob classification walk (blob_paths) fanned out blocking git children with no deadline and no process-group teardown: git for-each-ref, git cat-file, git rev-list, a git ls-tree per commit, and an uncounted git rev-parse (via store::head_commit). A hung or pathologically slow child pinned the caller's served-git permit for the whole hang, and on client disconnect the spawn_blocking task and its git children ran on, orphaned. blob_paths is the shared core of five callers: the upload-pack serve path (holds a read permit) AND, inside git_receive_pack, the post-push replication and encrypt-then-pin walks (hold the write permit U2 reserves for pushes). So the same unbounded walk could pin either pool, and leaving the write-side twin unbounded would have made U2's reservation a claim that does not match behavior. Bound every git child at the blob_paths spawn seam on the blocking side: each child runs in its own process group with a watchdog thread that SIGTERMs (then SIGKILLs) the group on one shared deadline spanning the whole walk, and retains admission until the group is reaped. This is the blocking-side counterpart of smart_http::drive_git_child (spawn_blocking cannot be cancelled by an async timeout). blob_paths stays sync, so all five callers keep their signatures and the 32 classification tests are unchanged; because every caller funnels through blob_paths, one seam bounds both the serve and replication paths. The previously unbounded store::head_commit child becomes a bounded git rev-parse inside the walk. A walk that hits its deadline carries GitServiceTimeout, which the serve handler now maps to 504 rather than a generic 500. Proof: a fake git that hangs on rev-list makes blob_paths return GitServiceTimeout within the watchdog budget (not block on the child) and the recorded process-group leader is reaped, not orphaned; neutralizing the watchdog kill makes it hang past the budget (RED). The 32 real-git classification tests stay green through the refactor, including detached-HEAD, non-standard-ref, and deleted-in-history cases. --- crates/gitlawb-node/src/api/repos.rs | 4 +- .../gitlawb-node/src/git/visibility_pack.rs | 333 +++++++++++++----- 2 files changed, 242 insertions(+), 95 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 3be94bfc..7a8aa5c2 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -781,7 +781,9 @@ pub async fn git_upload_pack( }) .await .map_err(|e| AppError::Git(e.to_string()))? - .map_err(|e| AppError::Git(e.to_string()))? + // A walk that hit its deadline carries GitServiceTimeout; map it to 504 + // like the smart_http paths, not a generic 500 (#174 U3). + .map_err(|e| git_service_app_error(&e))? }; if withheld.is_empty() { diff --git a/crates/gitlawb-node/src/git/visibility_pack.rs b/crates/gitlawb-node/src/git/visibility_pack.rs index cb70e39c..7b9bda69 100644 --- a/crates/gitlawb-node/src/git/visibility_pack.rs +++ b/crates/gitlawb-node/src/git/visibility_pack.rs @@ -4,11 +4,122 @@ //! content is held back. use crate::db::VisibilityRule; -use crate::git::store; use crate::visibility::{visibility_check, Decision}; use anyhow::{Context, Result}; use std::collections::{BTreeSet, HashMap, HashSet}; use std::path::Path; +use std::process::Stdio; +use std::sync::mpsc; +use std::time::{Duration, Instant}; + +/// Fixed budget bounding the whole withheld-blob classification walk (#174 U3). +/// The walk is fast for a real repo; this bound exists to reap a hung or +/// pathologically slow git child so it cannot pin a served-git permit (the read +/// permit on the upload-pack serve path, the write permit on the receive-pack +/// post-push replication path) past the deadline. Every caller funnels through +/// `blob_paths`, so bounding here bounds both paths at one seam. +const WALK_TIMEOUT: Duration = Duration::from_secs(600); + +/// Run one git child under a shared `deadline` with process-group teardown, +/// BLOCKING, and return its stdout. The child runs in its own process group; a +/// watchdog thread SIGTERMs (lets git clean up its `*.lock` files), then SIGKILLs, +/// the whole group if the deadline passes before the child is reaped, so a hung or +/// slow git can pin neither a served-git permit nor a blocking thread past the +/// deadline (jatmn's "retain admission until they are reaped"). This is the +/// blocking-side counterpart of `smart_http::drive_git_child`, needed because the +/// walk's callers run it inside `spawn_blocking`, which an async timeout cannot +/// cancel. Returns [`crate::git::smart_http::GitServiceTimeout`] on the deadline so +/// the serve handler maps it to 504. `git_bin` is injectable so a fake `git` can +/// drive the teardown in tests without mutating the process-global PATH; +/// `stdin_bytes` feeds children that read stdin (empty for the arg-only children). +#[cfg(unix)] +fn run_bounded_git( + git_bin: &str, + args: &[&str], + repo_path: &Path, + stdin_bytes: &[u8], + deadline: Instant, +) -> Result> { + use std::io::{Read, Write}; + use std::os::unix::process::CommandExt; + use std::sync::mpsc::RecvTimeoutError; + + let label = args.first().copied().unwrap_or("git"); + let mut child = std::process::Command::new(git_bin) + .args(args) + .current_dir(repo_path) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .process_group(0) + .spawn() + .with_context(|| format!("failed to spawn git {label}"))?; + // With process_group(0) the child leads its own group, so pgid == its pid. + let pgid = child.id() as i32; + + // Watchdog: on the deadline, SIGTERM the whole group, poll until it exits, + // escalate to SIGKILL after a grace, and report whether it killed. Signalled to + // stand down when the child is reaped in time. Kept off the main thread because + // the main thread's stdout drain is exactly what blocks until a hung child is + // torn down. + let (done_tx, done_rx) = mpsc::channel::<()>(); + let watchdog = std::thread::spawn(move || -> bool { + let wait = deadline.saturating_duration_since(Instant::now()); + match done_rx.recv_timeout(wait) { + Ok(()) | Err(RecvTimeoutError::Disconnected) => false, + Err(RecvTimeoutError::Timeout) => { + // SAFETY: kill(2) takes only integers and borrows no Rust memory; + // ESRCH on an already-gone group is ignored. + unsafe { libc::kill(-pgid, libc::SIGTERM) }; + for step in 0..200u32 { + if unsafe { libc::kill(-pgid, 0) } != 0 { + return true; // ESRCH: every group member has exited + } + if step == 100 { + unsafe { libc::kill(-pgid, libc::SIGKILL) }; + } + std::thread::sleep(Duration::from_millis(10)); + } + true + } + } + }); + + // Feed stdin on a writer thread and drain stderr on a reader thread so the main + // thread can drain stdout concurrently; writing all of stdin (or draining one + // pipe) before the others can deadlock once a pipe buffer fills. + let mut stdin = child.stdin.take(); + let input = stdin_bytes.to_vec(); + let writer = std::thread::spawn(move || { + if let Some(mut s) = stdin.take() { + let _ = s.write_all(&input); + } + }); + let mut stderr = child.stderr.take().context("git stderr was not piped")?; + let err_reader = std::thread::spawn(move || { + let mut err = Vec::new(); + let _ = stderr.read_to_end(&mut err); + err + }); + let mut stdout = child.stdout.take().context("git stdout was not piped")?; + let mut out = Vec::new(); + let read_result = stdout.read_to_end(&mut out); + let status = child.wait().context("git wait failed")?; + let err = err_reader.join().unwrap_or_default(); + let _ = writer.join(); + + // Reaped: stand the watchdog down and learn whether it killed on the deadline. + let _ = done_tx.send(()); + let killed = watchdog.join().unwrap_or(false); + if killed { + return Err(crate::git::smart_http::GitServiceTimeout.into()); + } + read_result.context("failed to read git stdout")?; + if !status.success() { + anyhow::bail!("git {label} failed: {}", String::from_utf8_lossy(&err)); + } + Ok(out) +} /// Fail closed unless every ref ultimately resolves to a commit (a ref pointing /// directly at a blob or tree, or an annotated tag — even a nested one — of such @@ -21,19 +132,15 @@ use std::path::Path; /// Full peeling is why this is not `for-each-ref %(*objecttype)`, which /// dereferences only one tag level and so misclassifies a tag-of-a-tag-of-a- /// commit as a non-commit. -fn assert_all_refs_are_commits(repo_path: &Path) -> Result<()> { - let refs = std::process::Command::new("git") - .args(["for-each-ref", "--format=%(refname)"]) - .current_dir(repo_path) - .output() - .context("git for-each-ref failed")?; - if !refs.status.success() { - anyhow::bail!( - "git for-each-ref failed: {}", - String::from_utf8_lossy(&refs.stderr) - ); - } - let refs_stdout = String::from_utf8_lossy(&refs.stdout); +fn assert_all_refs_are_commits(repo_path: &Path, git_bin: &str, deadline: Instant) -> Result<()> { + let refs_out = run_bounded_git( + git_bin, + &["for-each-ref", "--format=%(refname)"], + repo_path, + b"", + deadline, + )?; + let refs_stdout = String::from_utf8_lossy(&refs_out); let refnames: Vec<&str> = refs_stdout .lines() .map(str::trim) @@ -43,59 +150,25 @@ fn assert_all_refs_are_commits(repo_path: &Path) -> Result<()> { return Ok(()); } - // Peel every ref in one `git cat-file --batch-check` pass: one - // `^{}` query per line, one output line per input line, in order. - // The stdin write runs on a separate thread so this thread can drain stdout - // concurrently. cat-file echoes the full query on a ` missing` line, - // so output scales with refname length (not a fixed size per ref); writing - // all of stdin before reading any stdout would deadlock both pipes once the - // child's stdout buffer fills. Dropping `stdin` at the end of the closure - // sends EOF. + // Peel every ref in one `git cat-file --batch-check` pass: one `^{}` + // query per line, one output line per input line, in order. cat-file echoes the + // full query on a ` missing` line, so output scales with refname length; + // run_bounded_git drains stdout concurrently with the stdin write, so the pipe + // cannot deadlock, and the whole peel is bounded by the shared walk deadline. let queries = refnames .iter() .map(|r| format!("{r}^{{}}")) .collect::>() .join("\n"); - use std::io::Write; - let mut child = std::process::Command::new("git") - .args(["cat-file", "--batch-check=%(objecttype)"]) - .current_dir(repo_path) - .stdin(std::process::Stdio::piped()) - .stdout(std::process::Stdio::piped()) - .stderr(std::process::Stdio::piped()) - .spawn() - .context("failed to spawn git cat-file")?; - // Feed stdin on a writer thread so this thread can drain stdout via - // wait_with_output concurrently; a None handle (the pipe vanished) becomes a - // broken-pipe write error. wait_with_output reaps the child unconditionally - // before any error is surfaced, so no path drops it unwaited (#53), and the - // writer is joined only after the drain so the join cannot deadlock. - let writer = child - .stdin - .take() - .map(|mut stdin| std::thread::spawn(move || stdin.write_all(queries.as_bytes()))); - let peel_result = child.wait_with_output(); - let write_result = match writer { - Some(handle) => handle - .join() - .map_err(|_| anyhow::anyhow!("git cat-file stdin writer thread panicked"))?, - None => Err(std::io::Error::new( - std::io::ErrorKind::BrokenPipe, - "git cat-file stdin unavailable", - )), - }; - // Surface a write error only if the process didn't already fail with a - // clearer status. - let peel = peel_result.context("git cat-file failed")?; - if !peel.status.success() { - anyhow::bail!( - "git cat-file --batch-check failed: {}", - String::from_utf8_lossy(&peel.stderr) - ); - } - write_result.context("failed to write to git cat-file stdin")?; - - let peel_stdout = String::from_utf8_lossy(&peel.stdout); + let peel_out = run_bounded_git( + git_bin, + &["cat-file", "--batch-check=%(objecttype)"], + repo_path, + queries.as_bytes(), + deadline, + )?; + + let peel_stdout = String::from_utf8_lossy(&peel_out); let types: Vec<&str> = peel_stdout.lines().map(str::trim).collect(); // A short read means at least one ref went unclassified — fail closed. if types.len() != refnames.len() { @@ -147,47 +220,46 @@ fn assert_all_refs_are_commits(repo_path: &Path) -> Result<()> { /// Fails closed: if commit enumeration or any tree walk fails, returns an error so /// the caller aborts the serve/pin rather than producing a partial (under-withheld) /// set. -fn blob_paths(repo_path: &Path) -> Result> { - assert_all_refs_are_commits(repo_path)?; +fn blob_paths(repo_path: &Path, git_bin: &str, timeout: Duration) -> Result> { + // One deadline spans the whole walk (the ref check, the HEAD probe, rev-list, + // and every per-commit ls-tree), so a slow or hung walk is bounded as a unit + // rather than granting each git child a fresh timeout. + let deadline = Instant::now() + timeout; + assert_all_refs_are_commits(repo_path, git_bin, deadline)?; // Enumerate every reachable commit, not just ref tips. `--all` walks all refs; // append HEAD so a detached HEAD (reachable by rev-list/upload-pack but in no // ref) is still classified. When HEAD does not resolve (unborn branch on an - // empty repo) `--all` alone yields nothing, which is correct — no objects exist. - let head = store::head_commit(repo_path).context("resolve HEAD failed")?; + // empty repo) `--all` alone yields nothing, which is correct: no objects exist. + // The HEAD probe is a bounded `git rev-parse --verify HEAD` (a clean exit means + // HEAD resolves), replacing the previously unbounded `store::head_commit` child. + let head_resolves = run_bounded_git( + git_bin, + &["rev-parse", "--verify", "HEAD"], + repo_path, + b"", + deadline, + ) + .is_ok(); let mut rev_args = vec!["rev-list", "--all"]; - if head.is_some() { + if head_resolves { rev_args.push("HEAD"); } - let commits = std::process::Command::new("git") - .args(&rev_args) - .current_dir(repo_path) - .output() - .context("git rev-list --all failed")?; - if !commits.status.success() { - anyhow::bail!( - "git rev-list --all failed: {}", - String::from_utf8_lossy(&commits.stderr) - ); - } - let commits_stdout = String::from_utf8_lossy(&commits.stdout); + let commits_out = run_bounded_git(git_bin, &rev_args, repo_path, b"", deadline)?; + let commits_stdout = String::from_utf8_lossy(&commits_out); let mut out: HashSet<(String, String)> = HashSet::new(); for commit in commits_stdout.lines() { let commit = commit.trim(); if commit.is_empty() { continue; } - let listing = std::process::Command::new("git") - .args(["ls-tree", "-rz", commit]) - .current_dir(repo_path) - .output() - .context("git ls-tree -rz failed")?; - if !listing.status.success() { - anyhow::bail!( - "git ls-tree -rz {commit} failed: {}", - String::from_utf8_lossy(&listing.stderr) - ); - } + let listing_out = run_bounded_git( + git_bin, + &["ls-tree", "-rz", commit], + repo_path, + b"", + deadline, + )?; // `-z` NUL-delimits records and emits paths raw; plain `git ls-tree -r` // C-quotes any path with non-ASCII or special bytes (e.g. café.txt becomes // "secret/caf\303\251.txt"), and that quoted literal would not match a @@ -198,7 +270,7 @@ fn blob_paths(repo_path: &Path) -> Result> { // path (e.g. a non-UTF-8 directory name) with U+FFFD, and the mangled string // would no longer match its deny rule — the same under-withholding class, one // layer down. Fail closed instead so the caller aborts rather than leaks. - let Ok(listing_stdout) = std::str::from_utf8(&listing.stdout) else { + let Ok(listing_stdout) = std::str::from_utf8(&listing_out) else { anyhow::bail!( "git ls-tree -rz {commit} returned a non-UTF-8 path; \ refusing to produce a partial (under-withheld) set" @@ -236,7 +308,7 @@ pub fn withheld_blob_oids( owner_did: &str, caller: Option<&str>, ) -> Result> { - let pairs = blob_paths(repo_path)?; + let pairs = blob_paths(repo_path, "git", WALK_TIMEOUT)?; Ok(withheld_from_pairs( &pairs, rules, is_public, owner_did, caller, )) @@ -332,7 +404,7 @@ pub fn allowed_blob_set_for_caller( owner_did: &str, caller: Option<&str>, ) -> Result> { - let pairs = blob_paths(repo_path)?; + let pairs = blob_paths(repo_path, "git", WALK_TIMEOUT)?; let mut allowed = HashSet::new(); for (oid, path) in &pairs { if visibility_check(rules, is_public, owner_did, caller, path) == Decision::Allow { @@ -372,7 +444,7 @@ pub fn withheld_blob_recipients( owner_did: &str, ) -> Result>> { // One history walk feeds both the withheld set and the recipient mapping. - let pairs = blob_paths(repo_path)?; + let pairs = blob_paths(repo_path, "git", WALK_TIMEOUT)?; let withheld = withheld_from_pairs(&pairs, rules, is_public, owner_did, None); if withheld.is_empty() { return Ok(HashMap::new()); @@ -402,6 +474,79 @@ pub fn withheld_blob_recipients( #[cfg(test)] mod tests { use super::*; + + /// Write an executable fake `git` shell script into `dir` and return its path, + /// so a test can drive the walk's process-group teardown without a real git and + /// without mutating the process-global PATH (the crate's only injection seam). + #[cfg(unix)] + fn write_fake_git(dir: &Path, body: &str) -> String { + use std::os::unix::fs::PermissionsExt; + let p = dir.join("fakegit"); + std::fs::write(&p, body).unwrap(); + let mut perm = std::fs::metadata(&p).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&p, perm).unwrap(); + p.to_str().unwrap().to_string() + } + + /// #174 U3: the withheld-blob walk is bounded at the shared `blob_paths` seam, so + /// a hung git child cannot pin the caller's permit past the deadline. A fake git + /// that hangs on `rev-list` must make `blob_paths` return `GitServiceTimeout` + /// within the watchdog budget (not block for the child's lifetime), and the + /// child's process group must be reaped (its recorded leader PID gone). Every + /// caller (upload-pack serve, receive-pack replication) funnels through + /// `blob_paths`, so this seam-level proof covers both permit pools. Neutralize + /// the watchdog SIGTERM and this hangs past the recv budget (RED). + #[cfg(unix)] + #[test] + fn blob_paths_times_out_and_reaps_a_hung_walk() { + use std::time::Duration; + let tmp = TempDir::new().unwrap(); + // Fast on every stage except rev-list, which records its own (group-leader) + // PID and then hangs. `sleep 30` bounds the worst case if the watchdog is + // ever broken, so a regression cannot wedge the suite for 300s. + let body = "#!/bin/sh\ncase \"$1\" in\n rev-list) echo $$ > revlist.pid ; sleep 30 ;;\n rev-parse) echo deadbeef ;;\n *) : ;;\nesac\nexit 0\n"; + let git_bin = write_fake_git(tmp.path(), body); + + // Run the walk on a thread with a short budget; the recv_timeout succeeding + // is itself proof the walk did not block on the hung child. + let (tx, rx) = mpsc::channel(); + let path = tmp.path().to_path_buf(); + std::thread::spawn(move || { + let _ = tx.send(blob_paths(&path, &git_bin, Duration::from_millis(200))); + }); + let result = rx.recv_timeout(Duration::from_secs(10)).expect( + "blob_paths must return within the watchdog budget, not hang on a stuck git child", + ); + let err = result.expect_err("a hung rev-list must abort the walk with an error"); + assert!( + err.downcast_ref::() + .is_some(), + "a hung walk must abort with GitServiceTimeout (mapped to 504), got: {err}" + ); + + // The recorded process-group leader must be gone: the watchdog reaps the + // whole group before blob_paths returns, so no orphaned git lingers. + let pid: i32 = std::fs::read_to_string(tmp.path().join("revlist.pid")) + .expect("the fake git must have recorded its rev-list PID") + .trim() + .parse() + .expect("recorded PID must parse"); + let mut gone = false; + for _ in 0..200 { + // SAFETY: kill(2) with signal 0 only probes existence; ESRCH (-1) means + // the process is gone. Borrows no Rust memory. + if unsafe { libc::kill(pid, 0) } != 0 { + gone = true; + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + assert!( + gone, + "the hung git child (pid {pid}) must be reaped, not orphaned, after the walk aborts" + ); + } use crate::db::VisibilityMode; use chrono::Utc; use std::process::Command; From ab0ea240fcca3087bb7e3c08302885aede32627a Mon Sep 17 00:00:00 2001 From: t Date: Sun, 12 Jul 2026 00:20:07 -0500 Subject: [PATCH 14/58] docs(node): correct the git-service-timeout coverage note (#174) GITLAWB_GIT_SERVICE_TIMEOUT_SECS bounds the info/refs advertisement too: smart_http::info_refs drives it through drive_git_child under this timeout, with a passing test proving the 504. The old note claimed it does not. It also claimed the withheld-blob path is unbounded; after the blob_paths seam bound (this PR) the walk is bounded and reaped, by a fixed internal deadline rather than this env var, so the line now states that precisely instead of overclaiming this setting covers it. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 57ce2885..d02c4a0b 100644 --- a/README.md +++ b/README.md @@ -343,7 +343,7 @@ Important node settings: | `GITLAWB_REQUIRE_SIGNED_PEER_WRITES` | Require signed peer announce/sync writes. | | `GITLAWB_AUTO_SYNC` | Enable automatic sync from known peers. | | `GITLAWB_MAX_PACK_BYTES` | Max git pack body size for smart-HTTP routes. | -| `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack/receive-pack may run before it is aborted (504). Default 600. Does not bound `info/refs` or the withheld-blob path. | +| `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack, receive-pack, or `info/refs` advertisement may run before it is aborted (504). Default 600. The withheld-blob classification walk is also bounded and reaped, by a fixed internal deadline rather than this setting. | | `GITLAWB_TIGRIS_BUCKET` | Optional S3/Tigris shared repo storage bucket. | | `GITLAWB_PINATA_JWT` | Optional Pinata/IPFS warm-storage pinning. | | `GITLAWB_IRYS_URL` | Optional Irys/Arweave permanent anchoring. | From 87a7e40513b018b335d8f3a2e46084d323484798 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 12 Jul 2026 00:36:42 -0500 Subject: [PATCH 15/58] fix(review): close the withheld-walk watchdog reused-pgid race (#174) Code review found run_bounded_git's watchdog could return a spurious 504 and signal a recycled process group. The watchdog runs off a wall clock on its own thread; done_tx.send() only fires after child.wait() reaps the leader, so a walk that finished within microseconds of the deadline took the watchdog's Timeout branch, discarded a fully-captured successful result, and returned GitServiceTimeout (a 504 for a walk that actually completed). Worse, the Timeout branch SIGTERMed -pgid unconditionally after the leader was reaped, so a recycled pgid could be signalled, the exact hazard smart_http guards via disarm-after-wait. Set a reaped AtomicBool the instant the main thread reaps the child; the watchdog checks it before every kill and stands down if the leader is already reaped. Gate the timeout verdict on !status.success(), so a child that exited on its own is never reported as a timeout even if the watchdog fired late. Add the survived-SIGKILL warn smart_http's reap already emits, for operator visibility on a wedged (D-state) git. The hung-walk test stays green (a killed child exits by signal, not success, so it still surfaces GitServiceTimeout and reaps the group) and the 32 real-git classification tests stay green (a fast walk is never spuriously killed). --- .../gitlawb-node/src/git/visibility_pack.rs | 66 +++++++++++++------ 1 file changed, 45 insertions(+), 21 deletions(-) diff --git a/crates/gitlawb-node/src/git/visibility_pack.rs b/crates/gitlawb-node/src/git/visibility_pack.rs index 7b9bda69..0d6661e4 100644 --- a/crates/gitlawb-node/src/git/visibility_pack.rs +++ b/crates/gitlawb-node/src/git/visibility_pack.rs @@ -63,27 +63,46 @@ fn run_bounded_git( // the main thread's stdout drain is exactly what blocks until a hung child is // torn down. let (done_tx, done_rx) = mpsc::channel::<()>(); - let watchdog = std::thread::spawn(move || -> bool { - let wait = deadline.saturating_duration_since(Instant::now()); - match done_rx.recv_timeout(wait) { - Ok(()) | Err(RecvTimeoutError::Disconnected) => false, - Err(RecvTimeoutError::Timeout) => { - // SAFETY: kill(2) takes only integers and borrows no Rust memory; - // ESRCH on an already-gone group is ignored. - unsafe { libc::kill(-pgid, libc::SIGTERM) }; - for step in 0..200u32 { - if unsafe { libc::kill(-pgid, 0) } != 0 { - return true; // ESRCH: every group member has exited + // Set true the instant the main thread reaps the child, so the watchdog never + // SIGTERMs a pgid whose leader is already reaped and whose pid the kernel may + // recycle (the reused-pgid hazard smart_http guards via disarm-after-wait). + let reaped = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let watchdog = { + let reaped = reaped.clone(); + std::thread::spawn(move || -> bool { + use std::sync::atomic::Ordering; + let wait = deadline.saturating_duration_since(Instant::now()); + match done_rx.recv_timeout(wait) { + Ok(()) | Err(RecvTimeoutError::Disconnected) => false, + Err(RecvTimeoutError::Timeout) => { + // The child was reaped right as the deadline fired: do not signal + // its (possibly-recycled) pgid. + if reaped.load(Ordering::SeqCst) { + return false; } - if step == 100 { - unsafe { libc::kill(-pgid, libc::SIGKILL) }; + // SAFETY: kill(2) takes only integers and borrows no Rust memory; + // ESRCH on an already-gone group is ignored. + unsafe { libc::kill(-pgid, libc::SIGTERM) }; + for step in 0..200u32 { + if reaped.load(Ordering::SeqCst) || unsafe { libc::kill(-pgid, 0) } != 0 { + return true; // reaped, or ESRCH: every group member has exited + } + if step == 100 { + unsafe { libc::kill(-pgid, libc::SIGKILL) }; + } + std::thread::sleep(Duration::from_millis(10)); } - std::thread::sleep(Duration::from_millis(10)); + // Survived SIGKILL past the cap: a wedged (D-state) git, the same + // condition smart_http's reap logs. Warn so an operator sees it. + tracing::warn!( + pgid, + "withheld-walk git survived SIGKILL past the watchdog cap (uninterruptible I/O?)" + ); + true } - true } - } - }); + }) + }; // Feed stdin on a writer thread and drain stderr on a reader thread so the main // thread can drain stdout concurrently; writing all of stdin (or draining one @@ -105,16 +124,21 @@ fn run_bounded_git( let mut out = Vec::new(); let read_result = stdout.read_to_end(&mut out); let status = child.wait().context("git wait failed")?; + // Reaped: bar the watchdog from signalling the now-reaped (possibly recycled) + // pgid before it can fire, then stand it down. + reaped.store(true, std::sync::atomic::Ordering::SeqCst); let err = err_reader.join().unwrap_or_default(); let _ = writer.join(); - - // Reaped: stand the watchdog down and learn whether it killed on the deadline. let _ = done_tx.send(()); let killed = watchdog.join().unwrap_or(false); - if killed { + read_result.context("failed to read git stdout")?; + // The watchdog runs off a wall clock that can race a child finishing right at the + // deadline. A child that exited on its own (success) is not a timeout even if the + // watchdog fired late; only a child that did not exit successfully is a genuine + // timeout, which keeps a walk completing at its budget from a spurious 504. + if killed && !status.success() { return Err(crate::git::smart_http::GitServiceTimeout.into()); } - read_result.context("failed to read git stdout")?; if !status.success() { anyhow::bail!("git {label} failed: {}", String::from_utf8_lossy(&err)); } From e750c3042dff5dc0aa8b1959f5bb72ca7be68137 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 12 Jul 2026 00:56:24 -0500 Subject: [PATCH 16/58] fix(review): cap the receive-pack advertisement per source so anon cannot starve the write pool (#174) U2 moved the receive-pack info/refs advertisement onto git_write_semaphore to keep an anonymous read flood from starving the push handshake. But the advertisement is anon-reachable on public repos and holds its write permit across the slow acquire_fresh Tigris download, and the only per-source brake on it was the push RATE limiter, not a concurrency cap. So a multi-source flood of receive-pack advertisements could hold the write pool's slots across those downloads and shed authenticated pushes (both the advertisement and the owner-gated git-receive-pack POST draw from the same pool). U2 thus introduced the first anonymous consumer of the write pool the state doc promised anon could never reach; the plan's residual note (no worse than the POST) was wrong, because the POST is owner-gated and the advertisement is not. Add git_push_advert_per_caller, a per-source concurrency sub-cap on the receive-pack advertisement keyed on the resolved source IP (the same PerCallerConcurrency mechanism U1 uses for reads), sized to an eighth of the write pool so a single source holds at most that share and saturating the pool takes many distinct source IPs, each also braked by the per-IP push rate limiter. The upload-pack advertisement keeps its read-pool per-caller cap; the owner-gated POST is unchanged. Correct the state doc for git_write_semaphore accordingly. Handler-layer proof: a source at its receive-pack advertisement cap sheds 503 (RED before the acquisition, 500-not-503), while a different source and the upload-pack advertisement are unaffected. Full suite 510 green. --- crates/gitlawb-node/src/api/repos.rs | 101 +++++++++++++++++++++--- crates/gitlawb-node/src/auth/mod.rs | 2 + crates/gitlawb-node/src/main.rs | 7 ++ crates/gitlawb-node/src/state.rs | 22 ++++-- crates/gitlawb-node/src/test_support.rs | 3 + 5 files changed, 121 insertions(+), 14 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 7a8aa5c2..796558cf 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -574,16 +574,24 @@ pub async fn git_info_refs( } } - // Per-caller read sub-cap (#174): upload-pack advertisement only. The - // receive-pack advertisement is a write-path op (it drew from git_write_semaphore - // above and is braked per-IP by push_rate_limiter), so it must NOT consume a - // read caller's scarce slot (#174 U2). Acquired AFTER the visibility + push-rate - // gates (KTD7) so a denied or rate-limited request never consumes a slot. Held - // for the whole op. + // Per-source concurrency sub-cap (#174), keyed on the resolved source IP and + // acquired AFTER the visibility + push-rate gates (KTD7) so a denied or + // rate-limited request never consumes a slot; held for the whole op. The + // upload-pack advertisement is bounded on the read pool (git_read_per_caller). + // The receive-pack advertisement draws from the write pool, so it is bounded per + // source (git_push_advert_per_caller) instead: without this, an anonymous + // multi-source flood of push-handshake advertisements could hold the write pool's + // slots across acquire_fresh and shed authenticated pushes, since the per-IP push + // rate limiter caps rate, not concurrency (#174 review fix). + let caller_key = read_caller_key(&headers, peer, state.push_limiter_trust); let _caller_permit = if service == "git-receive-pack" { - None + acquire_read_caller_permit( + &state.git_push_advert_per_caller, + caller_key.as_deref(), + name, + "receive-pack advert", + )? } else { - let caller_key = read_caller_key(&headers, peer, state.push_limiter_trust); acquire_read_caller_permit( &state.git_read_per_caller, caller_key.as_deref(), @@ -676,7 +684,7 @@ fn acquire_read_caller_permit( None => { tracing::warn!(repo = %repo, caller = %k, handler, "per-caller read cap reached; shedding with 503"); Err(AppError::Overloaded( - "git read service at capacity for this caller, retry shortly".into(), + "git service at capacity for this caller, retry shortly".into(), )) } }, @@ -2827,6 +2835,81 @@ mod tests { ); } + /// #174 (review fix): the anon-reachable receive-pack advertisement draws from + /// the write pool, so it is bounded per source by `git_push_advert_per_caller` to + /// stop one source from monopolizing the write pool and shedding authenticated + /// pushes. Fill one source IP's advert slot; its next receive-pack advertisement + /// sheds 503, while a different source and the upload-pack advertisement are + /// unaffected. Remove the advert-cap acquisition and the same-source assertion + /// goes green-not-503. + #[sqlx::test] + async fn receive_pack_advertisement_capped_per_source(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + state.git_push_advert_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6advcap", "ac", "/tmp/ac-nonexistent", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.81:6000".parse().unwrap(); + // Fill this source IP's single receive-pack-advertisement slot. + let _slot = state + .git_push_advert_per_caller + .try_acquire(&peer.ip().to_string()) + .expect("first advert slot for this source IP"); + + // Same source: the receive-pack advertisement sheds 503 (advert cap full). + let router = crate::server::build_router(state.clone()); + let mut req = Request::builder() + .method(Method::GET) + .uri("/z6advcap/ac/info/refs?service=git-receive-pack") + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + assert_eq!( + router.oneshot(req).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a source at its receive-pack advertisement cap must shed 503, so it cannot monopolize the write pool" + ); + + // A DIFFERENT source keeps its own advert budget -> not shed. + let other: SocketAddr = "203.0.113.82:6000".parse().unwrap(); + let router2 = crate::server::build_router(state.clone()); + let mut req2 = Request::builder() + .method(Method::GET) + .uri("/z6advcap/ac/info/refs?service=git-receive-pack") + .body(Body::empty()) + .unwrap(); + req2.extensions_mut().insert(ConnectInfo(other)); + assert_ne!( + router2.oneshot(req2).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a different source must keep its own receive-pack advertisement budget" + ); + + // The upload-pack advertisement is NOT bounded by the receive-pack advert cap. + let router3 = crate::server::build_router(state); + let mut req3 = Request::builder() + .method(Method::GET) + .uri("/z6advcap/ac/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + req3.extensions_mut().insert(ConnectInfo(peer)); + assert_ne!( + router3.oneshot(req3).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "the upload-pack advertisement must not be shed by the receive-pack advert cap" + ); + } + /// #174 SC2 (info_refs probe): the per-caller read sub-cap sheds a caller that /// is already at its concurrency budget on the upload-pack advertisement, while /// a DIFFERENT caller still enters. Remove the sub-cap from `git_info_refs` and diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index 253b3210..b9e28e57 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -523,6 +523,8 @@ mod tests { git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), + git_push_advert_per_caller: + crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), } } diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index c617b434..0f33fde4 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -385,6 +385,13 @@ async fn main() -> Result<()> { git_read_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( config.max_concurrent_reads_per_caller, ), + // Per-source cap on the receive-pack advertisement, sized to an eighth of the + // write pool (min 1): a single source can hold at most this many write-pool + // slots via the anon advertisement, so saturating the pool takes ~8 distinct + // source IPs, each also rate-limited (#174). + git_push_advert_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( + (config.max_concurrent_git_pushes / 8).max(1), + ), }; // Periodic peer-count poll for the metrics gauge. If p2p is disabled diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 01007ac4..f55abeb7 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -95,14 +95,26 @@ pub struct AppState { /// authenticated push at admission (#174). pub git_read_semaphore: Arc, /// Bounds concurrent `git-receive-pack` (push) operations, a pool separate - /// from `git_read_semaphore` so anonymous reads can never shed an authenticated - /// push (#174). Sized by `max_concurrent_git_pushes`. + /// from `git_read_semaphore` so an anonymous READ flood can never shed an + /// authenticated push (#174). Sized by `max_concurrent_git_pushes`. Drawn from + /// by the `git-receive-pack` POST (owner-gated) and the receive-pack `info/refs` + /// advertisement (anon-reachable); the advertisement is additionally bounded per + /// source by `git_push_advert_per_caller` so no single source can monopolize this + /// pool and shed pushes. pub git_write_semaphore: Arc, - /// Per-caller concurrency sub-cap on the read pool: each caller (per-DID when - /// signed, else per-source-IP) may hold at most `max_concurrent_reads_per_caller` + /// Per-caller concurrency sub-cap on the read pool: each caller (keyed on the + /// resolved source IP, #174 U1) may hold at most `max_concurrent_reads_per_caller` /// in-flight read ops, so one caller cannot monopolize `git_read_semaphore` - /// (#174). Applied by `git_upload_pack` and both `info/refs` advertisements. + /// (#174). Applied by `git_upload_pack` and the upload-pack `info/refs` + /// advertisement. pub git_read_per_caller: crate::rate_limit::PerCallerConcurrency, + /// Per-source concurrency sub-cap on the anon-reachable receive-pack `info/refs` + /// advertisement: each source IP may hold at most a small share of the write + /// pool, so a multi-source flood of push-handshake advertisements cannot + /// saturate `git_write_semaphore` and shed authenticated pushes (#174). Sized as + /// a fraction of `max_concurrent_git_pushes`, so filling the write pool takes many + /// distinct source IPs (each also braked by the per-IP push rate limiter). + pub git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency, } impl AppState { diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 7745d233..03d3d5c4 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -86,6 +86,9 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), + git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys( + 8, + ), } } From c35452ed41091ba38f1b294a96f44d1d7b827f74 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 12 Jul 2026 02:12:08 -0500 Subject: [PATCH 17/58] test(node): vet the withheld-walk bound end to end; use the configured timeout (#174) Close the reasoned-not-run gaps from the code review by making the walk's git binary and timeout injectable, then driving the missing branches with a real handler and a fake git instead of reasoning about them. - Add state.git_bin and *_bounded variants of the walk entry points taking (git_bin, timeout); the served handlers (upload-pack serve, receive-pack replication and full-scan and encrypt-pin, and the ipfs gate) now pass the operator-configured GITLAWB_GIT_SERVICE_TIMEOUT_SECS, so the whole walk is bounded by the same budget as the other served-git ops rather than a fixed constant. The git_bin-less wrappers stay for the real-git classification tests. Newly vetted by execution (not reasoning): - receive-pack replication path is bounded: replication_withheld_set with an injected hung git returns within the budget and fails closed, so it cannot pin the write permit git_receive_pack holds across it. - a hung withheld-blob walk on the upload-pack POST returns 504 (real handler, real repo on disk, injected hung git), proving the GitServiceTimeout -> git_service_app_error wiring end to end. - the watchdog status-gate: a child that exits successfully is not reported as a timeout even when the watchdog fired (mutation-checked: drop the guard -> RED). - SIGKILL escalation: a SIGTERM-ignoring child is still reaped via SIGKILL and the group is gone; a truly uninterruptible D-state child (unreapable by any signal) is the documented residual, matching the async teardown. - the advertisement per-source cap sizing never derives 0. Full gitlawb-node suite 515 green. --- crates/gitlawb-node/src/api/ipfs.rs | 12 +- crates/gitlawb-node/src/api/repos.rs | 208 +++++++++++++++++- crates/gitlawb-node/src/auth/mod.rs | 1 + .../gitlawb-node/src/git/visibility_pack.rs | 173 ++++++++++++++- crates/gitlawb-node/src/main.rs | 1 + crates/gitlawb-node/src/state.rs | 5 + crates/gitlawb-node/src/test_support.rs | 1 + 7 files changed, 383 insertions(+), 18 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index f3de7570..e921c39f 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -27,7 +27,7 @@ use std::str::FromStr; use crate::auth::AuthenticatedDid; use crate::error::{AppError, Result}; use crate::git::store; -use crate::git::visibility_pack::{allowed_blob_set_for_caller, has_path_scoped_rule}; +use crate::git::visibility_pack::{allowed_blob_set_for_caller_bounded, has_path_scoped_rule}; use crate::state::AppState; use crate::visibility::{visibility_check, Decision}; @@ -142,10 +142,16 @@ pub async fn get_by_cid( let is_public = repo.is_public; let owner = repo.owner_did.clone(); let caller_for_walk = caller_owned.clone(); - // Full-history walk shells out to git — keep it off the async runtime. + let git_bin = state.git_bin.clone(); + let walk_timeout = + std::time::Duration::from_secs(state.config.git_service_timeout_secs); + // Full-history walk shells out to git — keep it off the async runtime, + // bounded and reaped like the served-git ops (#174). let walk = tokio::task::spawn_blocking(move || { - allowed_blob_set_for_caller( + allowed_blob_set_for_caller_bounded( &rp, + &git_bin, + walk_timeout, &r, is_public, &owner, diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 796558cf..1532be58 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -44,6 +44,8 @@ async fn replication_withheld_set( owner_did: &str, is_public: bool, disk_path: std::path::PathBuf, + git_bin: String, + timeout: std::time::Duration, ) -> (bool, Option>) { let announce = match &rules { Some(rules) => crate::visibility::listable_at_root(rules, is_public, owner_did, None), @@ -65,8 +67,8 @@ async fn replication_withheld_set( Some(rules) => { let owner_did = owner_did.to_string(); tokio::task::spawn_blocking(move || { - crate::git::visibility_pack::withheld_blob_oids( - &disk_path, &rules, is_public, &owner_did, None, + crate::git::visibility_pack::withheld_blob_oids_bounded( + &disk_path, &git_bin, timeout, &rules, is_public, &owner_did, None, ) }) .await @@ -106,10 +108,12 @@ async fn fail_closed_full_scan_objects( is_public: bool, owner_did: String, candidates: Vec, + git_bin: String, + timeout: std::time::Duration, ) -> Vec { tokio::task::spawn_blocking(move || -> anyhow::Result> { - let allowed = crate::git::visibility_pack::replicable_blob_set( - &disk_path, &rules, is_public, &owner_did, + let allowed = crate::git::visibility_pack::replicable_blob_set_bounded( + &disk_path, &git_bin, timeout, &rules, is_public, &owner_did, )?; let all_blobs = crate::git::push_delta::all_blob_oids(&disk_path)?; Ok(crate::git::visibility_pack::replicable_objects_fail_closed( @@ -778,9 +782,12 @@ pub async fn git_upload_pack( let owner_did = record.owner_did.clone(); let caller_owned = caller.map(str::to_string); let is_public = record.is_public; + let git_bin = state.git_bin.clone(); tokio::task::spawn_blocking(move || { - visibility_pack::withheld_blob_oids( + visibility_pack::withheld_blob_oids_bounded( &path, + &git_bin, + git_timeout, &rules, is_public, &owner_did, @@ -1185,6 +1192,8 @@ pub async fn git_receive_pack( &record.owner_did, record.is_public, disk_path.clone(), + state.git_bin.clone(), + std::time::Duration::from_secs(state.config.git_service_timeout_secs), ) .await; @@ -1219,6 +1228,8 @@ pub async fn git_receive_pack( record.is_public, record.owner_did.clone(), pin_set.candidates, + state.git_bin.clone(), + std::time::Duration::from_secs(state.config.git_service_timeout_secs), ) .await } else { @@ -1244,6 +1255,8 @@ pub async fn git_receive_pack( let node_did_str = state.node_did.to_string(); let node_seed = state.node_keypair.to_seed(); let repo_name = record.name.clone(); + let enc_git_bin = state.git_bin.clone(); + let enc_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); tokio::spawn(async move { let pinned = crate::ipfs_pin::pin_new_objects( &ipfs_api, @@ -1268,9 +1281,15 @@ pub async fn git_receive_pack( { let p = repo_path_clone.clone(); let owner = owner_did.clone(); + let git_bin = enc_git_bin.clone(); let recip = tokio::task::spawn_blocking(move || { - crate::git::visibility_pack::withheld_blob_recipients( - &p, &rules, is_public, &owner, + crate::git::visibility_pack::withheld_blob_recipients_bounded( + &p, + &git_bin, + enc_timeout, + &rules, + is_public, + &owner, ) }) .await; @@ -2031,16 +2050,39 @@ mod tests { let dummy = std::path::PathBuf::from("/nonexistent"); // Private: no rules at all. - let (announce, _) = replication_withheld_set(None, OWNER_DID, false, dummy.clone()).await; + let (announce, _) = replication_withheld_set( + None, + OWNER_DID, + false, + dummy.clone(), + "git".into(), + std::time::Duration::from_secs(600), + ) + .await; assert!(!announce, "private repo (no rules) must not announce"); // Private: empty rule set, is_public=false → still not listable at root. - let (announce, _) = - replication_withheld_set(Some(vec![]), OWNER_DID, false, dummy.clone()).await; + let (announce, _) = replication_withheld_set( + Some(vec![]), + OWNER_DID, + false, + dummy.clone(), + "git".into(), + std::time::Duration::from_secs(600), + ) + .await; assert!(!announce, "private repo (empty rules) must not announce"); // Public: empty rule set, is_public=true → listable at root, announces. - let (announce, _) = replication_withheld_set(Some(vec![]), OWNER_DID, true, dummy).await; + let (announce, _) = replication_withheld_set( + Some(vec![]), + OWNER_DID, + true, + dummy, + "git".into(), + std::time::Duration::from_secs(600), + ) + .await; assert!(announce, "public repo must announce"); } @@ -2132,6 +2174,150 @@ mod tests { } } + #[cfg(unix)] + fn write_fake_git(dir: &std::path::Path, body: &str) -> String { + use std::os::unix::fs::PermissionsExt; + let p = dir.join("fakegit"); + std::fs::write(&p, body).unwrap(); + let mut perm = std::fs::metadata(&p).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&p, perm).unwrap(); + p.to_str().unwrap().to_string() + } + + /// #174 (write-pool twin, vetted by execution not reasoning): the receive-pack + /// post-push replication walk is bounded. Drive `replication_withheld_set` with an + /// injected fake git that hangs on `rev-list` and a short budget: it must RETURN + /// within the budget (so `git_receive_pack` releases the write permit it holds + /// across this await, rather than pinning it for the hang) AND fail closed + /// (announce suppressed) because the walk could not be vetted. Proves this path + /// funnels through the bounded `blob_paths`, on the write-permit-holding side. + #[cfg(unix)] + #[tokio::test] + async fn replication_walk_is_bounded_and_fails_closed_on_a_hung_git() { + use std::time::Duration; + let tmp = tempfile::TempDir::new().unwrap(); + let body = "#!/bin/sh\ncase \"$1\" in\n rev-list) sleep 30 ;;\n rev-parse) echo deadbeef ;;\n *) : ;;\nesac\nexit 0\n"; + let git_bin = write_fake_git(tmp.path(), body); + // Public root (announceable) + a path-scoped rule, so the walk actually runs + // rather than taking the has_path_scoped_rule short-circuit. + let rules = Some(vec![vis_rule("/secret/**", &[])]); + + let result = tokio::time::timeout( + Duration::from_secs(10), + replication_withheld_set( + rules, + OWNER_DID, + true, + tmp.path().to_path_buf(), + git_bin, + Duration::from_millis(200), + ), + ) + .await + .expect( + "replication_withheld_set must return within the budget; a hung walk must \ + not pin the write permit git_receive_pack holds across it", + ); + assert_eq!( + result, + (false, None), + "a walk that could not be vetted must suppress the announce (fail closed)" + ); + } + + /// #174 (serve-path 504, vetted by execution): a hung withheld-blob walk on the + /// upload-pack POST maps to 504, not a generic 500. Real repo dir on disk (so + /// acquire's fast path returns it) + a path-scoped rule (so the walk runs) + + /// an injected fake git that hangs on rev-list. The handler must return 504, + /// proving git_upload_pack routes the walk's GitServiceTimeout through + /// git_service_app_error end to end. + #[cfg(unix)] + #[sqlx::test] + async fn upload_pack_hung_withheld_walk_returns_504(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use tower::ServiceExt; + + let tmp = tempfile::TempDir::new().unwrap(); + let body = "#!/bin/sh\ncase \"$1\" in\n rev-list) sleep 30 ;;\n rev-parse) echo deadbeef ;;\n *) : ;;\nesac\nexit 0\n"; + let fake = write_fake_git(tmp.path(), body); + + let mut state = crate::test_support::test_state(pool).await; + state.git_bin = fake; + let mut cfg = (*state.config).clone(); + cfg.git_service_timeout_secs = 1; + state.config = std::sync::Arc::new(cfg); + state + .db + .upsert_mirror_repo("z6srv504", "sv", "/tmp/z6srv504-sv", None, false) + .await + .unwrap(); + let rec = state.db.get_repo("z6srv504", "sv").await.unwrap().unwrap(); + // Path-scoped rule so has_path_scoped_rule() is true and the walk runs; the + // public root still lets an anonymous caller past the "/" gate. + state + .db + .set_visibility_rule( + &rec.id, + "/secret/**", + crate::db::VisibilityMode::B, + &[], + OWNER_DID, + ) + .await + .unwrap(); + // acquire()'s fast path returns the local path when it exists on disk. + let disk = std::path::Path::new("/tmp/z6srv504/sv.git"); + std::fs::create_dir_all(disk).unwrap(); + + let peer: SocketAddr = "203.0.113.91:7000".parse().unwrap(); + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::POST) + .uri("/z6srv504/sv/git-upload-pack") + .body(Body::from(&b"0000"[..])) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + let status = router.oneshot(req).await.unwrap().status(); + let _ = std::fs::remove_dir_all("/tmp/z6srv504"); + assert_eq!( + status, + StatusCode::GATEWAY_TIMEOUT, + "a hung withheld-blob walk must surface as 504, not a generic 500" + ); + } + + /// #174 (F2 sizing edge, vetted by execution): the receive-pack advertisement + /// per-source cap is derived in main.rs as `(max_concurrent_git_pushes / 8).max(1)`, + /// so it is never 0 even at the minimum write-pool size (1). A 0 cap would make + /// PerCallerConcurrency shed EVERY receive-pack advertisement and break all pushes. + #[test] + fn advert_per_caller_cap_sizing_is_never_zero() { + let cap = |pushes: usize| (pushes / 8).max(1); + for pushes in [1usize, 4, 8, 32, 256] { + assert!( + cap(pushes) >= 1, + "advert cap must be >= 1 for pushes={pushes}" + ); + } + assert_eq!(cap(1), 1, "minimum write pool must derive cap 1, not 0"); + assert_eq!( + cap(32), + 4, + "default write pool 32 derives cap 4 (~8 source IPs to fill)" + ); + // A cap of 1 admits one and sheds the second from the same source. + let lim = crate::rate_limit::PerCallerConcurrency::new(cap(1), 100); + let _held = lim.try_acquire("src").expect("first advert admitted"); + assert!( + lim.try_acquire("src").is_none(), + "second advert from the same source is shed" + ); + } + #[test] fn fork_owner_full_did_with_path_rule_allowed() { // Owner reads everything (implicit reader), so nothing is withheld. diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index b9e28e57..5befa7f3 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -525,6 +525,7 @@ mod tests { git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), + git_bin: "git".to_string(), } } diff --git a/crates/gitlawb-node/src/git/visibility_pack.rs b/crates/gitlawb-node/src/git/visibility_pack.rs index 0d6661e4..09a80aba 100644 --- a/crates/gitlawb-node/src/git/visibility_pack.rs +++ b/crates/gitlawb-node/src/git/visibility_pack.rs @@ -17,7 +17,10 @@ use std::time::{Duration, Instant}; /// pathologically slow git child so it cannot pin a served-git permit (the read /// permit on the upload-pack serve path, the write permit on the receive-pack /// post-push replication path) past the deadline. Every caller funnels through -/// `blob_paths`, so bounding here bounds both paths at one seam. +/// `blob_paths`, so bounding here bounds both paths at one seam. Production callers +/// pass the operator-configured `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` instead; this +/// fixed budget only backs the `git_bin`-less test wrappers. +#[cfg(test)] const WALK_TIMEOUT: Duration = Duration::from_secs(600); /// Run one git child under a shared `deadline` with process-group teardown, @@ -122,6 +125,12 @@ fn run_bounded_git( }); let mut stdout = child.stdout.take().context("git stdout was not piped")?; let mut out = Vec::new(); + // Blocking drain, unblocked by the child closing stdout on exit. The watchdog's + // SIGTERM/SIGKILL is what makes a hung child exit; a git wedged in uninterruptible + // (D-state) I/O survives even SIGKILL, so this drain and the wait below can block + // until the kernel returns, pinning the walk thread and its permit. That residual + // is unreachable in userspace (no signal reaps a D-state process) and matches the + // async `reap_group_on_timeout`, which likewise only warns and gives up there. let read_result = stdout.read_to_end(&mut out); let status = child.wait().context("git wait failed")?; // Reaped: bar the watchdog from signalling the now-reaped (possibly recycled) @@ -325,6 +334,7 @@ fn blob_paths(repo_path: &Path, git_bin: &str, timeout: Duration) -> Result, ) -> Result> { - let pairs = blob_paths(repo_path, "git", WALK_TIMEOUT)?; + withheld_blob_oids_bounded( + repo_path, + "git", + WALK_TIMEOUT, + rules, + is_public, + owner_did, + caller, + ) +} + +/// [`withheld_blob_oids`] with an injectable `git_bin` and walk `timeout`. Served +/// handlers call this with the operator-configured git binary and +/// `GITLAWB_GIT_SERVICE_TIMEOUT_SECS`, so the whole walk is bounded by the same +/// budget as the other served-git ops and a fake `git` can drive its teardown in +/// tests. The `git_bin`-less wrapper above keeps the fixed [`WALK_TIMEOUT`] for the +/// classification tests that run against real git. +pub fn withheld_blob_oids_bounded( + repo_path: &Path, + git_bin: &str, + timeout: Duration, + rules: &[VisibilityRule], + is_public: bool, + owner_did: &str, + caller: Option<&str>, +) -> Result> { + let pairs = blob_paths(repo_path, git_bin, timeout)?; Ok(withheld_from_pairs( &pairs, rules, is_public, owner_did, caller, )) @@ -400,6 +436,7 @@ pub fn replicable_objects(all: Vec, withheld: &HashSet) -> Vec Result> { + allowed_blob_set_for_caller_bounded( + repo_path, git_bin, timeout, rules, is_public, owner_did, None, + ) +} + /// Reachable blob OIDs that visibility ALLOWS `caller` at some path. The /// caller-aware generalization of `replicable_blob_set` (which is the anonymous /// `caller = None` case). Used by `GET /ipfs/{cid}` to gate fail-closed against @@ -421,6 +473,7 @@ pub fn replicable_blob_set( /// elsewhere (its content is readable to this caller elsewhere). Trees and /// commits are NOT included here; the caller decides per object type whether /// the allow-set applies (it does not for trees/commits — KTD3). +#[cfg(test)] pub fn allowed_blob_set_for_caller( repo_path: &Path, rules: &[VisibilityRule], @@ -428,7 +481,29 @@ pub fn allowed_blob_set_for_caller( owner_did: &str, caller: Option<&str>, ) -> Result> { - let pairs = blob_paths(repo_path, "git", WALK_TIMEOUT)?; + allowed_blob_set_for_caller_bounded( + repo_path, + "git", + WALK_TIMEOUT, + rules, + is_public, + owner_did, + caller, + ) +} + +/// [`allowed_blob_set_for_caller`] with an injectable `git_bin` and walk `timeout`, +/// for the `GET /ipfs/{cid}` gate. +pub fn allowed_blob_set_for_caller_bounded( + repo_path: &Path, + git_bin: &str, + timeout: Duration, + rules: &[VisibilityRule], + is_public: bool, + owner_did: &str, + caller: Option<&str>, +) -> Result> { + let pairs = blob_paths(repo_path, git_bin, timeout)?; let mut allowed = HashSet::new(); for (oid, path) in &pairs { if visibility_check(rules, is_public, owner_did, caller, path) == Decision::Allow { @@ -461,14 +536,28 @@ pub fn replicable_objects_fail_closed( /// owner plus any reader DID that `visibility_check` Allows at some path the /// blob appears at. Least-privilege: a reader of one private subtree is not a /// recipient of a blob that only lives in another. +#[cfg(test)] pub fn withheld_blob_recipients( repo_path: &Path, rules: &[VisibilityRule], is_public: bool, owner_did: &str, +) -> Result>> { + withheld_blob_recipients_bounded(repo_path, "git", WALK_TIMEOUT, rules, is_public, owner_did) +} + +/// [`withheld_blob_recipients`] with an injectable `git_bin` and walk `timeout`, for +/// the receive-pack encrypt-then-pin path. +pub fn withheld_blob_recipients_bounded( + repo_path: &Path, + git_bin: &str, + timeout: Duration, + rules: &[VisibilityRule], + is_public: bool, + owner_did: &str, ) -> Result>> { // One history walk feeds both the withheld set and the recipient mapping. - let pairs = blob_paths(repo_path, "git", WALK_TIMEOUT)?; + let pairs = blob_paths(repo_path, git_bin, timeout)?; let withheld = withheld_from_pairs(&pairs, rules, is_public, owner_did, None); if withheld.is_empty() { return Ok(HashMap::new()); @@ -571,6 +660,82 @@ mod tests { "the hung git child (pid {pid}) must be reaped, not orphaned, after the walk aborts" ); } + + /// #174 (F1 status-gate, vetted by execution): a child that exits SUCCESSFULLY is + /// never reported as a timeout even when the watchdog fires, so a walk finishing + /// right at its deadline is not a spurious 504. The fake only exits when signalled + /// and exits 0 on SIGTERM, so with a deadline already elapsed the watchdog always + /// reaches its kill path (killed == true) yet the child's status is success. + /// Drop the `!status.success()` guard and this returns GitServiceTimeout (RED). + #[cfg(unix)] + #[test] + fn run_bounded_git_success_at_the_deadline_is_not_a_timeout() { + use std::time::{Duration, Instant}; + let tmp = TempDir::new().unwrap(); + let body = "#!/bin/sh\ntrap 'exit 0' TERM\nsleep 30 &\nwait\n"; + let git_bin = write_fake_git(tmp.path(), body); + let out = run_bounded_git( + &git_bin, + &["rev-list"], + tmp.path(), + b"", + Instant::now() + Duration::from_millis(100), + ); + assert!( + out.is_ok(), + "a child that exited successfully must not be reported as a timeout even if the watchdog fired: {out:?}" + ); + } + + /// #174 (F3, vetted by execution): a child that IGNORES SIGTERM is still reaped + /// via the watchdog's SIGKILL escalation, so it cannot pin the walk thread or its + /// permit. The fake traps SIGTERM and keeps sleeping; run_bounded_git must still + /// return (via SIGKILL at the grace step) with a timeout error and the group must + /// be gone. (A truly uninterruptible D-state child, which no signal can reap, is + /// the documented residual this teardown, like the async twin, cannot cover.) + #[cfg(unix)] + #[test] + fn run_bounded_git_reaps_a_sigterm_ignoring_child_via_sigkill() { + use std::time::{Duration, Instant}; + let tmp = TempDir::new().unwrap(); + let body = "#!/bin/sh\ntrap '' TERM\necho $$ > pid\nwhile true; do sleep 1; done\n"; + let git_bin = write_fake_git(tmp.path(), body); + let (tx, rx) = std::sync::mpsc::channel(); + let path = tmp.path().to_path_buf(); + std::thread::spawn(move || { + let _ = tx.send(run_bounded_git( + &git_bin, + &["rev-list"], + &path, + b"", + Instant::now() + Duration::from_millis(100), + )); + }); + let out = rx + .recv_timeout(Duration::from_secs(10)) + .expect("run_bounded_git must return via SIGKILL even for a SIGTERM-ignoring child"); + assert!( + out.is_err(), + "a SIGTERM-ignoring child killed by SIGKILL is a timeout, not a success: {out:?}" + ); + let pid: i32 = std::fs::read_to_string(tmp.path().join("pid")) + .unwrap() + .trim() + .parse() + .unwrap(); + let mut gone = false; + for _ in 0..300 { + if unsafe { libc::kill(pid, 0) } != 0 { + gone = true; + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + assert!( + gone, + "the SIGTERM-ignoring child (pid {pid}) must be reaped via SIGKILL, not left running" + ); + } use crate::db::VisibilityMode; use chrono::Utc; use std::process::Command; diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 0f33fde4..9b646c2b 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -392,6 +392,7 @@ async fn main() -> Result<()> { git_push_advert_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( (config.max_concurrent_git_pushes / 8).max(1), ), + git_bin: "git".to_string(), }; // Periodic peer-count poll for the metrics gauge. If p2p is disabled diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index f55abeb7..94952120 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -115,6 +115,11 @@ pub struct AppState { /// a fraction of `max_concurrent_git_pushes`, so filling the write pool takes many /// distinct source IPs (each also braked by the per-IP push rate limiter). pub git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency, + /// The `git` executable the served-git withheld-blob walk spawns. Production is + /// `"git"` (resolved via PATH); injectable so a fake `git` can drive the walk's + /// process-group teardown in handler tests without mutating the process-global + /// PATH (#174). + pub git_bin: String, } impl AppState { diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 03d3d5c4..a43a79a0 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -89,6 +89,7 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys( 8, ), + git_bin: "git".to_string(), } } From 454e59ba533535168a6eb763447275ef107564ad Mon Sep 17 00:00:00 2001 From: t Date: Sun, 12 Jul 2026 02:12:37 -0500 Subject: [PATCH 18/58] docs(node): the withheld-walk is bounded by the service timeout, not a fixed const (#174) Follow-up to threading the configured timeout into the walk: the walk now honors GITLAWB_GIT_SERVICE_TIMEOUT_SECS on both the serve and replication paths, so the README no longer says a fixed internal deadline. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d02c4a0b..0f760449 100644 --- a/README.md +++ b/README.md @@ -343,7 +343,7 @@ Important node settings: | `GITLAWB_REQUIRE_SIGNED_PEER_WRITES` | Require signed peer announce/sync writes. | | `GITLAWB_AUTO_SYNC` | Enable automatic sync from known peers. | | `GITLAWB_MAX_PACK_BYTES` | Max git pack body size for smart-HTTP routes. | -| `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack, receive-pack, or `info/refs` advertisement may run before it is aborted (504). Default 600. The withheld-blob classification walk is also bounded and reaped, by a fixed internal deadline rather than this setting. | +| `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack, receive-pack, or `info/refs` advertisement may run before it is aborted (504). Default 600. Also bounds the withheld-blob classification walk (on both the upload-pack serve and receive-pack replication paths), which is reaped at the deadline. | | `GITLAWB_TIGRIS_BUCKET` | Optional S3/Tigris shared repo storage bucket. | | `GITLAWB_PINATA_JWT` | Optional Pinata/IPFS warm-storage pinning. | | `GITLAWB_IRYS_URL` | Optional Irys/Arweave permanent anchoring. | From 79b9ee5c89070f2c9b8d534d28195803110385a1 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 12 Jul 2026 14:32:24 -0500 Subject: [PATCH 19/58] fix(node): shed the served-git pool before the DB and cap per-source before it (#174) CodeRabbit review: the held git_permit was acquired at the top of git_info_refs and git_upload_pack, before the per-source cap, so one source could occupy the global pool during the DB/visibility window before its sub-cap rejected the excess. Move the held git_permit to after acquire_read_caller_permit (still before acquire_fresh/git, so INV-10's bound on the fresh Tigris acquire holds), and add a cheap pre-DB early shed (available_permits() == 0 -> 503, holds no permit) so the #62 shed-before-DB property is preserved without holding a permit across the DB work. The receive-pack POST is unchanged (owner-only; its top-of-handler git_permit is deliberate for acquire_write bounding). Tests: the two #62 shed-before-DB tests now exercise the explicit early check (load-bearing: disable it and they fall through to the DB); three new *_per_source_cap_sheds_with_global_capacity tests prove the per-source sub-cap sheds a capped source even with global capacity free (load-bearing on the caller cap). Full workspace green. --- crates/gitlawb-node/src/api/repos.rs | 253 ++++++++++++++++++++++-- crates/gitlawb-node/src/test_support.rs | 62 ++++-- 2 files changed, 290 insertions(+), 25 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 1532be58..85eaa227 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -515,17 +515,27 @@ pub async fn git_info_refs( let service = query .service .ok_or_else(|| AppError::BadRequest("missing ?service= parameter".into()))?; - // Shed with a 503 before spawning git when the concurrency cap is saturated; - // held for the whole op (incl. the smart_http call), released on return. The - // receive-pack advertisement is phase one of a push, so it draws from the WRITE - // pool (like the git-receive-pack POST), not the global read pool an anonymous - // clone flood can exhaust and thereby starve the push handshake (#174 U2). The - // upload-pack advertisement stays on the read pool with its per-caller sub-cap. - let _permit = if service == "git-receive-pack" { - git_permit(&state.git_write_semaphore)? - } else { - git_permit(&state.git_read_semaphore)? - }; + // #62 cheap pre-DB load shed: if the pool this service draws from is already + // saturated, shed with 503 before any DB/disk work. Best-effort (holds no + // permit); the authoritative hold is `git_permit` below, after the per-source + // cap. Restores the shed-before-DB property the reordered held acquire alone + // would drop, while the reorder still prevents one source from occupying global + // slots during the DB/visibility window. + { + let pool = if service == "git-receive-pack" { + &state.git_write_semaphore + } else { + &state.git_read_semaphore + }; + if pool.available_permits() == 0 { + tracing::warn!( + "served-git concurrency cap reached; shedding request with 503 (pre-DB)" + ); + return Err(AppError::Overloaded( + "git service at capacity, retry shortly".into(), + )); + } + } tracing::info!(owner = %owner, repo = %name, "info/refs request"); let record = state .db @@ -604,6 +614,22 @@ pub async fn git_info_refs( )? }; + // Shed with a 503 before spawning git when the concurrency cap is saturated; + // held for the whole op (incl. the smart_http call), released on return. Taken + // AFTER the per-source cap above so one source cannot occupy global slots it + // would be sub-cap-denied for during the DB/visibility window and starve other + // sources; still before acquire_fresh/git so it bounds the fresh Tigris acquire + // and git exec (INV-10). The receive-pack advertisement is phase one of a push, + // so it draws from the WRITE pool (like the git-receive-pack POST), not the + // global read pool an anonymous clone flood can exhaust and thereby starve the + // push handshake (#174 U2). The upload-pack advertisement stays on the read + // pool with its per-caller sub-cap. + let _permit = if service == "git-receive-pack" { + git_permit(&state.git_write_semaphore)? + } else { + git_permit(&state.git_read_semaphore)? + }; + // For receive-pack (push), download the latest from Tigris so the client // sees the same refs that acquire_write() will operate on. let disk_path = if service == "git-receive-pack" { @@ -725,9 +751,15 @@ pub async fn git_upload_pack( headers: axum::http::HeaderMap, body: Bytes, ) -> Result { - // Shed with a 503 before spawning git when the concurrency cap is saturated; - // held for the whole op (incl. the smart_http call), released on return. - let _permit = git_permit(&state.git_read_semaphore)?; + // #62 cheap pre-DB load shed (see git_info_refs): shed before DB when the read + // pool is saturated; the authoritative hold is `git_permit` below, after the + // per-source cap. + if state.git_read_semaphore.available_permits() == 0 { + tracing::warn!("served-git concurrency cap reached; shedding request with 503 (pre-DB)"); + return Err(AppError::Overloaded( + "git service at capacity, retry shortly".into(), + )); + } let name = smart_http_repo_name(&repo)?; let record = state .db @@ -760,6 +792,14 @@ pub async fn git_upload_pack( "upload-pack", )?; + // Shed with a 503 before spawning git when the concurrency cap is saturated; + // held for the whole op (incl. the smart_http call), released on return. Taken + // AFTER the per-source cap above so one source cannot occupy global slots it + // would be sub-cap-denied for during the DB/visibility window and starve other + // sources; still before acquire/git so it bounds the Tigris acquire and git + // exec (INV-10). + let _permit = git_permit(&state.git_read_semaphore)?; + let disk_path = state .repo_store .acquire(&record.owner_did, &record.name) @@ -3209,6 +3249,191 @@ mod tests { ); } + /// #174 (review fix): the per-source caller cap is an independent brake that + /// sheds a capped source even when the global pool has free capacity — the + /// sub-cap is not a mere pre-filter for pool exhaustion. Proven by leaving the + /// global read pool with capacity (so the pre-DB early shed passes) AND + /// pre-holding the source's upload-pack read sub-cap: the request reaches the + /// caller cap and sheds there, so its 503 body reads "for this caller". Remove + /// the `acquire_read_caller_permit` call and the capped source falls through to + /// the git op instead of shedding with "for this caller" — this is the + /// caller-cap acquire probe for the info/refs upload-pack branch. + #[sqlx::test] + async fn info_refs_upload_pack_per_source_cap_sheds_with_global_capacity(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + // Global read pool has free capacity (early shed passes); source pre-held at + // its per-caller cap so it sheds on the caller cap, not the global pool. + state.git_read_semaphore = Arc::new(Semaphore::new(4)); + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6ordir", "oi", "/tmp/oi-nonexistent", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.91:5000".parse().unwrap(); + // Pin this source at its single upload-pack read slot. + let _slot = state + .git_read_per_caller + .try_acquire(&peer.ip().to_string()) + .expect("first read slot for this source IP"); + + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::GET) + .uri("/z6ordir/oi/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + let resp = router.oneshot(req).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a source at its read sub-cap must shed 503 even with global pool capacity" + ); + let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .expect("body bytes"); + let body = String::from_utf8_lossy(&bytes); + assert!( + body.contains("for this caller"), + "the per-source cap is an independent brake: with global capacity free, the capped source must still shed with the caller-cap body, got {body}" + ); + } + + /// #174 (review fix): same independent-brake guarantee for the receive-pack + /// advertisement branch of info/refs — its per-source cap + /// (`git_push_advert_per_caller`) sheds a capped source even when the global + /// write pool has capacity. Leave the global write pool with capacity (so the + /// pre-DB early shed passes) and pre-hold the source's advert slot: the request + /// reaches the caller cap, so the 503 body reads "for this caller". Remove the + /// caller-cap acquire and the capped source falls through instead of shedding + /// with "for this caller". The push rate limiter is left permissive so the + /// request reaches the caller cap. + #[sqlx::test] + async fn info_refs_receive_pack_per_source_cap_sheds_with_global_capacity(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::sync::Arc; + use std::time::Duration; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + // Global write pool has free capacity (early shed passes); source pre-held at + // its advert sub-cap so it sheds on the caller cap, not the global pool. + state.git_write_semaphore = Arc::new(Semaphore::new(4)); + state.git_push_advert_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + // Permissive push rate limiter so the advertisement passes the rate gate and + // reaches the per-source concurrency cap. + state.push_rate_limiter = crate::rate_limit::RateLimiter::new(100, Duration::from_secs(60)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6ordrp", "or", "/tmp/or-nonexistent", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.92:5000".parse().unwrap(); + // Pin this source at its single receive-pack advertisement slot. + let _slot = state + .git_push_advert_per_caller + .try_acquire(&peer.ip().to_string()) + .expect("first advert slot for this source IP"); + + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::GET) + .uri("/z6ordrp/or/info/refs?service=git-receive-pack") + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + let resp = router.oneshot(req).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a source at its advert sub-cap must shed 503 even with global write pool capacity" + ); + let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .expect("body bytes"); + let body = String::from_utf8_lossy(&bytes); + assert!( + body.contains("for this caller"), + "the per-source advert cap is an independent brake: with global write capacity free, the capped source must still shed with the caller-cap body, got {body}" + ); + } + + /// #174 (review fix): same independent-brake guarantee for the POST upload-pack + /// handler — its per-source read cap sheds a capped source even when the global + /// read pool has capacity. Leave the global read pool with capacity (so the + /// pre-DB early shed passes) and pre-hold the source's read slot: the request + /// reaches the caller cap, so the 503 body reads "for this caller". Remove the + /// caller-cap acquire and the capped source falls through instead of shedding + /// with "for this caller". + #[sqlx::test] + async fn upload_pack_per_source_cap_sheds_with_global_capacity(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + // Global read pool has free capacity (early shed passes); source pre-held at + // its per-caller cap so it sheds on the caller cap, not the global pool. + state.git_read_semaphore = Arc::new(Semaphore::new(4)); + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6ordup", "ou", "/tmp/ou-nonexistent", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.93:5000".parse().unwrap(); + // Pin this source at its single read slot. + let _slot = state + .git_read_per_caller + .try_acquire(&peer.ip().to_string()) + .expect("first read slot for this source IP"); + + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::POST) + .uri("/z6ordup/ou/git-upload-pack") + .body(Body::from(&b"0000"[..])) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + let resp = router.oneshot(req).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a source at its read sub-cap must shed 503 even with global pool capacity" + ); + let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .expect("body bytes"); + let body = String::from_utf8_lossy(&bytes); + assert!( + body.contains("for this caller"), + "the per-source cap is an independent brake: with global capacity free, the capped source must still shed with the caller-cap body, got {body}" + ); + } + /// #174 SC2 (per-source key, U1): the per-caller read sub-cap keys on the /// resolved source IP, NOT the signed DID, so a disposable-DID farm cannot /// multiply its budget. Fill the source IP's single read slot, then drive two diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index a43a79a0..765545f1 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -195,13 +195,14 @@ mod tests { ); } - /// PR3 (#62): the served-git concurrency cap sheds at the HTTP layer. One test - /// per git handler drives the `let _permit = git_permit(...)` wiring end to end - /// (this one plus the git_upload_pack / git_receive_pack siblings below); the - /// git_permit unit test covers the helper in isolation. DB-free: an exhausted - /// semaphore sheds before any DB/disk access, so a lazy state works. Remove the - /// permit line from git_info_refs and this goes red (the request falls through - /// to the DB and returns something other than 503). + /// PR3 (#62): the served-git concurrency cap sheds at the HTTP layer before the + /// DB. The held `git_permit` acquire now sits after the per-source cap, so the + /// shed-before-DB property is carried by an explicit `available_permits() == 0` + /// early check at the top of the handler (the held permit remains the + /// authoritative bound further down). DB-free: an exhausted semaphore sheds + /// before any DB/disk access, so a lazy state works. Remove the early-shed block + /// from git_info_refs and this goes red (the request falls through to the DB and + /// returns something other than 503). #[tokio::test] async fn git_info_refs_sheds_with_503_when_semaphore_exhausted() { let mut state = test_state_lazy(); @@ -234,10 +235,11 @@ mod tests { ); } - /// PR3 (#62) sibling of the info/refs shed test: git-upload-pack also acquires a - /// permit at the top, so an exhausted semaphore must shed it with a 503 before - /// any DB/disk work. Anonymous-reachable, so no auth injection is needed. Remove - /// the permit line from git_upload_pack and this goes red. + /// PR3 (#62) sibling of the info/refs shed test: git-upload-pack carries the same + /// explicit `available_permits() == 0` early-shed check at the top, so an + /// exhausted semaphore must shed it with a 503 before any DB/disk work. + /// Anonymous-reachable, so no auth injection is needed. Remove the early-shed + /// block from git_upload_pack and this goes red. #[tokio::test] async fn git_upload_pack_sheds_with_503_when_semaphore_exhausted() { let mut state = test_state_lazy(); @@ -270,6 +272,44 @@ mod tests { ); } + /// PR3 (#62) receive-pack sibling of the info/refs shed test: the early shed + /// selects the WRITE pool for a git-receive-pack advertisement (phase one of a + /// push, #174 U2), so an exhausted write pool sheds the advert with 503 before + /// any DB/disk work — even while the read pool is free (only the write pool is + /// zeroed here). Flip the pool selection to the read pool, or remove the + /// early-shed block, and this goes red. + #[tokio::test] + async fn git_info_refs_receive_pack_sheds_with_503_when_write_pool_exhausted() { + let mut state = test_state_lazy(); + state.git_write_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + + let router = Router::new() + .route( + "/{owner}/{repo}/info/refs", + axum::routing::get(crate::api::repos::git_info_refs), + ) + .with_state(state); + let resp = router + .oneshot(anon_get( + "/alice/repo.git/info/refs?service=git-receive-pack", + )) + .await + .unwrap(); + + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an exhausted WRITE pool must shed the receive-pack advertisement with 503 before touching the DB" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the 503 shed must carry Retry-After" + ); + } + /// PR3 (#62) sibling for the push path: git-receive-pack requires an /// AuthenticatedDid extension (production: require_signature injects it), so the /// request carries one via signed_request_as — without it the Extension From f19f2813a2b73c326001e5724e0ae175d0b9d72c Mon Sep 17 00:00:00 2001 From: t Date: Mon, 13 Jul 2026 09:44:53 -0500 Subject: [PATCH 20/58] fix(node): give the receive-pack advert its own pool so it can't shed authed pushes (#174) jatmn P1: the anon-reachable GET info/refs?service=git-receive-pack drew from the same git_write_semaphore as the authenticated git-receive-pack POST. The per-source advert cap reserves no global slots, so ~8 sources (each within the per-IP push rate limit) could hold all 32 write permits across acquire_fresh + the advertisement, and the next authenticated push then 503s at admission. That is an anonymous caller shedding an authenticated push across the auth boundary (INV-10). Add a dedicated git_push_advert_semaphore, disjoint from the write pool. Both the pre-DB early-shed peek and the held acquire for the receive-pack advertisement now draw from it; the write pool is left exclusively for the POST. An advert flood can at worst exhaust the advert pool (each source still capped by git_push_advert_per_caller and the per-IP push rate limiter); the authenticated push pool is untouched. Test receive_pack_advertisement_draws_from_dedicated_advert_pool proves at the handler layer that the advert sheds when its own pool is saturated AND survives when the write pool is saturated (the reservation). Both flip under revert, verified independently for the pre-DB peek and the held acquire. --- crates/gitlawb-node/src/api/repos.rs | 74 ++++++++++++++++--------- crates/gitlawb-node/src/auth/mod.rs | 1 + crates/gitlawb-node/src/main.rs | 7 +++ crates/gitlawb-node/src/state.rs | 16 ++++-- crates/gitlawb-node/src/test_support.rs | 17 +++--- 5 files changed, 76 insertions(+), 39 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 85eaa227..719246e0 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -522,8 +522,11 @@ pub async fn git_info_refs( // would drop, while the reorder still prevents one source from occupying global // slots during the DB/visibility window. { + // The receive-pack advertisement peeks its DEDICATED advert pool, not the + // write pool the authenticated POST uses (#174) — matching the held acquire + // below, so the pre-DB shed and the authoritative hold agree on the pool. let pool = if service == "git-receive-pack" { - &state.git_write_semaphore + &state.git_push_advert_semaphore } else { &state.git_read_semaphore }; @@ -620,12 +623,14 @@ pub async fn git_info_refs( // would be sub-cap-denied for during the DB/visibility window and starve other // sources; still before acquire_fresh/git so it bounds the fresh Tigris acquire // and git exec (INV-10). The receive-pack advertisement is phase one of a push, - // so it draws from the WRITE pool (like the git-receive-pack POST), not the - // global read pool an anonymous clone flood can exhaust and thereby starve the - // push handshake (#174 U2). The upload-pack advertisement stays on the read - // pool with its per-caller sub-cap. + // but it is ANON-reachable, so it draws from the dedicated advert pool + // (`git_push_advert_semaphore`), NOT the write pool the authenticated POST uses: + // an advert flood can at worst exhaust the advert pool, never a permit a push + // POST needs at admission (#174 U2). A clone flood on the read pool likewise + // can't touch either. The upload-pack advertisement stays on the read pool with + // its per-caller sub-cap. let _permit = if service == "git-receive-pack" { - git_permit(&state.git_write_semaphore)? + git_permit(&state.git_push_advert_semaphore)? } else { git_permit(&state.git_read_semaphore)? }; @@ -2944,14 +2949,19 @@ mod tests { ); } - /// #174 U2: the receive-pack advertisement (`GET info/refs?service=git-receive-pack`) - /// is phase one of a push, so it draws from the WRITE pool, not the global read - /// pool an anonymous clone flood can exhaust. Proven at the handler by saturating - /// each pool to zero and checking who shares it (INV-10). Revert the branch to the - /// read pool and the read-saturated receive-pack assertion goes 503 (the exact - /// push-handshake starvation jatmn flagged). + /// #174 (jatmn P1): the anon-reachable receive-pack advertisement + /// (`GET info/refs?service=git-receive-pack`) draws from a DEDICATED advert pool + /// (`git_push_advert_semaphore`), NOT the write pool the authenticated POST uses. + /// Proven at the handler by saturating each pool to zero and checking who shares + /// it (INV-10, across the auth boundary). The load-bearing pair: + /// * advert pool at 0 -> the advert SHEDS 503 (it is bound to that pool); + /// * write pool at 0 -> the advert SURVIVES (it can NOT consume a permit the + /// authenticated POST needs — the reservation jatmn asked for). + /// Revert the branch to `git_write_semaphore` and BOTH flip: the advert-pool-0 + /// case stops shedding and the write-pool-0 case starts shedding (the exact + /// anon-sheds-authed-push starvation). #[sqlx::test] - async fn receive_pack_advertisement_draws_from_write_pool(pool: sqlx::PgPool) { + async fn receive_pack_advertisement_draws_from_dedicated_advert_pool(pool: sqlx::PgPool) { use axum::body::Body; use axum::extract::ConnectInfo; use axum::http::{Method, Request, StatusCode}; @@ -2960,17 +2970,19 @@ mod tests { use tokio::sync::Semaphore; use tower::ServiceExt; - // Build a fresh state with the two pools sized independently, then drive one + // Build a fresh state with the three pools sized independently, then drive one // info/refs advertisement for `service` and return its handler status. async fn advert_status( pool: &sqlx::PgPool, read_permits: usize, write_permits: usize, + advert_permits: usize, service: &str, ) -> StatusCode { let mut state = crate::test_support::test_state(pool.clone()).await; state.git_read_semaphore = Arc::new(Semaphore::new(read_permits)); state.git_write_semaphore = Arc::new(Semaphore::new(write_permits)); + state.git_push_advert_semaphore = Arc::new(Semaphore::new(advert_permits)); state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; state .db @@ -2988,31 +3000,39 @@ mod tests { router.oneshot(req).await.unwrap().status() } - // Read pool saturated, write pool free: the push handshake SURVIVES. + // Advert pool saturated (read + write free): the receive-pack advert SHEDS, + // proving it is bound to the dedicated advert pool. + assert_eq!( + advert_status(&pool, 8, 8, 0, "git-receive-pack").await, + StatusCode::SERVICE_UNAVAILABLE, + "receive-pack advertisement draws from the dedicated advert pool: a saturated advert pool sheds it 503" + ); + // WRITE pool saturated (advert + read free): the advert SURVIVES. This is the + // reservation — an advert flood can never occupy a permit the authenticated + // push POST relies on at admission. assert_ne!( - advert_status(&pool, 0, 8, "git-receive-pack").await, + advert_status(&pool, 8, 0, 8, "git-receive-pack").await, StatusCode::SERVICE_UNAVAILABLE, - "receive-pack advertisement must draw from the write pool: a saturated read pool must not shed it" + "receive-pack advertisement must NOT draw from the write pool: a saturated write pool must not shed it" ); - // Write pool saturated, read pool free: the receive-pack advertisement SHEDS, - // proving it now consumes the write pool (not the read pool). - assert_eq!( - advert_status(&pool, 8, 0, "git-receive-pack").await, + // Read pool saturated (advert + write free): the advert SURVIVES (never on the read pool). + assert_ne!( + advert_status(&pool, 0, 8, 8, "git-receive-pack").await, StatusCode::SERVICE_UNAVAILABLE, - "receive-pack advertisement draws from the write pool, so a saturated write pool sheds it 503" + "receive-pack advertisement must not draw from the read pool" ); // Read pool saturated: the upload-pack advertisement still SHEDS (unchanged). assert_eq!( - advert_status(&pool, 0, 8, "git-upload-pack").await, + advert_status(&pool, 0, 8, 8, "git-upload-pack").await, StatusCode::SERVICE_UNAVAILABLE, "upload-pack advertisement stays on the read pool: a saturated read pool sheds it 503" ); - // Write pool saturated, read pool free: the upload-pack advertisement is - // UNAFFECTED, proving reads never touch the write pool in either direction. + // Write + advert pools saturated, read free: the upload-pack advertisement is + // UNAFFECTED, proving reads never touch either write-side pool. assert_ne!( - advert_status(&pool, 8, 0, "git-upload-pack").await, + advert_status(&pool, 8, 0, 0, "git-upload-pack").await, StatusCode::SERVICE_UNAVAILABLE, - "upload-pack advertisement never touches the write pool" + "upload-pack advertisement never touches the write or advert pool" ); } diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index 5befa7f3..5d827794 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -522,6 +522,7 @@ mod tests { shutdown_tx: tokio::sync::watch::channel(false).0, git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 9b646c2b..18019395 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -382,6 +382,13 @@ async fn main() -> Result<()> { git_write_semaphore: Arc::new(tokio::sync::Semaphore::new( config.max_concurrent_git_pushes, )), + // Anon receive-pack advertisements get their OWN pool, same size as the + // write pool but disjoint, so filling it (which takes many source IPs, each + // capped by git_push_advert_per_caller) never occupies a permit the + // authenticated POST needs (#174). + git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new( + config.max_concurrent_git_pushes, + )), git_read_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( config.max_concurrent_reads_per_caller, ), diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 94952120..eb07a3e4 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -97,11 +97,19 @@ pub struct AppState { /// Bounds concurrent `git-receive-pack` (push) operations, a pool separate /// from `git_read_semaphore` so an anonymous READ flood can never shed an /// authenticated push (#174). Sized by `max_concurrent_git_pushes`. Drawn from - /// by the `git-receive-pack` POST (owner-gated) and the receive-pack `info/refs` - /// advertisement (anon-reachable); the advertisement is additionally bounded per - /// source by `git_push_advert_per_caller` so no single source can monopolize this - /// pool and shed pushes. + /// by the `git-receive-pack` POST (owner-gated) ONLY. The anon-reachable + /// receive-pack `info/refs` advertisement draws from the SEPARATE + /// `git_push_advert_semaphore` below, never this pool, so a multi-source flood + /// of push-handshake advertisements can never occupy a permit an authenticated + /// POST needs at admission (#174). pub git_write_semaphore: Arc, + /// Bounds concurrent anon-reachable `git-receive-pack` `info/refs` + /// advertisements — a pool SEPARATE from `git_write_semaphore` so adverts (which + /// hold a permit across `acquire_fresh` + `info/refs`) can never consume a slot + /// the authenticated POST relies on. A per-source flood can at worst exhaust this + /// advert pool (each source also capped by `git_push_advert_per_caller` and the + /// per-IP push rate limiter), and the reserved POST pool is untouched (#174). + pub git_push_advert_semaphore: Arc, /// Per-caller concurrency sub-cap on the read pool: each caller (keyed on the /// resolved source IP, #174 U1) may hold at most `max_concurrent_reads_per_caller` /// in-flight read ops, so one caller cannot monopolize `git_read_semaphore` diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 765545f1..0805fec1 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -85,6 +85,7 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { // Generous — no test drives the handler-level shed (git_permit is unit-tested). git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys( 8, @@ -273,15 +274,15 @@ mod tests { } /// PR3 (#62) receive-pack sibling of the info/refs shed test: the early shed - /// selects the WRITE pool for a git-receive-pack advertisement (phase one of a - /// push, #174 U2), so an exhausted write pool sheds the advert with 503 before - /// any DB/disk work — even while the read pool is free (only the write pool is - /// zeroed here). Flip the pool selection to the read pool, or remove the - /// early-shed block, and this goes red. + /// selects the dedicated ADVERT pool for a git-receive-pack advertisement (#174), + /// so an exhausted advert pool sheds the advert with 503 before any DB/disk work + /// — while the write pool (reserved for authenticated POSTs) is left free here. + /// Flip the pool selection back to the write pool, or remove the early-shed + /// block, and this goes red. #[tokio::test] - async fn git_info_refs_receive_pack_sheds_with_503_when_write_pool_exhausted() { + async fn git_info_refs_receive_pack_sheds_with_503_when_advert_pool_exhausted() { let mut state = test_state_lazy(); - state.git_write_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + state.git_push_advert_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); let router = Router::new() .route( @@ -299,7 +300,7 @@ mod tests { assert_eq!( resp.status(), StatusCode::SERVICE_UNAVAILABLE, - "an exhausted WRITE pool must shed the receive-pack advertisement with 503 before touching the DB" + "an exhausted ADVERT pool must shed the receive-pack advertisement with 503 before touching the DB" ); assert_eq!( resp.headers() From 473924dcfec11c22b8ea5da5f125478bbcd5f845 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 13 Jul 2026 09:49:33 -0500 Subject: [PATCH 21/58] fix(node): compile the bounded visibility git runner on non-Unix targets (#174) jatmn P1: run_bounded_git was gated #[cfg(unix)], but its callers (assert_all_refs_are_commits, blob_paths) and the module are compiled on all targets, so a non-Unix build (the continue-on-error Windows release target) could not resolve the function. Add a #[cfg(not(unix))] twin with the same signature and result semantics. It bounds a single child (no process-group teardown, which is Unix-only): threads feed stdin and drain stderr, the main thread drains stdout, and a watchdog thread kills the child at the deadline. The child is shared with the watchdog behind a mutex the main thread does not hold while draining, so the watchdog can always acquire it to kill, and killing closes stdout to unblock the drain. Best-effort (reaps only the direct child), which is why the hardened group-aware path stays Unix-only. Verified the twin type-checks by compiling its (portable-std) body on the native target; the full non-Unix link is CI's Windows job. --- .../gitlawb-node/src/git/visibility_pack.rs | 92 +++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/crates/gitlawb-node/src/git/visibility_pack.rs b/crates/gitlawb-node/src/git/visibility_pack.rs index 09a80aba..0eaeed2d 100644 --- a/crates/gitlawb-node/src/git/visibility_pack.rs +++ b/crates/gitlawb-node/src/git/visibility_pack.rs @@ -154,6 +154,98 @@ fn run_bounded_git( Ok(out) } +/// Non-Unix fallback for [`run_bounded_git`]. Windows and other non-Unix targets +/// have no process-group teardown (`process_group(0)` / `kill(-pgid)` are Unix-only), +/// so this bounds a single child on its own: threads feed stdin and drain stderr +/// while the main thread drains stdout, and a watchdog thread kills the child at the +/// deadline (which closes stdout and unblocks the drain). The child is shared with +/// the watchdog behind a mutex that the main thread does NOT hold while draining, so +/// the watchdog can always acquire it to kill. Best-effort — it reaps only the direct +/// child, not a descendant group — which is why the hardened, group-aware path above +/// is gated to Unix, the only target the served node actually runs on (the Windows +/// release binary is best-effort / `continue-on-error` in CI). Kept in lockstep with +/// the Unix version's signature and result semantics so every caller compiles on all +/// targets (#174). +#[cfg(not(unix))] +fn run_bounded_git( + git_bin: &str, + args: &[&str], + repo_path: &Path, + stdin_bytes: &[u8], + deadline: Instant, +) -> Result> { + use std::io::{Read, Write}; + use std::sync::mpsc::RecvTimeoutError; + + let label = args.first().copied().unwrap_or("git"); + let mut child = std::process::Command::new(git_bin) + .args(args) + .current_dir(repo_path) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .with_context(|| format!("failed to spawn git {label}"))?; + + let mut stdin = child.stdin.take(); + let input = stdin_bytes.to_vec(); + let writer = std::thread::spawn(move || { + if let Some(mut s) = stdin.take() { + let _ = s.write_all(&input); + } + }); + let mut stderr = child.stderr.take().context("git stderr was not piped")?; + let err_reader = std::thread::spawn(move || { + let mut err = Vec::new(); + let _ = stderr.read_to_end(&mut err); + err + }); + let mut stdout = child.stdout.take().context("git stdout was not piped")?; + + // Share the child with the watchdog. The main thread drains stdout WITHOUT + // holding this lock, so the watchdog can always acquire it to kill on timeout; + // killing closes stdout and unblocks the drain below. + let child = std::sync::Arc::new(std::sync::Mutex::new(child)); + let (done_tx, done_rx) = mpsc::channel::<()>(); + let watchdog = { + let child = child.clone(); + std::thread::spawn(move || -> bool { + let wait = deadline.saturating_duration_since(Instant::now()); + match done_rx.recv_timeout(wait) { + Ok(()) | Err(RecvTimeoutError::Disconnected) => false, + Err(RecvTimeoutError::Timeout) => { + if let Ok(mut c) = child.lock() { + let _ = c.kill(); + } + true + } + } + }) + }; + + let mut out = Vec::new(); + let read_result = stdout.read_to_end(&mut out); + // The drain has returned (child exited or was killed), so taking the lock here + // cannot deadlock against the watchdog. + let status = child + .lock() + .expect("git child mutex poisoned") + .wait() + .context("git wait failed")?; + let _ = done_tx.send(()); + let killed = watchdog.join().unwrap_or(false); + let err = err_reader.join().unwrap_or_default(); + let _ = writer.join(); + read_result.context("failed to read git stdout")?; + if killed && !status.success() { + return Err(crate::git::smart_http::GitServiceTimeout.into()); + } + if !status.success() { + anyhow::bail!("git {label} failed: {}", String::from_utf8_lossy(&err)); + } + Ok(out) +} + /// Fail closed unless every ref ultimately resolves to a commit (a ref pointing /// directly at a blob or tree, or an annotated tag — even a nested one — of such /// an object is refused). `git rev-list --all` silently *skips* such refs, but From 7a6bd359eb915285ba4e459ae3339f0ed8716891 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 13 Jul 2026 10:11:41 -0500 Subject: [PATCH 22/58] fix(node): SIGKILL a SIGTERM-ignoring member of the visibility-walk process group (#174) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit jatmn/CodeRabbit P1: the walk watchdog stood down as soon as the group LEADER was reaped. A leader that exits cleanly on SIGTERM while a background member ignores SIGTERM and closes its inherited pipes let the main drain unblock, the leader be reaped, and the watchdog skip its SIGKILL escalation — so the handler returned while the descendant kept running. jatmn's suggested "continue until ESRCH" could not work: the main thread reaping the leader is what frees the pgid, so continuing to kill(-pgid) after that races a recycled group. Restructure the coordination instead: the main thread now defers reaping the leader until the watchdog thread returns. The watchdog always escalates through an unconditional group SIGKILL (no leader-reap short-circuit), and because the leader is still unreaped while every kill(-pgid) fires, the pgid cannot have been recycled. The grace before SIGKILL is a fixed budget (only paid on the exceptional timeout path). Regression run_bounded_git_sigkills_a_sigterm_ignoring_descendant_after_leader_exits: a leader that exits 0 on SIGTERM spawns an sh -c descendant (its own $$, not a ( ) subshell's parent pid) that ignores SIGTERM and closes its pipes; the descendant must be dead when the runner returns. RED on the old teardown (survives), GREEN now. The existing leader-ignores-SIGTERM and hung-walk teardown tests still pass. --- .../gitlawb-node/src/git/visibility_pack.rs | 162 +++++++++++++----- 1 file changed, 118 insertions(+), 44 deletions(-) diff --git a/crates/gitlawb-node/src/git/visibility_pack.rs b/crates/gitlawb-node/src/git/visibility_pack.rs index 0eaeed2d..ec55423e 100644 --- a/crates/gitlawb-node/src/git/visibility_pack.rs +++ b/crates/gitlawb-node/src/git/visibility_pack.rs @@ -23,6 +23,12 @@ use std::time::{Duration, Instant}; #[cfg(test)] const WALK_TIMEOUT: Duration = Duration::from_secs(600); +/// How long the process-group watchdog waits after SIGTERM before escalating to +/// SIGKILL, giving a well-behaved git child time to clean up its `*.lock` files. Only +/// paid on a timeout (already the exceptional path). +#[cfg(unix)] +const WATCHDOG_TERM_GRACE: Duration = Duration::from_secs(1); + /// Run one git child under a shared `deadline` with process-group teardown, /// BLOCKING, and return its stdout. The child runs in its own process group; a /// watchdog thread SIGTERMs (lets git clean up its `*.lock` files), then SIGKILLs, @@ -60,52 +66,46 @@ fn run_bounded_git( // With process_group(0) the child leads its own group, so pgid == its pid. let pgid = child.id() as i32; - // Watchdog: on the deadline, SIGTERM the whole group, poll until it exits, - // escalate to SIGKILL after a grace, and report whether it killed. Signalled to - // stand down when the child is reaped in time. Kept off the main thread because - // the main thread's stdout drain is exactly what blocks until a hung child is - // torn down. + // Watchdog: on the deadline, tear the WHOLE process group down — SIGTERM, a grace + // for a well-behaved child to clean up its `*.lock` files, then an UNCONDITIONAL + // SIGKILL of the group. It never stands down on leader-reap alone: a group member + // that ignores SIGTERM while the leader exits cleanly would otherwise escape the + // SIGKILL and keep running past the deadline (finding 3, #174). The main thread + // defers reaping the leader until this thread returns (see below), so the leader's + // pid is still unreaped while every `kill(-pgid)` fires and the pgid cannot have + // been recycled — which is why this no longer needs the old `reaped` short-circuit. + // Kept off the main thread because the main thread's stdout drain is exactly what + // blocks until a hung child is torn down. let (done_tx, done_rx) = mpsc::channel::<()>(); - // Set true the instant the main thread reaps the child, so the watchdog never - // SIGTERMs a pgid whose leader is already reaped and whose pid the kernel may - // recycle (the reused-pgid hazard smart_http guards via disarm-after-wait). - let reaped = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); - let watchdog = { - let reaped = reaped.clone(); - std::thread::spawn(move || -> bool { - use std::sync::atomic::Ordering; - let wait = deadline.saturating_duration_since(Instant::now()); - match done_rx.recv_timeout(wait) { - Ok(()) | Err(RecvTimeoutError::Disconnected) => false, - Err(RecvTimeoutError::Timeout) => { - // The child was reaped right as the deadline fired: do not signal - // its (possibly-recycled) pgid. - if reaped.load(Ordering::SeqCst) { - return false; - } - // SAFETY: kill(2) takes only integers and borrows no Rust memory; - // ESRCH on an already-gone group is ignored. - unsafe { libc::kill(-pgid, libc::SIGTERM) }; - for step in 0..200u32 { - if reaped.load(Ordering::SeqCst) || unsafe { libc::kill(-pgid, 0) } != 0 { - return true; // reaped, or ESRCH: every group member has exited - } - if step == 100 { - unsafe { libc::kill(-pgid, libc::SIGKILL) }; - } - std::thread::sleep(Duration::from_millis(10)); - } - // Survived SIGKILL past the cap: a wedged (D-state) git, the same - // condition smart_http's reap logs. Warn so an operator sees it. + let watchdog = std::thread::spawn(move || -> bool { + let wait = deadline.saturating_duration_since(Instant::now()); + match done_rx.recv_timeout(wait) { + Ok(()) | Err(RecvTimeoutError::Disconnected) => false, + Err(RecvTimeoutError::Timeout) => { + // SAFETY: kill(2) takes only integers and borrows no Rust memory; + // ESRCH on an already-gone group is ignored. + unsafe { libc::kill(-pgid, libc::SIGTERM) }; + // Fixed grace: because the main thread defers the leader's reap, a + // fully-exited group still shows a zombie leader here, so polling for + // ESRCH cannot detect early completion — just wait the grace, then + // SIGKILL. On a group of only zombies the SIGKILL is a harmless no-op; + // on a SIGTERM-ignoring member it is what actually kills it. + std::thread::sleep(WATCHDOG_TERM_GRACE); + unsafe { libc::kill(-pgid, libc::SIGKILL) }; + // Brief settle so the SIGKILL is delivered before the main thread + // reaps the leader and frees the pgid. A wedged (D-state) member + // survives even SIGKILL — the documented residual, as in smart_http. + std::thread::sleep(Duration::from_millis(20)); + if unsafe { libc::kill(-pgid, 0) } == 0 { tracing::warn!( pgid, "withheld-walk git survived SIGKILL past the watchdog cap (uninterruptible I/O?)" ); - true } + true } - }) - }; + } + }); // Feed stdin on a writer thread and drain stderr on a reader thread so the main // thread can drain stdout concurrently; writing all of stdin (or draining one @@ -132,14 +132,18 @@ fn run_bounded_git( // is unreachable in userspace (no signal reaps a D-state process) and matches the // async `reap_group_on_timeout`, which likewise only warns and gives up there. let read_result = stdout.read_to_end(&mut out); + // The drain has returned: either the child exited on its own, or the watchdog's + // teardown closed its pipes. Tell the watchdog the drain is done, then WAIT for it + // to return BEFORE reaping the leader. On the timeout path the watchdog runs the + // full SIGTERM -> grace -> SIGKILL teardown; joining it before `child.wait()` keeps + // the leader's pid unreaped while every `kill(-pgid)` fires (so the pgid can't be + // recycled), and guarantees a SIGTERM-ignoring group member has already been + // SIGKILLed rather than left running past the deadline (finding 3, #174). + let _ = done_tx.send(()); + let killed = watchdog.join().unwrap_or(false); let status = child.wait().context("git wait failed")?; - // Reaped: bar the watchdog from signalling the now-reaped (possibly recycled) - // pgid before it can fire, then stand it down. - reaped.store(true, std::sync::atomic::Ordering::SeqCst); let err = err_reader.join().unwrap_or_default(); let _ = writer.join(); - let _ = done_tx.send(()); - let killed = watchdog.join().unwrap_or(false); read_result.context("failed to read git stdout")?; // The watchdog runs off a wall clock that can race a child finishing right at the // deadline. A child that exited on its own (success) is not a timeout even if the @@ -828,6 +832,76 @@ mod tests { "the SIGTERM-ignoring child (pid {pid}) must be reaped via SIGKILL, not left running" ); } + + /// #174 finding 3 (jatmn/CodeRabbit): a group MEMBER that ignores SIGTERM must + /// still be SIGKILLed even when the group LEADER exits cleanly on SIGTERM. The + /// leader traps SIGTERM to exit 0, but first spawns a descendant (`sh -c`, so its + /// `$$` is its OWN pid — a `( )` subshell's `$$` is the parent's) that ignores + /// SIGTERM and closes its inherited stdout/stderr. When the watchdog SIGTERMs the + /// group, the leader exits, its stdout closes, the main drain unblocks, and the + /// leader is reaped — the exact window a `reaped`-gated watchdog stands down in, + /// before escalating to SIGKILL. The descendant must be dead when run_bounded_git + /// returns; a teardown that stands down on leader-reap leaves it running (RED). + #[cfg(unix)] + #[test] + fn run_bounded_git_sigkills_a_sigterm_ignoring_descendant_after_leader_exits() { + use std::time::{Duration, Instant}; + let tmp = TempDir::new().unwrap(); + // Both loops are bounded (~30s) so a broken teardown cannot leak a permanent + // orphan or wedge the suite; the assertion fires well before then. + let body = "#!/bin/sh\n\ +case \"$1\" in\n\ + rev-list)\n\ + sh -c 'trap \"\" TERM; echo $$ > desc.pid; exec 1>&- 2>&-; i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done' &\n\ + trap 'exit 0' TERM\n\ + i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done ;;\n\ + *) : ;;\n\ +esac\n"; + let git_bin = write_fake_git(tmp.path(), body); + let (tx, rx) = std::sync::mpsc::channel(); + let path = tmp.path().to_path_buf(); + std::thread::spawn(move || { + let _ = tx.send(run_bounded_git( + &git_bin, + &["rev-list"], + &path, + b"", + Instant::now() + Duration::from_millis(100), + )); + }); + let _ = rx + .recv_timeout(Duration::from_secs(10)) + .expect("run_bounded_git must return within the watchdog budget"); + + // Wait for the descendant to record its OWN pid, then assert it is gone. + let desc_pid_path = tmp.path().join("desc.pid"); + let mut desc: Option = None; + for _ in 0..200 { + if let Some(p) = std::fs::read_to_string(&desc_pid_path) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + desc = Some(p); + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + let desc = desc.expect("the fake leader must have spawned and recorded a descendant"); + let mut gone = false; + for _ in 0..300 { + if unsafe { libc::kill(desc, 0) } != 0 { + gone = true; + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + // Kill it regardless so a RED run leaks no orphan. + unsafe { libc::kill(desc, libc::SIGKILL) }; + assert!( + gone, + "a SIGTERM-ignoring descendant (pid {desc}) must be SIGKILLed even after the leader exits cleanly, not orphaned" + ); + } use crate::db::VisibilityMode; use chrono::Utc; use std::process::Command; From 9412aa9c5bdc56d5c27cbf5be3dfebea995e7039 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 13 Jul 2026 10:34:44 -0500 Subject: [PATCH 23/58] fix(node): bound and reap push-candidate git children under the write permit (#174) Route push_delta's candidate-discovery git children (cat-file / rev-list) through the bounded run_bounded_git (deadline + process-group SIGKILL teardown) instead of bare Command::output(). A hung scan no longer pins the git-receive-pack write permit past the deadline or a client disconnect. Threads git_bin + a shared per-scan deadline (git_service_timeout_secs) through resolve_candidates_for_push and all_blob_oids. --- crates/gitlawb-node/src/api/repos.rs | 4 +- crates/gitlawb-node/src/git/push_delta.rs | 278 ++++++++++++------ .../gitlawb-node/src/git/visibility_pack.rs | 11 +- 3 files changed, 193 insertions(+), 100 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 719246e0..ea6f126e 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -115,7 +115,7 @@ async fn fail_closed_full_scan_objects( let allowed = crate::git::visibility_pack::replicable_blob_set_bounded( &disk_path, &git_bin, timeout, &rules, is_public, &owner_did, )?; - let all_blobs = crate::git::push_delta::all_blob_oids(&disk_path)?; + let all_blobs = crate::git::push_delta::all_blob_oids(&disk_path, &git_bin, std::time::Instant::now() + timeout)?; Ok(crate::git::visibility_pack::replicable_objects_fail_closed( candidates, &allowed, &all_blobs, )) @@ -1264,6 +1264,8 @@ pub async fn git_receive_pack( disk_path.clone(), new_tips, old_tips, + state.git_bin.clone(), + std::time::Duration::from_secs(state.config.git_service_timeout_secs), ) .await; if pin_set.full_scan { diff --git a/crates/gitlawb-node/src/git/push_delta.rs b/crates/gitlawb-node/src/git/push_delta.rs index 7ab00816..51b6f836 100644 --- a/crates/gitlawb-node/src/git/push_delta.rs +++ b/crates/gitlawb-node/src/git/push_delta.rs @@ -29,9 +29,9 @@ use std::collections::HashSet; use std::path::{Path, PathBuf}; -use std::process::Command; +use std::time::{Duration, Instant}; -use anyhow::{bail, Context, Result}; +use anyhow::Result; /// Env var that forces the push path to always full-scan, bypassing the delta /// optimization (KTD7 kill-switch). Reuses the already-tested fallback branch @@ -64,11 +64,21 @@ pub enum PinCandidates { /// Fail-closed: any condition where the introduced set cannot be safely /// determined returns [`PinCandidates::FullScanRequired`] rather than a partial /// set, so the caller full-scans instead of silently under-pinning. -pub fn resolve_push_delta(repo_path: &Path, new_tips: &[&str], old_tips: &[&str]) -> PinCandidates { - // KTD7 kill-switch: force the (already-tested) full-scan fallback. The env - // read is split out from the pure logic so the resolver stays unit-testable - // without touching process-global state. - resolve_push_delta_inner(repo_path, new_tips, old_tips, force_full_scan()) +pub fn resolve_push_delta( + repo_path: &Path, + new_tips: &[&str], + old_tips: &[&str], + git_bin: &str, + deadline: Instant, +) -> PinCandidates { + resolve_push_delta_inner( + repo_path, + new_tips, + old_tips, + force_full_scan(), + git_bin, + deadline, + ) } /// Whether the force-full-scan kill-switch env var is set. @@ -83,6 +93,8 @@ fn resolve_push_delta_inner( new_tips: &[&str], old_tips: &[&str], force_full_scan: bool, + git_bin: &str, + deadline: Instant, ) -> PinCandidates { if force_full_scan { tracing::debug!("{FORCE_FULL_SCAN_ENV} set — forcing full scan"); @@ -102,7 +114,7 @@ fn resolve_push_delta_inner( // commit). Bare `cat-file -t` returns `tag` for an annotated tag, and // `for-each-ref %(*objecttype)` peels only one level — neither is correct. for tip in new_tips { - match peeled_object_type(repo_path, tip) { + match peeled_object_type(repo_path, tip, git_bin, deadline) { Some(t) if t == "commit" => {} other => { tracing::debug!( @@ -115,7 +127,7 @@ fn resolve_push_delta_inner( } } - match rev_list_delta(repo_path, new_tips, old_tips) { + match rev_list_delta(repo_path, new_tips, old_tips, git_bin, deadline) { Ok(oids) => PinCandidates::Delta(oids), Err(e) => { tracing::debug!(err = %e, "push-delta rev-list failed — forcing full scan"); @@ -126,41 +138,43 @@ fn resolve_push_delta_inner( /// Return the fully-peeled object type of `sha` (e.g. `commit`, `tree`, /// `blob`), or `None` if the object is missing/unpeelable or git errored. -fn peeled_object_type(repo_path: &Path, sha: &str) -> Option { - let output = Command::new("git") - .args(["cat-file", "-t", &format!("{sha}^{{}}")]) - .current_dir(repo_path) - .output() - .ok()?; - if !output.status.success() { - return None; - } - Some(String::from_utf8_lossy(&output.stdout).trim().to_string()) +fn peeled_object_type( + repo_path: &Path, + sha: &str, + git_bin: &str, + deadline: Instant, +) -> Option { + let peel = format!("{sha}^{{}}"); + let out = crate::git::visibility_pack::run_bounded_git( + git_bin, + &["cat-file", "-t", &peel], + repo_path, + b"", + deadline, + ) + .ok()?; + Some(String::from_utf8_lossy(&out).trim().to_string()) } /// Run `git rev-list --objects --no-object-names --not ` and return /// the bare OID set. Decides on `status.success()` *before* parsing stdout, so /// a walk that prints a valid prefix then errors mid-walk is discarded. -fn rev_list_delta(repo_path: &Path, new_tips: &[&str], old_tips: &[&str]) -> Result> { +fn rev_list_delta( + repo_path: &Path, + new_tips: &[&str], + old_tips: &[&str], + git_bin: &str, + deadline: Instant, +) -> Result> { let mut args: Vec<&str> = vec!["rev-list", "--objects", "--no-object-names"]; args.extend_from_slice(new_tips); if !old_tips.is_empty() { args.push("--not"); args.extend_from_slice(old_tips); } - - let output = Command::new("git") - .args(&args) - .current_dir(repo_path) - .output() - .context("failed to run git rev-list for push delta")?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - bail!("git rev-list failed: {stderr}"); - } - - let stdout = String::from_utf8_lossy(&output.stdout); + let out = + crate::git::visibility_pack::run_bounded_git(git_bin, &args, repo_path, b"", deadline)?; + let stdout = String::from_utf8_lossy(&out); Ok(stdout .lines() .map(|l| l.trim().to_string()) @@ -175,23 +189,19 @@ fn rev_list_delta(repo_path: &Path, new_tips: &[&str], old_tips: &[&str]) -> Res /// reconciliation sweep relies on. It returns *all* objects (including /// unreachable/dangling ones), which is what the sweep needs to catch /// stragglers — do not swap it for a reachability walk. -pub fn list_all_objects(repo_path: &Path) -> Result> { - let output = Command::new("git") - .args([ +pub fn list_all_objects(repo_path: &Path, git_bin: &str, deadline: Instant) -> Result> { + let out = crate::git::visibility_pack::run_bounded_git( + git_bin, + &[ "cat-file", "--batch-all-objects", "--batch-check=%(objectname)", - ]) - .current_dir(repo_path) - .output() - .context("failed to run git cat-file")?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - bail!("git cat-file failed: {stderr}"); - } - - let stdout = String::from_utf8_lossy(&output.stdout); + ], + repo_path, + b"", + deadline, + )?; + let stdout = String::from_utf8_lossy(&out); Ok(stdout .lines() .map(|l| l.trim().to_string()) @@ -203,23 +213,23 @@ pub fn list_all_objects(repo_path: &Path) -> Result> { /// `--batch-check='%(objectname) %(objecttype)'`. The pin path's fail-closed /// filter needs to tell blobs (content, withholdable) from commits/trees /// (structural, never withheld) without typing the candidate list itself. -pub fn list_all_objects_with_type(repo_path: &Path) -> Result> { - let output = Command::new("git") - .args([ +pub fn list_all_objects_with_type( + repo_path: &Path, + git_bin: &str, + deadline: Instant, +) -> Result> { + let out = crate::git::visibility_pack::run_bounded_git( + git_bin, + &[ "cat-file", "--batch-all-objects", "--batch-check=%(objectname) %(objecttype)", - ]) - .current_dir(repo_path) - .output() - .context("failed to run git cat-file")?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - bail!("git cat-file failed: {stderr}"); - } - - let stdout = String::from_utf8_lossy(&output.stdout); + ], + repo_path, + b"", + deadline, + )?; + let stdout = String::from_utf8_lossy(&out); Ok(stdout .lines() .filter_map(|l| { @@ -235,8 +245,12 @@ pub fn list_all_objects_with_type(repo_path: &Path) -> Result Result> { - Ok(list_all_objects_with_type(repo_path)? +pub fn all_blob_oids( + repo_path: &Path, + git_bin: &str, + deadline: Instant, +) -> Result> { + Ok(list_all_objects_with_type(repo_path, git_bin, deadline)? .into_iter() .filter(|(_, ty)| ty == "blob") .map(|(oid, _)| oid) @@ -272,18 +286,22 @@ pub async fn resolve_candidates_for_push( repo_path: PathBuf, new_tips: Vec, old_tips: Vec, + git_bin: String, + timeout: Duration, ) -> PinCandidateSet { tokio::task::spawn_blocking(move || { + // ONE shared deadline for the whole scan, per jatmn ("the same deadline"). + let deadline = Instant::now() + timeout; let new_refs: Vec<&str> = new_tips.iter().map(String::as_str).collect(); let old_refs: Vec<&str> = old_tips.iter().map(String::as_str).collect(); - match resolve_push_delta(&repo_path, &new_refs, &old_refs) { + match resolve_push_delta(&repo_path, &new_refs, &old_refs, &git_bin, deadline) { PinCandidates::Delta(objs) => { tracing::info!(delta = objs.len(), repo = %repo_path.display(), "pin candidate set from push delta"); PinCandidateSet { candidates: objs, full_scan: false } } PinCandidates::FullScanRequired => { tracing::warn!(repo = %repo_path.display(), "pin delta unavailable (non-commit tip, git error, or force-full-scan) — full-scan fallback"); - match list_all_objects(&repo_path) { + match list_all_objects(&repo_path, &git_bin, deadline) { Ok(objs) => PinCandidateSet { candidates: objs, full_scan: true }, Err(e) => { tracing::warn!(repo = %repo_path.display(), err = %e, "full-scan fallback failed; pinning nothing this push (reconciliation sweep backstops)"); @@ -304,8 +322,13 @@ pub async fn resolve_candidates_for_push( mod tests { use super::*; use std::collections::HashSet; + use std::process::Command; use tempfile::TempDir; + fn td() -> std::time::Instant { + std::time::Instant::now() + std::time::Duration::from_secs(600) + } + /// Minimal git helper for building test repos. struct Repo { _td: TempDir, @@ -363,9 +386,10 @@ mod tests { let repo = Repo::new(); let c1 = repo.commit_file("a.txt", "one\n"); let c2 = repo.commit_file("b.txt", "two\n"); - let got: HashSet = delta(resolve_push_delta(&repo.path, &[&c2], &[&c1])) - .into_iter() - .collect(); + let got: HashSet = + delta(resolve_push_delta(&repo.path, &[&c2], &[&c1], "git", td())) + .into_iter() + .collect(); // The new blob b.txt and commit c2 are in the delta; the old blob a.txt // and commit c1 are not. let new_blob = repo.rev("HEAD:b.txt"); @@ -383,7 +407,7 @@ mod tests { // genuinely new objects, never fewer. let repo = Repo::new(); let c1 = repo.commit_file("a.txt", "one\n"); - let got: HashSet = delta(resolve_push_delta(&repo.path, &[&c1], &[])) + let got: HashSet = delta(resolve_push_delta(&repo.path, &[&c1], &[], "git", td())) .into_iter() .collect(); assert!(got.contains(&c1)); @@ -399,9 +423,15 @@ mod tests { // Rewrite history: reset to base, commit a different file. repo.git(&["reset", "-q", "--hard", &base]); let new_tip = repo.commit_file("c.txt", "three\n"); - let got: HashSet = delta(resolve_push_delta(&repo.path, &[&new_tip], &[&old_tip])) - .into_iter() - .collect(); + let got: HashSet = delta(resolve_push_delta( + &repo.path, + &[&new_tip], + &[&old_tip], + "git", + td(), + )) + .into_iter() + .collect(); assert!(got.contains(&new_tip), "new tip in delta"); assert!(got.contains(&repo.rev(&format!("{new_tip}:c.txt")))); // No error; force-push computes new-minus-old cleanly. @@ -413,7 +443,7 @@ mod tests { repo.commit_file("a.txt", "one\n"); // All updates were deletions => new_tips empty after the caller strips zeros. assert_eq!( - resolve_push_delta(&repo.path, &[], &[ZERO]), + resolve_push_delta(&repo.path, &[], &[ZERO], "git", td()), PinCandidates::Delta(Vec::new()) ); } @@ -424,7 +454,7 @@ mod tests { repo.commit_file("a.txt", "one\n"); let blob = repo.rev("HEAD:a.txt"); assert_eq!( - resolve_push_delta(&repo.path, &[&blob], &[]), + resolve_push_delta(&repo.path, &[&blob], &[], "git", td()), PinCandidates::FullScanRequired, "a blob tip must force full scan (rev-list would exit 0 and walk it)" ); @@ -436,7 +466,7 @@ mod tests { repo.commit_file("a.txt", "one\n"); let tree = repo.rev("HEAD^{tree}"); assert_eq!( - resolve_push_delta(&repo.path, &[&tree], &[]), + resolve_push_delta(&repo.path, &[&tree], &[], "git", td()), PinCandidates::FullScanRequired ); } @@ -449,7 +479,7 @@ mod tests { repo.git(&["tag", "-a", "treetag", "-m", "x", &tree]); let tag = repo.rev("treetag"); assert_eq!( - resolve_push_delta(&repo.path, &[&tag], &[]), + resolve_push_delta(&repo.path, &[&tag], &[], "git", td()), PinCandidates::FullScanRequired, "annotated tag peeling to a tree must force full scan" ); @@ -465,7 +495,7 @@ mod tests { repo.git(&["tag", "-a", "t2", "-m", "x", &t1]); let t2 = repo.rev("t2"); assert_eq!( - resolve_push_delta(&repo.path, &[&t2], &[]), + resolve_push_delta(&repo.path, &[&t2], &[], "git", td()), PinCandidates::FullScanRequired ); } @@ -480,7 +510,7 @@ mod tests { let c1 = repo.commit_file("a.txt", "one\n"); repo.git(&["tag", "-a", "rel", "-m", "release", &c1]); let tag = repo.rev("rel"); - let got: HashSet = delta(resolve_push_delta(&repo.path, &[&tag], &[])) + let got: HashSet = delta(resolve_push_delta(&repo.path, &[&tag], &[], "git", td())) .into_iter() .collect(); assert!( @@ -501,7 +531,7 @@ mod tests { let t1 = repo.rev("t1"); repo.git(&["tag", "-a", "t2", "-m", "x", &t1]); let t2 = repo.rev("t2"); - let got: HashSet = delta(resolve_push_delta(&repo.path, &[&t2], &[])) + let got: HashSet = delta(resolve_push_delta(&repo.path, &[&t2], &[], "git", td())) .into_iter() .collect(); assert!( @@ -516,9 +546,14 @@ mod tests { let repo = Repo::new(); let c1 = repo.commit_file("a.txt", "one\n"); let c2 = repo.commit_file("b.txt", "two\n"); - let set = - resolve_candidates_for_push(repo.path.clone(), vec![c2.clone()], vec![c1.clone()]) - .await; + let set = resolve_candidates_for_push( + repo.path.clone(), + vec![c2.clone()], + vec![c1.clone()], + "git".to_string(), + std::time::Duration::from_secs(600), + ) + .await; assert!(!set.full_scan, "happy-path delta is not a full scan"); let got: HashSet = set.candidates.into_iter().collect(); let new_blob = repo.rev("HEAD:b.txt"); @@ -536,8 +571,18 @@ mod tests { let repo = Repo::new(); repo.commit_file("a.txt", "one\n"); let blob = repo.rev("HEAD:a.txt"); - let all: HashSet = list_all_objects(&repo.path).unwrap().into_iter().collect(); - let set = resolve_candidates_for_push(repo.path.clone(), vec![blob], vec![]).await; + let all: HashSet = list_all_objects(&repo.path, "git", td()) + .unwrap() + .into_iter() + .collect(); + let set = resolve_candidates_for_push( + repo.path.clone(), + vec![blob], + vec![], + "git".to_string(), + std::time::Duration::from_secs(600), + ) + .await; assert!(set.full_scan, "non-commit tip is signalled as a full scan"); let got: HashSet = set.candidates.into_iter().collect(); assert_eq!(got, all, "non-commit tip falls back to full repo scan"); @@ -549,7 +594,7 @@ mod tests { repo.commit_file("a.txt", "one\n"); let bogus = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef"; assert_eq!( - resolve_push_delta(&repo.path, &[bogus], &[]), + resolve_push_delta(&repo.path, &[bogus], &[], "git", td()), PinCandidates::FullScanRequired, "a missing/corrupt tip OID must force full scan" ); @@ -564,7 +609,7 @@ mod tests { let c1 = repo.commit_file("a.txt", "one\n"); let c2 = repo.commit_file("b.txt", "two\n"); let old_tree = repo.rev(&format!("{c1}^{{tree}}")); - let result = resolve_push_delta(&repo.path, &[&c2], &[&old_tree]); + let result = resolve_push_delta(&repo.path, &[&c2], &[&old_tree], "git", td()); match result { PinCandidates::FullScanRequired => {} // safe: caller full-scans PinCandidates::Delta(objs) => { @@ -583,10 +628,15 @@ mod tests { // branch2 from base advances independently repo.git(&["checkout", "-q", "-b", "branch2", &base]); let b2 = repo.commit_file("c.txt", "three\n"); - let got: HashSet = - delta(resolve_push_delta(&repo.path, &[&b1, &b2], &[&base, &base])) - .into_iter() - .collect(); + let got: HashSet = delta(resolve_push_delta( + &repo.path, + &[&b1, &b2], + &[&base, &base], + "git", + td(), + )) + .into_iter() + .collect(); assert!(got.contains(&b1), "branch1 new commit"); assert!(got.contains(&b2), "branch2 new commit"); } @@ -595,7 +645,7 @@ mod tests { fn empty_repo_no_tips_yields_empty_delta() { let repo = Repo::new(); assert_eq!( - resolve_push_delta(&repo.path, &[], &[]), + resolve_push_delta(&repo.path, &[], &[], "git", td()), PinCandidates::Delta(Vec::new()) ); } @@ -609,12 +659,12 @@ mod tests { let repo = Repo::new(); let c1 = repo.commit_file("a.txt", "one\n"); assert_eq!( - resolve_push_delta_inner(&repo.path, &[&c1], &[], true), + resolve_push_delta_inner(&repo.path, &[&c1], &[], true, "git", td()), PinCandidates::FullScanRequired ); // And with the flag off, the same push yields a Delta. assert!(matches!( - resolve_push_delta_inner(&repo.path, &[&c1], &[], false), + resolve_push_delta_inner(&repo.path, &[&c1], &[], false, "git", td()), PinCandidates::Delta(_) )); } @@ -624,7 +674,7 @@ mod tests { let repo = Repo::new(); repo.commit_file("a.txt", "one\n"); repo.commit_file("b.txt", "two\n"); - let all = list_all_objects(&repo.path).unwrap(); + let all = list_all_objects(&repo.path, "git", td()).unwrap(); // 2 commits + 2 trees + 2 blobs = 6 objects. assert_eq!(all.len(), 6, "got: {all:?}"); } @@ -642,7 +692,7 @@ mod tests { std::fs::write(repo.path.join("orphan.bin"), b"dangling\n").unwrap(); let dangling = repo.git(&["hash-object", "-w", "orphan.bin"]); - let blobs = all_blob_oids(&repo.path).unwrap(); + let blobs = all_blob_oids(&repo.path, "git", td()).unwrap(); assert!(blobs.contains(&reachable_blob), "reachable blob present"); assert!( blobs.contains(&dangling), @@ -652,7 +702,7 @@ mod tests { assert!(!blobs.contains(&tree), "tree is not a blob"); // The typed lister tags each object; spot-check the dangling blob's type. - let typed = list_all_objects_with_type(&repo.path).unwrap(); + let typed = list_all_objects_with_type(&repo.path, "git", td()).unwrap(); assert!( typed .iter() @@ -660,4 +710,40 @@ mod tests { "dangling object is typed as a blob" ); } + + #[cfg(unix)] + #[test] + fn list_all_objects_times_out_on_a_hung_git_instead_of_blocking() { + use std::os::unix::fs::PermissionsExt; + use std::time::{Duration, Instant}; + let dir = tempfile::TempDir::new().unwrap(); + // Fake git: hang on cat-file (bounded to 30s so a broken test can't wedge). + let fake = dir.path().join("fakegit"); + std::fs::write( + &fake, + "#!/bin/sh\ncase \"$1\" in\n cat-file) i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done ;;\n *) : ;;\nesac\n", + ) + .unwrap(); + let mut perm = std::fs::metadata(&fake).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&fake, perm).unwrap(); + let git_bin = fake.to_str().unwrap().to_string(); + + let (tx, rx) = std::sync::mpsc::channel(); + let path = dir.path().to_path_buf(); + std::thread::spawn(move || { + let _ = tx.send(list_all_objects( + &path, + &git_bin, + Instant::now() + Duration::from_millis(150), + )); + }); + let res = rx.recv_timeout(Duration::from_secs(10)).expect( + "list_all_objects must return within the watchdog budget, not block on a hung git", + ); + assert!( + res.is_err(), + "a hung git must make list_all_objects error out, not hang" + ); + } } diff --git a/crates/gitlawb-node/src/git/visibility_pack.rs b/crates/gitlawb-node/src/git/visibility_pack.rs index ec55423e..9f40cd4b 100644 --- a/crates/gitlawb-node/src/git/visibility_pack.rs +++ b/crates/gitlawb-node/src/git/visibility_pack.rs @@ -42,7 +42,7 @@ const WATCHDOG_TERM_GRACE: Duration = Duration::from_secs(1); /// drive the teardown in tests without mutating the process-global PATH; /// `stdin_bytes` feeds children that read stdin (empty for the arg-only children). #[cfg(unix)] -fn run_bounded_git( +pub(crate) fn run_bounded_git( git_bin: &str, args: &[&str], repo_path: &Path, @@ -171,7 +171,7 @@ fn run_bounded_git( /// the Unix version's signature and result semantics so every caller compiles on all /// targets (#174). #[cfg(not(unix))] -fn run_bounded_git( +pub(crate) fn run_bounded_git( git_bin: &str, args: &[&str], repo_path: &Path, @@ -1235,7 +1235,12 @@ esac\n"; String::from_utf8_lossy(&out.stdout).trim().to_string() }; - let all_blobs = crate::git::push_delta::all_blob_oids(&work).unwrap(); + let all_blobs = crate::git::push_delta::all_blob_oids( + &work, + "git", + std::time::Instant::now() + std::time::Duration::from_secs(600), + ) + .unwrap(); assert!( all_blobs.contains(&dangling_oid), "precondition: the dangling blob is in the all-objects universe" From 4410b834df04163c6121629e3e3b05b227501e27 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 13 Jul 2026 10:46:24 -0500 Subject: [PATCH 24/58] docs(node): correct concurrency + timeout docs to match the final git routing (#174) jatmn P2 (INV-24): the config help, .env.example, and README described routing the handler no longer does. - The read pool serves upload-pack and the UPLOAD-PACK info/refs advertisement only; the anon receive-pack info/refs advertisement draws from its own dedicated pool (sized like the write pool but disjoint), not the read pool and not the write pool. - The write pool is the authenticated push POST only. - The per-caller read cap is keyed on the resolved source IP, never the DID: a signature does not move a caller off it (the docs claimed a per-DID budget, which would tell a NATed authenticated client it escapes the shared cap when it does not). - git_service_timeout_secs now also bounds the push-side pin-candidate discovery (rev-list / cat-file), reaped via process-group teardown, alongside the info/refs advertisements and the withheld-blob walk. --- .env.example | 40 ++++++++++++--------- README.md | 2 +- crates/gitlawb-node/src/config.rs | 60 +++++++++++++++++-------------- 3 files changed, 57 insertions(+), 45 deletions(-) diff --git a/.env.example b/.env.example index cbf3db3a..0ef193f8 100644 --- a/.env.example +++ b/.env.example @@ -109,29 +109,35 @@ GITLAWB_MAX_PACK_BYTES=2147483648 # Max seconds a served git upload-pack / receive-pack (clone / push) may run # before it is aborted with a 504. Bounds a hung git that would otherwise pin a -# worker and, on push, the repo write lock. Also bounds the info/refs -# advertisement and the withheld-blob pack build (#174). Default 600. +# worker and, on push, the repo write lock. Also bounds both info/refs +# advertisements, the withheld-blob pack build, and the push-side candidate +# discovery (rev-list / cat-file), all reaped via process-group teardown (#174). +# Default 600. GITLAWB_GIT_SERVICE_TIMEOUT_SECS=600 -# Max concurrent git read ops (upload-pack + info/refs advertisements) served at -# once, a global pool separate from the push pool below. Over-cap sheds a clean -# 503 + Retry-After. Anonymous reads draw from here, so pair it with -# GITLAWB_MAX_CONCURRENT_READS_PER_CALLER (below) so one caller cannot monopolize -# the pool. Default 128. +# Max concurrent git READ ops (upload-pack + the upload-pack info/refs +# advertisement) served at once, a global pool separate from the push pool below. +# The anon receive-pack info/refs advertisement has its OWN pool (see below), not +# this one. Over-cap sheds a clean 503 + Retry-After. Anonymous reads draw from +# here, so pair it with GITLAWB_MAX_CONCURRENT_READS_PER_CALLER (below) so one +# caller cannot monopolize the pool. Default 128. GITLAWB_MAX_CONCURRENT_GIT_OPS=128 -# Max concurrent git-receive-pack (push) operations, in a pool separate from the -# read pool (GITLAWB_MAX_CONCURRENT_GIT_OPS) so anonymous reads cannot shed an -# authenticated push at admission. Over-cap sheds a 503 + Retry-After. Default 32. +# Max concurrent git-receive-pack (push) POST operations, in a pool separate from +# the read pool (GITLAWB_MAX_CONCURRENT_GIT_OPS) so anonymous reads cannot shed an +# authenticated push at admission. The anon receive-pack info/refs advertisement +# runs in a SEPARATE pool of the same size (disjoint from this one), so an +# advertisement flood cannot shed a push either. Over-cap sheds a 503 + +# Retry-After. Default 32. GITLAWB_MAX_CONCURRENT_GIT_PUSHES=32 -# Max concurrent read ops (upload-pack + info/refs advertisements) a single caller -# may hold, so one caller cannot monopolize the read pool. Keyed per-DID when -# signed, else per-source-IP. The per-IP key is only as granular as -# GITLAWB_TRUSTED_PROXY below: left unset, a node behind an edge/NAT keys all -# anonymous callers on the edge IP and this collapses to one global anonymous cap. -# Set GITLAWB_TRUSTED_PROXY for per-client keying; a high-fanout caller (CI behind -# one NAT) should authenticate for a per-DID budget or the operator raises this. +# Max concurrent read ops (upload-pack + the upload-pack info/refs advertisement) +# a single caller may hold, so one caller cannot monopolize the read pool. Keyed +# on the resolved SOURCE IP, never the DID: a signature does not move a caller off +# this cap. The source-IP key is only as granular as GITLAWB_TRUSTED_PROXY below: +# left unset, a node behind an edge/NAT keys all callers on the edge IP and this +# collapses to one global cap. Set GITLAWB_TRUSTED_PROXY for per-client keying; a +# high-fanout caller (CI behind one NAT) then needs the operator to raise this. # Default 16. GITLAWB_MAX_CONCURRENT_READS_PER_CALLER=16 diff --git a/README.md b/README.md index 0f760449..542a988e 100644 --- a/README.md +++ b/README.md @@ -343,7 +343,7 @@ Important node settings: | `GITLAWB_REQUIRE_SIGNED_PEER_WRITES` | Require signed peer announce/sync writes. | | `GITLAWB_AUTO_SYNC` | Enable automatic sync from known peers. | | `GITLAWB_MAX_PACK_BYTES` | Max git pack body size for smart-HTTP routes. | -| `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack, receive-pack, or `info/refs` advertisement may run before it is aborted (504). Default 600. Also bounds the withheld-blob classification walk (on both the upload-pack serve and receive-pack replication paths), which is reaped at the deadline. | +| `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack, receive-pack, or `info/refs` advertisement may run before it is aborted (504). Default 600. Also bounds the withheld-blob classification walk (on both the upload-pack serve and receive-pack replication paths) and the push-side pin-candidate discovery (`rev-list` / `cat-file`), each reaped via process-group teardown at the deadline. | | `GITLAWB_TIGRIS_BUCKET` | Optional S3/Tigris shared repo storage bucket. | | `GITLAWB_PINATA_JWT` | Optional Pinata/IPFS warm-storage pinning. | | `GITLAWB_IRYS_URL` | Optional Irys/Arweave permanent anchoring. | diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index 90a061e0..d0a4c319 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -243,24 +243,25 @@ pub struct Config { /// cap is a different axis (500 connections each fan out to git + /// pack-objects + threads). Size below the process budget with headroom. /// - /// This is the READ pool (`git_read_semaphore`): upload-pack and both info/refs - /// advertisements. The authenticated push POST draws from a separate write pool - /// (`max_concurrent_git_pushes`) that anonymous reads can never reach, and each - /// caller is additionally bounded by `max_concurrent_reads_per_caller`, so an - /// anonymous flood cannot shed the actual push nor monopolize reads (#174). (The - /// receive-pack advertisement itself shares the read pool; a shed advertisement - /// is a cheap retryable GET, and the write POST it precedes always has capacity.) + /// This is the READ pool (`git_read_semaphore`): upload-pack and the UPLOAD-PACK + /// `info/refs` advertisement only. The authenticated push POST draws from a + /// separate write pool (`max_concurrent_git_pushes`) that anonymous reads can + /// never reach, and each read caller is additionally bounded by + /// `max_concurrent_reads_per_caller`, so an anonymous flood cannot shed the actual + /// push nor monopolize reads (#174). The anon-reachable RECEIVE-PACK `info/refs` + /// advertisement draws from its OWN dedicated pool (sized like the write pool but + /// disjoint), so an advertisement flood can never occupy a permit the + /// authenticated push POST needs at admission (#174). /// /// A permit is held for the whole op. Every git subprocess that STREAMS is /// duration-bounded and reaps its process group on disconnect: upload-pack, /// receive-pack, and both info/refs advertisements run under /// `git_service_timeout_secs` with `process_group(0)` teardown, and the - /// withheld-blob (`upload_pack_excluding`) pack-objects stage runs on the async - /// side under the same teardown (#174 closed the two duration/cancellation gaps - /// this comment previously tracked). The one remaining blocking stage is the - /// `rev-list` object enumeration in the withheld-blob path — a bounded walk that - /// terminates, run off the async runtime; it is not process-group-reaped on - /// disconnect, so a stuck rev-list can still hold its slot until git exits. + /// withheld-blob (`upload_pack_excluding`) pack-objects stage plus the push-side + /// candidate-discovery children (`rev-list` / `cat-file`) now run under the same + /// bounded runner with process-group teardown, so a stuck git child no longer + /// holds its slot indefinitely (#174 closed the duration/cancellation gaps this + /// comment previously tracked). /// /// Default: 128. Must be between 1 and 1_048_576; the ceiling keeps the value /// well under tokio's `Semaphore` permit limit so an oversized value is a @@ -273,10 +274,14 @@ pub struct Config { )] pub max_concurrent_git_ops: usize, - /// Maximum number of concurrent `git-receive-pack` (push) operations. Pushes - /// draw from this dedicated pool, separate from `max_concurrent_git_ops` - /// (reads), so a flood of anonymous reads cannot shed an authenticated push at - /// admission (#174). Beyond this a push sheds a clean 503 + Retry-After. + /// Maximum number of concurrent `git-receive-pack` (push) operations. The + /// authenticated push POST draws from this dedicated pool, separate from + /// `max_concurrent_git_ops` (reads), so a flood of anonymous reads cannot shed an + /// authenticated push at admission (#174). The anon-reachable receive-pack + /// `info/refs` advertisement runs in a SEPARATE pool of the same size (derived + /// from this knob), disjoint from this one, so an advertisement flood cannot + /// occupy a POST's slot either (#174). Beyond this a push sheds a clean 503 + + /// Retry-After. /// /// Default: 32. Must be between 1 and 1_048_576 (the ceiling keeps the value /// under tokio's `Semaphore` permit limit so an oversized value is a clean CLI @@ -289,16 +294,17 @@ pub struct Config { )] pub max_concurrent_git_pushes: usize, - /// Maximum concurrent read operations (`upload-pack` and the `info/refs` - /// advertisements) a single caller may hold at once, so one caller cannot - /// monopolize the `max_concurrent_git_ops` read pool (#174). Callers are keyed - /// per-DID when signed, else per-source-IP. IMPORTANT: the per-source-IP key is - /// only as granular as `GITLAWB_TRUSTED_PROXY`. Left unset (the default), a node - /// behind an edge/NAT keys all anonymous callers on the edge IP, so this cap - /// collapses to a single global anonymous cap rather than per-client. Set - /// `GITLAWB_TRUSTED_PROXY` to key on the real client; a known high-fanout caller - /// (a CI fleet behind one NAT) should authenticate for a per-DID budget or the - /// operator raises this. Over-cap for a caller sheds a clean 503 + Retry-After. + /// Maximum concurrent read operations (`upload-pack` and the upload-pack + /// `info/refs` advertisement) a single caller may hold at once, so one caller + /// cannot monopolize the `max_concurrent_git_ops` read pool (#174). Callers are + /// keyed on the RESOLVED SOURCE IP, never the DID — a signature does not move a + /// caller off this cap, so an authenticated client cannot mint DIDs to escape it. + /// IMPORTANT: the source-IP key is only as granular as `GITLAWB_TRUSTED_PROXY`. + /// Left unset (the default), a node behind an edge/NAT keys all callers on the + /// edge IP, so this cap collapses to a single global cap rather than per-client. + /// Set `GITLAWB_TRUSTED_PROXY` to key on the real client; a high-fanout caller (a + /// CI fleet behind one NAT) then needs the operator to raise this. Over-cap for a + /// caller sheds a clean 503 + Retry-After. /// /// Default: 16. Must be between 1 and 1_048_576. #[arg( From cf97405bc188e67beaff37c6a708694dda04fe94 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 13 Jul 2026 10:55:57 -0500 Subject: [PATCH 25/58] test(node): execution-cover the delta-path push-scan timeout wiring (#174) The finding-2 wiring test covered only list_all_objects (the full-scan fallback). Add delta_path_exec_fns_time_out_on_a_hung_git covering the common push path: peeled_object_type (cat-file), rev_list_delta (rev-list), and list_all_objects_with_type (cat-file, which all_blob_oids delegates to). Each must return within the watchdog budget on a hung git; reverting any to a bare Command::output() blocks past the recv budget (RED-verified). --- crates/gitlawb-node/src/git/push_delta.rs | 56 +++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/crates/gitlawb-node/src/git/push_delta.rs b/crates/gitlawb-node/src/git/push_delta.rs index 51b6f836..962c9c65 100644 --- a/crates/gitlawb-node/src/git/push_delta.rs +++ b/crates/gitlawb-node/src/git/push_delta.rs @@ -746,4 +746,60 @@ mod tests { "a hung git must make list_all_objects error out, not hang" ); } + + /// #174 finding 2: the DELTA-path children — `peeled_object_type` (cat-file), + /// `rev_list_delta` (rev-list) — and `list_all_objects_with_type` (cat-file) are + /// the remaining candidate-discovery execs (the common push path). Each must + /// return within the watchdog budget on a hung git rather than block and pin the + /// write permit. Revert any one to a bare `Command::output()` and its arm blocks + /// past the recv budget (RED). + #[cfg(unix)] + #[test] + fn delta_path_exec_fns_time_out_on_a_hung_git() { + use std::os::unix::fs::PermissionsExt; + use std::time::{Duration, Instant}; + let dir = tempfile::TempDir::new().unwrap(); + // Fake git hangs on BOTH cat-file and rev-list (bounded 30s so a broken + // test can't leak a permanent orphan or wedge the suite). + let fake = dir.path().join("fakegit"); + std::fs::write( + &fake, + "#!/bin/sh\ncase \"$1\" in\n cat-file|rev-list) i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done ;;\n *) : ;;\nesac\n", + ) + .unwrap(); + let mut perm = std::fs::metadata(&fake).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&fake, perm).unwrap(); + let git_bin = fake.to_str().unwrap().to_string(); + let path = dir.path().to_path_buf(); + + // Run `f` on a thread and require it to RETURN (not block) within 10s. + fn returns_within(f: impl FnOnce() -> T + Send + 'static) -> T { + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let _ = tx.send(f()); + }); + rx.recv_timeout(Duration::from_secs(10)).expect( + "a delta-path exec fn must return within the watchdog budget, not block on a hung git", + ) + } + let dl = || Instant::now() + Duration::from_millis(150); + let sha = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef"; + + let (p, g, d) = (path.clone(), git_bin.clone(), dl()); + assert!( + returns_within(move || peeled_object_type(&p, sha, &g, d)).is_none(), + "peeled_object_type must time out to None on a hung cat-file" + ); + let (p, g, d) = (path.clone(), git_bin.clone(), dl()); + assert!( + returns_within(move || rev_list_delta(&p, &[sha], &[], &g, d)).is_err(), + "rev_list_delta must error out on a hung rev-list" + ); + let (p, g, d) = (path.clone(), git_bin.clone(), dl()); + assert!( + returns_within(move || list_all_objects_with_type(&p, &g, d)).is_err(), + "list_all_objects_with_type must error out on a hung cat-file" + ); + } } From a030eefb704d68a008e760a01077abc823cbe258 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 13 Jul 2026 16:01:49 -0500 Subject: [PATCH 26/58] test(node): retry the ETXTBSY exec race in build_filtered_pack timeout tests (#174) build_filtered_pack_times_out_a_hung_rev_list and _a_hung_pack_objects spawn a freshly-written fake git directly, bypassing the fake_git_run_with_pids retry the rest of the module uses. Under cargo test fork-storm load a concurrent worker forks while the fake's write fd is still open, so execve returns ETXTBSY and the raw io::Error surfaces where the test asserts GitServiceTimeout, failing test (stable) intermittently. Route both tests through build_filtered_pack_or_exec_race_retry, which retries ONLY errno 26 (ETXTBSY) and keeps the outer 10s watchdog per attempt, so a genuinely-missing async timeout bound still trips the watchdog and a wrong error type still fails at the assertion. Production paths unchanged. --- crates/gitlawb-node/src/git/smart_http.rs | 99 ++++++++++++++++------- 1 file changed, 71 insertions(+), 28 deletions(-) diff --git a/crates/gitlawb-node/src/git/smart_http.rs b/crates/gitlawb-node/src/git/smart_http.rs index 206e7b27..f885aef3 100644 --- a/crates/gitlawb-node/src/git/smart_http.rs +++ b/crates/gitlawb-node/src/git/smart_http.rs @@ -1160,6 +1160,61 @@ mod tests { ); } + /// True when `err` is the transient ETXTBSY exec race a freshly-written fake + /// `git` hits under fork-storm load — a concurrent worker forked while this + /// file's write fd was still open, so `execve` sees it as busy (the same race + /// `fake_git_run_with_pids` retries). Deliberately narrow: only this raw-OS + /// error is retried, so a wrong error *type* (e.g. a missing async bound + /// surfacing as anything other than `GitServiceTimeout`) still fails loudly at + /// the caller's assertion instead of being swallowed. `spawn()?` propagates the + /// `io::Error` with no context, so it survives as the anyhow root. + #[cfg(unix)] + fn is_transient_exec_race(err: &anyhow::Error) -> bool { + // ETXTBSY == 26 on Linux and the BSDs/macOS; std has no stable ErrorKind. + err.downcast_ref::() + .and_then(std::io::Error::raw_os_error) + == Some(26) + } + + /// Drive `build_filtered_pack` under a per-attempt watchdog, retrying ONLY the + /// transient ETXTBSY exec race and returning the terminal error for the caller + /// to classify. Every attempt keeps its own outer `tokio::time::timeout`, so a + /// MISSING async bound — the regression the `build_filtered_pack_times_out_*` + /// tests guard — still trips the watchdog loudly on every attempt: the retry + /// can only absorb a fast exec failure, never a hang (a hang never returns, so + /// it can never reach the retry decision). + #[cfg(unix)] + async fn build_filtered_pack_or_exec_race_retry( + git_bin: &str, + repo_path: &std::path::Path, + withheld: &HashSet, + stage_timeout: Duration, + ) -> anyhow::Error { + for i in 0..FAKE_GIT_RETRY_ATTEMPTS { + let result = tokio::time::timeout( + Duration::from_secs(10), + build_filtered_pack(git_bin, repo_path, withheld, stage_timeout), + ) + .await + .expect( + "build_filtered_pack must return within the watchdog — the git stage \ + must be timeout-bounded, not an uncancellable spawn_blocking", + ); + let err = result.expect_err("a hung git stage must return an error, not hang"); + if is_transient_exec_race(&err) { + // Fresh-fake-git ETXTBSY: back off (growing) so a bursty fork-pressure + // spike subsides before retrying, per fake_git_run_with_pids. + tokio::time::sleep(Duration::from_millis(FAKE_GIT_BACKOFF_STEP_MS * (i + 1))).await; + continue; + } + return err; + } + panic!( + "fake git kept hitting ETXTBSY after {FAKE_GIT_RETRY_ATTEMPTS} attempts \ + (persistent exec failure, not a transient parallel-runner miss)" + ); + } + // Dropping the request future mid-flight (client disconnect) must SIGTERM the // whole group so git AND its pack-objects grandchild die together. Goes RED // if `process_group(0)` or the guard-arming is removed: without its own @@ -1314,21 +1369,15 @@ mod tests { let git_bin = write_fake_git(tmp.path(), body); let withheld = HashSet::new(); - let result = tokio::time::timeout( - Duration::from_secs(10), - build_filtered_pack( - git_bin.to_str().unwrap(), - tmp.path(), - &withheld, - Duration::from_millis(200), - ), + // Retry only the transient ETXTBSY exec race (see the helper); the + // per-attempt watchdog keeps the missing-bound regression loud. + let err = build_filtered_pack_or_exec_race_retry( + git_bin.to_str().unwrap(), + tmp.path(), + &withheld, + Duration::from_millis(200), ) - .await - .expect( - "build_filtered_pack must return within the watchdog — the pack-objects \ - stage must be timeout-bounded, not an uncancellable spawn_blocking", - ); - let err = result.expect_err("a hung pack-objects must return an error, not hang"); + .await; assert!( err.downcast_ref::().is_some(), "a hung pack-objects must abort with GitServiceTimeout, got: {err}" @@ -1352,21 +1401,15 @@ mod tests { let git_bin = write_fake_git(tmp.path(), body); let withheld = HashSet::new(); - let result = tokio::time::timeout( - Duration::from_secs(10), - build_filtered_pack( - git_bin.to_str().unwrap(), - tmp.path(), - &withheld, - Duration::from_millis(200), - ), + // Retry only the transient ETXTBSY exec race (see the helper); the + // per-attempt watchdog keeps the missing-bound regression loud. + let err = build_filtered_pack_or_exec_race_retry( + git_bin.to_str().unwrap(), + tmp.path(), + &withheld, + Duration::from_millis(200), ) - .await - .expect( - "build_filtered_pack must return within the watchdog — the rev-list \ - stage must be timeout-bounded, not an uncancellable spawn_blocking", - ); - let err = result.expect_err("a hung rev-list must return an error, not hang"); + .await; assert!( err.downcast_ref::().is_some(), "a hung rev-list must abort with GitServiceTimeout, got: {err}" From 3cc01890fbb3b395a336df951cff0e5b89393530 Mon Sep 17 00:00:00 2001 From: t Date: Tue, 14 Jul 2026 19:27:12 -0500 Subject: [PATCH 27/58] fix(node): keep the visibility-walk watchdog armed until the child is reaped (#174) A group member (or the leader itself) that closes stdout before the deadline made the stdout drain return EOF early, which stood the watchdog down via done_tx before it could ever fire; child.wait() then blocked on the still-live child, pinning the walk thread and its read/write permit past GITLAWB_GIT_SERVICE_TIMEOUT_SECS. This is distinct from the descendant case already covered: there the leader sleeps until the deadline so the watchdog does time out; here the drain-EOF races ahead of it. Stand the watchdog down only once the child has actually terminated, detected without reaping (waitid + WNOWAIT) so the leader's pid stays unreaped and its pgid un-recycled until the watchdog finishes its teardown and is joined. Past the deadline the watchdog still owns the full SIGTERM -> grace -> SIGKILL sequence. Regression: run_bounded_git_reaps_a_leader_that_closes_stdout_then_hangs, RED before (blocks on child.wait past the recv budget), GREEN after; the three existing teardown tests still pass (no recycle-hazard regression). --- .../gitlawb-node/src/git/visibility_pack.rs | 116 ++++++++++++++++-- 1 file changed, 108 insertions(+), 8 deletions(-) diff --git a/crates/gitlawb-node/src/git/visibility_pack.rs b/crates/gitlawb-node/src/git/visibility_pack.rs index 9f40cd4b..9f07020d 100644 --- a/crates/gitlawb-node/src/git/visibility_pack.rs +++ b/crates/gitlawb-node/src/git/visibility_pack.rs @@ -41,6 +41,31 @@ const WATCHDOG_TERM_GRACE: Duration = Duration::from_secs(1); /// the serve handler maps it to 504. `git_bin` is injectable so a fake `git` can /// drive the teardown in tests without mutating the process-global PATH; /// `stdin_bytes` feeds children that read stdin (empty for the arg-only children). +/// Returns true if `pid` (a process-group leader we spawned) has terminated, WITHOUT +/// reaping it. `waitid(..., WNOWAIT)` reports the exit state but leaves the child +/// waitable, so the caller's later `child.wait()` still collects the status and the +/// pid/pgid stays live until then — which is what keeps the watchdog's `kill(-pgid)` +/// teardown from ever racing a recycled pgid. Used to distinguish "the child actually +/// exited" from "the child merely closed stdout" after the drain returns (#174 P1-a). +#[cfg(unix)] +fn child_terminated_without_reaping(pid: i32) -> bool { + // SAFETY: waitid writes only into the zeroed siginfo and borrows no Rust memory; + // WNOWAIT leaves the child unreaped, WNOHANG makes the probe non-blocking. + let mut info: libc::siginfo_t = unsafe { std::mem::zeroed() }; + let rc = unsafe { + libc::waitid( + libc::P_PID, + pid as libc::id_t, + &mut info, + libc::WEXITED | libc::WNOHANG | libc::WNOWAIT, + ) + }; + // rc == 0 with si_pid == 0 means "no state change yet" (still running); a non-zero + // si_pid means the child has entered a waitable, exited state. EINTR/other errors + // (rc != 0) are treated as "not yet terminated" and the caller re-polls. + rc == 0 && unsafe { info.si_pid() } != 0 +} + #[cfg(unix)] pub(crate) fn run_bounded_git( git_bin: &str, @@ -132,14 +157,28 @@ pub(crate) fn run_bounded_git( // is unreachable in userspace (no signal reaps a D-state process) and matches the // async `reap_group_on_timeout`, which likewise only warns and gives up there. let read_result = stdout.read_to_end(&mut out); - // The drain has returned: either the child exited on its own, or the watchdog's - // teardown closed its pipes. Tell the watchdog the drain is done, then WAIT for it - // to return BEFORE reaping the leader. On the timeout path the watchdog runs the - // full SIGTERM -> grace -> SIGKILL teardown; joining it before `child.wait()` keeps - // the leader's pid unreaped while every `kill(-pgid)` fires (so the pgid can't be - // recycled), and guarantees a SIGTERM-ignoring group member has already been - // SIGKILLed rather than left running past the deadline (finding 3, #174). - let _ = done_tx.send(()); + // The drain has returned, but that only means all stdout write ends are closed — + // NOT that the child has exited. A group member, or the leader itself, can close + // stdout and keep running; standing the watchdog down on the drain alone (as the + // old code did) would then let `child.wait()` block forever on that live child, + // past the deadline, pinning the walk thread and its permit (finding P1-a, #174). + // So stand the watchdog down only once the child has ACTUALLY terminated, detected + // WITHOUT reaping (waitid + WNOWAIT) so the leader's pid stays unreaped and its + // pgid un-recycled until the watchdog finishes and we join it below. Past the + // deadline the watchdog owns the teardown, so we stop polling and let it run the + // full SIGTERM -> grace -> SIGKILL; joining it before `child.wait()` keeps every + // `kill(-pgid)` firing while the pid is still unreaped and guarantees a + // stdout-closing-then-hanging member has been SIGKILLed rather than left running. + loop { + if child_terminated_without_reaping(pgid) { + let _ = done_tx.send(()); + break; + } + if Instant::now() >= deadline { + break; + } + std::thread::sleep(Duration::from_millis(2)); + } let killed = watchdog.join().unwrap_or(false); let status = child.wait().context("git wait failed")?; let err = err_reader.join().unwrap_or_default(); @@ -902,6 +941,67 @@ esac\n"; "a SIGTERM-ignoring descendant (pid {desc}) must be SIGKILLed even after the leader exits cleanly, not orphaned" ); } + + /// #174 U1 (P1-a, RED-before/GREEN-after): the group LEADER closes its own + /// stdout/stderr BEFORE the deadline and then keeps running. On the pre-fix code + /// the stdout drain returns EOF early, `done_tx.send` stands the watchdog down + /// before it ever fires (`recv` gets `Ok` -> `false`, no kill), and `child.wait()` + /// then blocks on the still-alive leader — pinning the walk thread and its read/ + /// write permit past the deadline, bypassing GITLAWB_GIT_SERVICE_TIMEOUT_SECS. + /// This is distinct from the descendant case above: there the leader sleeps until + /// the deadline so the watchdog DOES time out; here the drain-EOF races ahead of + /// the deadline. The fix keeps the watchdog armed until the child is actually + /// reaped, so the deadline SIGTERM still fires and the call returns within budget. + /// A pre-fix build blocks on `child.wait()` past the recv budget (RED). + #[cfg(unix)] + #[test] + fn run_bounded_git_reaps_a_leader_that_closes_stdout_then_hangs() { + use std::time::{Duration, Instant}; + let tmp = TempDir::new().unwrap(); + // rev-list records its (leader) pid, closes stdout+stderr so the drain EOFs + // immediately, then sleeps without trapping TERM. `sleep 30` bounds the worst + // case so a RED run cannot wedge the suite; the recv budget fires first. + let body = "#!/bin/sh\ncase \"$1\" in\n rev-list) echo $$ > leader.pid; exec 1>&- 2>&-; sleep 30 ;;\n *) : ;;\nesac\nexit 0\n"; + let git_bin = write_fake_git(tmp.path(), body); + let (tx, rx) = std::sync::mpsc::channel(); + let path = tmp.path().to_path_buf(); + std::thread::spawn(move || { + let _ = tx.send(run_bounded_git( + &git_bin, + &["rev-list"], + &path, + b"", + Instant::now() + Duration::from_millis(100), + )); + }); + let out = rx.recv_timeout(Duration::from_secs(10)).expect( + "run_bounded_git must return within the watchdog budget when the leader closes stdout then hangs, not block on child.wait()", + ); + assert!( + out.is_err(), + "a leader killed at the deadline (no TERM trap) is a timeout, not a success: {out:?}" + ); + let pid: i32 = std::fs::read_to_string(tmp.path().join("leader.pid")) + .unwrap() + .trim() + .parse() + .unwrap(); + let mut gone = false; + for _ in 0..300 { + if unsafe { libc::kill(pid, 0) } != 0 { + gone = true; + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + // Kill it regardless so a RED run leaks no orphan. + unsafe { libc::kill(pid, libc::SIGKILL) }; + assert!( + gone, + "the hung leader (pid {pid}) must be killed and reaped at the deadline, not left running" + ); + } + use crate::db::VisibilityMode; use chrono::Utc; use std::process::Command; From ac59bd7e01bf42b7d42b7adb57ccb927c0309993 Mon Sep 17 00:00:00 2001 From: t Date: Tue, 14 Jul 2026 19:50:41 -0500 Subject: [PATCH 28/58] fix(node): reap the served-git process group on disconnect, not just SIGTERM (#174) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On a client disconnect KillGroupOnDrop sent a lone SIGTERM with no escalation or reap, so a group member that ignores SIGTERM (a wedged pack-objects, a malicious helper) survived while the handler released its concurrency permit — letting disconnect-spam accumulate live git processes and defeat the load shed (P1-c). The timeout path already ran the full teardown. Make the disconnect teardown as strong as the timeout path: the guard now owns the child and, on drop, launches a detached reaper that runs the same TERM -> grace -> SIGKILL -> reap-the-group sequence (reap_group_on_timeout). Owning the tokio Child in the reaper keeps it the sole reaper of the leader, so it never races tokio's own orphan reaper (a raw waitpid(-pgid) would). Regression: run_git_service_sigkills_a_sigterm_ignoring_child_on_disconnect, RED before (the descendant survives the lone SIGTERM), GREEN after; all existing teardown/timeout/disarm tests still pass (the three hand-built-guard tests were updated to the owned-child shape). Residual: capacity (the handler's semaphore permit) still releases at disconnect rather than strictly after the reaper confirms the group dead, so accumulation is now bounded to ~one teardown window (<=4s) rather than eliminated; gating the permit on the reap needs threading it through the serve handlers and is left for a follow-up (it also touches the receive-pack permit lifetime U4 changes). --- crates/gitlawb-node/src/git/smart_http.rs | 235 +++++++++++++++++----- 1 file changed, 188 insertions(+), 47 deletions(-) diff --git a/crates/gitlawb-node/src/git/smart_http.rs b/crates/gitlawb-node/src/git/smart_http.rs index f885aef3..a5649a42 100644 --- a/crates/gitlawb-node/src/git/smart_http.rs +++ b/crates/gitlawb-node/src/git/smart_http.rs @@ -108,24 +108,60 @@ pub async fn receive_pack( /// `wait_with_output()` returns, so a request that completed cleanly never signals. #[cfg(unix)] struct KillGroupOnDrop { + // Holds the child + its pgid while armed. The interaction drives the child through + // `child_mut()`; the success/timeout paths call `disarm` once they have reaped it. + // On drop — a client disconnect that drops the whole request future — the guard + // launches a detached reaper that OWNS the child and runs the full + // SIGTERM -> grace -> SIGKILL -> reap sequence (`reap_group_on_timeout`), as strong + // as the timeout path, so a SIGTERM-ignoring group member is SIGKILLed and reaped + // rather than left running until EPIPE to accumulate past the concurrency cap + // (#174 P1-c). Owning the tokio `Child` in the reaper (rather than a raw + // `waitpid(-pgid)`) keeps a single reaper of the leader, so it never races tokio's + // own SIGCHLD-driven orphan reaper. + child: Option, pgid: Option, } #[cfg(unix)] impl KillGroupOnDrop { + /// The child, while armed. The interaction drives stdin/stdout/`wait` through this. + fn child_mut(&mut self) -> &mut tokio::process::Child { + self.child.as_mut().expect("child present while armed") + } + + /// Disarm on the success/timeout path: the body has already reaped the child (its + /// `wait()` returned, or `reap_group_on_timeout` ran), so drop the handle and clear + /// the pgid, leaving the guard's Drop a no-op. fn disarm(&mut self) { self.pgid = None; + self.child = None; } } #[cfg(unix)] impl Drop for KillGroupOnDrop { fn drop(&mut self) { - if let Some(pgid) = self.pgid { - // SAFETY: kill(2) takes only integer arguments and borrows no Rust - // memory. Signalling a stale group just returns ESRCH, which we ignore. - unsafe { - libc::kill(-pgid, libc::SIGTERM); + let (Some(mut child), Some(pgid)) = (self.child.take(), self.pgid) else { + return; + }; + // A sync Drop cannot await, so launch a detached reaper that owns the child and + // runs the same teardown as the timeout path (TERM -> grace -> SIGKILL -> reap + // the whole group). Owning the tokio `Child` means this task is the sole reaper + // of the leader, so it cannot race tokio's orphan reaper. Prefer the current + // runtime handle; if dropped outside a runtime, fall back to a best-effort + // synchronous SIGTERM so the group is at least signalled. + match tokio::runtime::Handle::try_current() { + Ok(handle) => { + handle.spawn(async move { + reap_group_on_timeout(&mut child).await; + }); + } + Err(_) => { + // SAFETY: kill(2) takes only integers and borrows no Rust memory; + // ESRCH on an already-gone group is ignored. + unsafe { + libc::kill(-pgid, libc::SIGTERM); + } } } } @@ -244,20 +280,28 @@ async fn drive_git_child( let mut child = command.spawn()?; - // Arm the group-kill guard for the lifetime of the request. With - // process_group(0) the child is its own group leader, so pgid == its pid. - // This fires on a client disconnect (the whole future is dropped mid-request). + // Own the pipes so the child stays reap-able: wait_with_output would consume it, + // but on a timeout we must actively reap the group first (see + // reap_group_on_timeout), and on a disconnect the guard's detached reaper does. + // Take the pipes before the child moves into the guard below. + let mut stdin = child.stdin.take(); + let mut stdout = child.stdout.take().context("git stdout was not piped")?; + let mut stderr = child.stderr.take().context("git stderr was not piped")?; + + // Arm the group-kill guard for the lifetime of the request. With process_group(0) + // the child is its own group leader, so pgid == its pid. On a client disconnect + // (the whole future is dropped mid-request) the guard's Drop launches a detached + // reaper that OWNS the child and runs the full TERM/grace/SIGKILL/reap — as strong + // as the timeout path below (#174 P1-c). The guard owns the child; the interaction + // drives it through `child_mut()`. + #[cfg(unix)] + let pgid = child.id().map(|id| id as i32); #[cfg(unix)] let mut group_guard = KillGroupOnDrop { - pgid: child.id().map(|id| id as i32), + child: Some(child), + pgid, }; - // Own the pipes so `child` stays reap-able after a timeout: wait_with_output - // would consume it, but on timeout we must actively reap the group before - // returning (see reap_group_on_timeout). - let mut stdin = child.stdin.take(); - let mut stdout = child.stdout.take().context("git stdout was not piped")?; - let mut stderr = child.stderr.take().context("git stderr was not piped")?; let mut out = Vec::new(); let mut err = Vec::new(); @@ -274,11 +318,15 @@ async fn drive_git_child( None => Ok(()), } }; + #[cfg(unix)] + let child_ref = group_guard.child_mut(); + #[cfg(not(unix))] + let child_ref = &mut child; let (write_result, r_out, r_err, status) = tokio::join!( write, stdout.read_to_end(&mut out), stderr.read_to_end(&mut err), - child.wait(), + child_ref.wait(), ); r_out?; r_err?; @@ -290,7 +338,7 @@ async fn drive_git_child( Ok(result) => { // The join runs all arms to completion, so the child is reaped: disarm // before surfacing any interaction error (a read/wait error), else the - // guard's drop would fire SIGTERM on the reaped, possibly-reused pgid. + // guard's drop would reap an already-reaped child / signal a reused pgid. #[cfg(unix)] group_guard.disarm(); result? @@ -301,7 +349,7 @@ async fn drive_git_child( // the (now redundant) guard so its drop can't hit a reused pgid. #[cfg(unix)] { - reap_group_on_timeout(&mut child).await; + reap_group_on_timeout(group_guard.child_mut()).await; group_guard.disarm(); } #[cfg(not(unix))] @@ -960,23 +1008,36 @@ mod tests { #[cfg(unix)] #[tokio::test] async fn kill_group_guard_terminates_child_on_drop() { - let mut child = tokio::process::Command::new("sleep") + let child = tokio::process::Command::new("sleep") .arg("300") .process_group(0) .spawn() .unwrap(); - let pgid = child.id().map(|id| id as i32); + let pid = child.id().unwrap() as i32; { - let _guard = KillGroupOnDrop { pgid }; - } // guard drops here -> SIGTERM to the group - - use std::os::unix::process::ExitStatusExt; - let status = child.wait().await.unwrap(); - assert_eq!( - status.signal(), - Some(libc::SIGTERM), - "child must be terminated by SIGTERM via its process group" + // The guard owns the child; on drop its detached reaper TERM/grace/KILL/ + // reaps the group (a plain sleep dies on the first SIGTERM). + let _guard = KillGroupOnDrop { + child: Some(child), + pgid: Some(pid), + }; + } + + let mut gone = false; + for _ in 0..300 { + if !alive(pid) { + gone = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + unsafe { + libc::kill(pid, libc::SIGKILL); + } + assert!( + gone, + "child must be terminated and reaped via its process group on guard drop" ); } @@ -994,9 +1055,10 @@ mod tests { .process_group(0) .spawn() .unwrap(); - let pgid = child.id().map(|id| id as i32); + let pid = child.id().unwrap() as i32; - // Read the backgrounded grandchild's pid from the first stdout line. + // Read the backgrounded grandchild's pid from the first stdout line, before + // the child moves into the guard. let mut stdout = child.stdout.take().unwrap(); let mut buf = Vec::new(); loop { @@ -1011,20 +1073,25 @@ mod tests { assert!(alive(grandchild), "grandchild should be running"); { - let _guard = KillGroupOnDrop { pgid }; - } // group SIGTERM reaches sh AND the sleep grandchild - - let _ = child.wait().await; // reap sh + // The guard owns the child; on drop the detached reaper group-kills sh AND + // the sleep grandchild. + let _guard = KillGroupOnDrop { + child: Some(child), + pgid: Some(pid), + }; + } - // The grandchild reparents to init and is reaped; poll until it's gone. let mut gone = false; - for _ in 0..200 { + for _ in 0..300 { if !alive(grandchild) { gone = true; break; } tokio::time::sleep(std::time::Duration::from_millis(10)).await; } + unsafe { + libc::kill(grandchild, libc::SIGKILL); + } assert!( gone, "grandchild must be terminated by the group signal (#53)" @@ -1036,27 +1103,31 @@ mod tests { async fn kill_group_guard_disarmed_does_not_kill() { // A request that completed cleanly disarms the guard; dropping it must not // signal anything. - let mut child = tokio::process::Command::new("sleep") + let child = tokio::process::Command::new("sleep") .arg("300") .process_group(0) .spawn() .unwrap(); + let pid = child.id().unwrap() as i32; { let mut guard = KillGroupOnDrop { - pgid: child.id().map(|id| id as i32), + child: Some(child), + pgid: Some(pid), }; guard.disarm(); - } // disarmed -> no kill + } // disarmed -> no reaper, no kill - assert!( - child.try_wait().unwrap().is_none(), - "disarmed guard must not kill the child" - ); + // Give any erroneously-spawned reaper a chance to run, then assert alive. + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + assert!(alive(pid), "disarmed guard must not kill the child"); - // Clean up the still-running child. - let _ = child.kill().await; - let _ = child.wait().await; + // Clean up the still-running child (disarm dropped the handle, so reap by pid). + unsafe { + libc::kill(pid, libc::SIGKILL); + let mut status = 0; + libc::waitpid(pid, &mut status, 0); + } } // ── #62 PR1: end-to-end teardown wiring through run_git_service ───────── @@ -1320,6 +1391,76 @@ mod tests { ); } + /// #174 U2 (P1-c, RED-before/GREEN-after): on a client disconnect the teardown must + /// be as strong as the timeout path — a group member that IGNORES SIGTERM is still + /// SIGKILLed and reaped, not left running to accumulate past the concurrency cap. + /// The old `KillGroupOnDrop::drop` sent a lone SIGTERM with no escalation or reap, + /// so a SIGTERM-ignoring descendant survived the disconnect (RED). The fix launches + /// a detached reaper owning the child that runs the full TERM/grace/SIGKILL/reap. + /// This is distinct from the well-behaved-grandchild test above, whose `sleep` dies + /// on the first SIGTERM. + #[cfg(unix)] + #[tokio::test] + async fn run_git_service_sigkills_a_sigterm_ignoring_child_on_disconnect() { + let tmp = tempfile::TempDir::new().unwrap(); + let descfile = tmp.path().join("desc.pid"); + // Leader (dies on the group SIGTERM) spawns a descendant that traps SIGTERM, + // records its own pid, and loops ~30s (bounded so a RED run leaks no permanent + // orphan; the assertion fires well before then). + let body = format!( + "#!/bin/sh\n\ + sh -c 'trap \"\" TERM; echo $$ > \"{}\"; i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done' &\n\ + wait\n", + descfile.display() + ); + let git_bin = write_fake_git(tmp.path(), &body); + + let mut fut = Box::pin(run_git_service( + git_bin.to_str().unwrap(), + "git-upload-pack", + tmp.path(), + Bytes::new(), + Duration::from_secs(60), + )); + // Drive the future a slice at a time until the descendant records its pid. + let mut desc: Option = None; + for _ in 0..500 { + let _ = tokio::time::timeout(Duration::from_millis(10), &mut fut).await; + if let Some(p) = std::fs::read_to_string(&descfile) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + desc = Some(p); + break; + } + } + let desc = desc.expect("the fake leader must have spawned and recorded a descendant"); + let _cleanup = ReapOnPanic(vec![desc]); + assert!(alive(desc), "descendant should be running before the drop"); + + // Client disconnect: drop the request future. The guard's detached reaper must + // escalate SIGTERM -> SIGKILL and reap the SIGTERM-ignoring descendant. + drop(fut); + + let mut gone = false; + for _ in 0..500 { + if !alive(desc) { + gone = true; + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + // Kill regardless so a RED run leaks no orphan. + unsafe { + libc::kill(desc, libc::SIGKILL); + } + assert!( + gone, + "a SIGTERM-ignoring descendant (pid {desc}) must be SIGKILLed and reaped on \ + disconnect, not left running (old KillGroupOnDrop sent SIGTERM only)" + ); + } + // #174 (SC3): the info/refs advertisement is now duration-bounded — previously // it ran a bare `Command::output()` with no deadline, so a hung git pinned its // concurrency slot forever. A hung fake git must abort with GitServiceTimeout From be0cdd6a9eae1b39618f16fdf4e28eae6e83ca51 Mon Sep 17 00:00:00 2001 From: t Date: Tue, 14 Jul 2026 20:09:40 -0500 Subject: [PATCH 29/58] fix(node): hold upload-pack admission through the walk on disconnect (#174) On the path-scoped upload-pack path the per-source and global read permits were handler locals, so a client disconnect dropped the handler future and released both permits while the awaited spawn_blocking withheld-blob walk kept running (spawn_blocking can't be cancelled). A permitless caller could disconnect-spam to exceed the read cap while real git work continued (P1-b). Move both permits into the blocking task and return them out: on success they flow back so the serve phase keeps them; on a dropped future the returned tuple (with the permits) is discarded only when the blocking task completes, so admission tracks the walk's real duration. Regression: upload_pack_permit_held_through_walk_after_disconnect drives the handler to mid-walk (fake git hangs on rev-list), disconnects, and asserts the global read slot stays held (available_permits == 0). RED before (the slot frees to 1 the instant the future drops), GREEN after. The test roots the repo store at its own TempDir so it is isolated per run. --- crates/gitlawb-node/src/api/repos.rs | 166 +++++++++++++++++++++++++-- 1 file changed, 158 insertions(+), 8 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index ea6f126e..8612709c 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -819,9 +819,16 @@ pub async fn git_upload_pack( let resp = if !visibility_pack::has_path_scoped_rule(&rules) { smart_http::upload_pack(&disk_path, body, git_timeout).await } else { - // withheld_blob_oids walks every ref with blocking `git ls-tree`; keep - // that off the async worker thread. - let withheld = { + // withheld_blob_oids walks every ref with blocking `git ls-tree`; keep that + // off the async worker thread. Move BOTH admission permits INTO the blocking + // task so they are held for the walk's real duration: spawn_blocking cannot be + // cancelled, so on a client disconnect the handler future drops but the walk + // keeps running — and now so do its permits, released only when the walk + // finishes rather than the instant the future drops (#174 P1-b). On success the + // task hands the permits back so the serve phase below keeps them; on a + // dropped future the returned tuple (with the permits) is discarded only when + // the blocking task completes, so admission tracks the real git work. + let (withheld, _permit, _caller_permit) = { let path = disk_path.clone(); let rules = rules.clone(); let owner_did = record.owner_did.clone(); @@ -829,7 +836,7 @@ pub async fn git_upload_pack( let is_public = record.is_public; let git_bin = state.git_bin.clone(); tokio::task::spawn_blocking(move || { - visibility_pack::withheld_blob_oids_bounded( + let withheld = visibility_pack::withheld_blob_oids_bounded( &path, &git_bin, git_timeout, @@ -837,14 +844,15 @@ pub async fn git_upload_pack( is_public, &owner_did, caller_owned.as_deref(), - ) + ); + (withheld, _permit, _caller_permit) }) .await .map_err(|e| AppError::Git(e.to_string()))? - // A walk that hit its deadline carries GitServiceTimeout; map it to 504 - // like the smart_http paths, not a generic 500 (#174 U3). - .map_err(|e| git_service_app_error(&e))? }; + // A walk that hit its deadline carries GitServiceTimeout; map it to 504 like + // the smart_http paths, not a generic 500 (#174 U3). + let withheld = withheld.map_err(|e| git_service_app_error(&e))?; if withheld.is_empty() { smart_http::upload_pack(&disk_path, body, git_timeout).await @@ -3456,6 +3464,148 @@ mod tests { ); } + /// #174 U3 (P1-b, RED-before/GREEN-after): a client disconnect during the + /// path-scoped withheld-blob walk must NOT release the read admission while the + /// uncancellable `spawn_blocking` walk is still running. The handler takes the + /// global read permit, enters the walk (a fake git hangs on rev-list), then the + /// request future is dropped mid-walk. With both permits moved into the blocking + /// task the global slot stays occupied until the walk finishes; on the pre-fix code + /// the handler-local permits drop on future-drop and the slot frees instantly (RED), + /// letting disconnect-spam exceed the cap while real git work keeps running. + #[sqlx::test] + async fn upload_pack_permit_held_through_walk_after_disconnect(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request}; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + let tmp = tempfile::TempDir::new().unwrap(); + let revlist_pid = tmp.path().join("revlist.pid"); + // Fake git: resolve refs fast, hang on rev-list (recording its pid first). The + // ~6s sleep bounds the walk so a broken fix cannot wedge the suite. + let body = format!( + "#!/bin/sh\ncase \"$1\" in\n rev-list) echo $$ > \"{}\" ; sleep 6 ;;\n rev-parse) echo deadbeef ;;\n *) : ;;\nesac\nexit 0\n", + revlist_pid.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + + let mut state = crate::test_support::test_state(pool.clone()).await; + // Root the repo store at this test's TempDir so the bare repo is isolated per + // run (the default for_testing store uses a fixed /tmp path that would collide + // across runs). + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.git_read_semaphore = Arc::new(Semaphore::new(1)); + state.git_bin = git_path.to_str().unwrap().to_string(); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let owner = "z6up3rd"; + let name = "up3"; + state + .db + .upsert_mirror_repo(owner, name, "/unused", None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + // Real bare repo at the path acquire() computes, so the handler reaches the walk. + state + .repo_store + .init(&rec.owner_did, &rec.name) + .await + .unwrap(); + // A path-scoped rule so has_path_scoped_rule() is true (the walk path) without + // denying the "/" gate for the public repo. + state + .db + .set_visibility_rule( + &rec.id, + "src/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkU3ReaderAAAAAAAAAAAAAAAAAAAAAAAAAAAA".to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + + let sem = state.git_read_semaphore.clone(); + assert_eq!( + sem.available_permits(), + 1, + "one read slot before the request" + ); + + let router = crate::server::build_router(state); + let peer: SocketAddr = "203.0.113.77:5000".parse().unwrap(); + let mut req = Request::builder() + .method(Method::POST) + .uri(format!("/{owner}/{name}/git-upload-pack")) + .body(Body::from(&b"0000"[..])) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + + let mut fut = Box::pin(router.oneshot(req)); + // Drive until the walk's rev-list starts (its pidfile appears) — i.e. the + // request is inside the spawn_blocking walk, holding the global read permit. + let mut in_walk = false; + for _ in 0..500 { + let _ = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + if revlist_pid.exists() { + in_walk = true; + break; + } + } + assert!( + in_walk, + "the walk's rev-list must start (request reached the spawn_blocking walk)" + ); + assert_eq!( + sem.available_permits(), + 0, + "the read slot is held while the walk runs" + ); + + // Client disconnect: drop the request future mid-walk. + drop(fut); + + // Load-bearing: the slot must STAY held while the uncancellable walk runs. On + // the pre-fix code the handler-local permits drop here and the slot frees at + // once (RED). + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + assert_eq!( + sem.available_permits(), + 0, + "on disconnect the read admission must be held until the spawn_blocking walk \ + finishes, not released the instant the future drops (P1-b)" + ); + + // Cleanup: let the walk finish so the slot releases and no blocking task leaks. + for _ in 0..400 { + if sem.available_permits() == 1 { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + if let Some(p) = std::fs::read_to_string(&revlist_pid) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + unsafe { + libc::kill(p, libc::SIGKILL); + } + } + } + /// #174 SC2 (per-source key, U1): the per-caller read sub-cap keys on the /// resolved source IP, NOT the signed DID, so a disposable-DID farm cannot /// multiply its budget. Fill the source IP's single read slot, then drive two From 12643573f6c9f4805bd1876008141edda58f9192 Mon Sep 17 00:00:00 2001 From: t Date: Tue, 14 Jul 2026 20:20:49 -0500 Subject: [PATCH 30/58] fix(node): cap concurrent receive-pack pushes per source IP (#174) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The authenticated git-receive-pack POST acquired only the global write semaphore — no per-source sub-cap, unlike the read and anon-advert pools. Owner enforcement defaults off, so one host minting disposable did:key identities could open max_concurrent_git_pushes slow POSTs from a single source IP and 503 every other source's push; the 600/hour push limiter bounds arrival rate, not in-flight concurrency (P1-d). Add git_write_per_caller (a PerCallerConcurrency sized like the advert cap, max_concurrent_git_pushes/8), and acquire it before the global write permit, keyed on the resolved source IP via read_caller_key — never the signed DID (a DID farm defeats a DID key). This required adding the PeerAddr + HeaderMap extractors to git_receive_pack, which it lacked (without them the key is None and the cap is inert). Neutralized the shared acquire helper's log wording now that it serves both read and write paths. Regression: receive_pack_per_source_write_cap_sheds_capped_source_not_others — a source at its write sub-cap sheds Overloaded/503 (proving the extractors resolve a key), while a different source is not shed. RED before (the capped source proceeds past admission to a git error instead of shedding), GREEN after; the existing receive-pack advert cap test still passes. --- crates/gitlawb-node/src/api/repos.rs | 103 ++++++++++++++++++++++-- crates/gitlawb-node/src/auth/mod.rs | 1 + crates/gitlawb-node/src/main.rs | 7 ++ crates/gitlawb-node/src/state.rs | 7 ++ crates/gitlawb-node/src/test_support.rs | 1 + 5 files changed, 114 insertions(+), 5 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 8612709c..67c6c82a 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -717,7 +717,7 @@ fn acquire_read_caller_permit( Some(k) => match limiter.try_acquire(k) { Some(p) => Ok(Some(p)), None => { - tracing::warn!(repo = %repo, caller = %k, handler, "per-caller read cap reached; shedding with 503"); + tracing::warn!(repo = %repo, caller = %k, handler, "per-caller cap reached; shedding with 503"); Err(AppError::Overloaded( "git service at capacity for this caller, retry shortly".into(), )) @@ -1033,15 +1033,32 @@ pub async fn git_receive_pack( State(state): State, Path((owner, repo)): Path<(String, String)>, Extension(auth): Extension, + crate::rate_limit::PeerAddr(peer): crate::rate_limit::PeerAddr, + headers: axum::http::HeaderMap, body: Bytes, ) -> Result { + let name = smart_http_repo_name(&repo)?; + // Per-source write sub-cap (#174 P1-d): before the global write permit so one + // source IP cannot occupy the whole write pool via many slow authenticated pushes + // and 503 every other source. Owner enforcement defaults off, so any valid did:key + // is accepted (auth != authz), and the 600/hour push limiter bounds arrival RATE, + // not in-flight concurrency — so without this a single host minting disposable DIDs + // saturates the pool. Keyed on the resolved source IP, NEVER the signed DID (a DID + // farm defeats a DID key); no resolvable key -> global write pool only. + let caller_key = read_caller_key(&headers, peer, state.push_limiter_trust); + let _caller_permit = acquire_read_caller_permit( + &state.git_write_per_caller, + caller_key.as_deref(), + name, + "receive-pack", + )?; // Shed with a 503 before spawning git when the concurrency cap is saturated. // Pushes draw from the dedicated WRITE pool, separate from reads, so a flood of - // anonymous reads cannot shed an authenticated push (#174). Acquired at the very - // top so it wraps the write-guard below (and precedes the Tigris acquire_write, - // bounding concurrent fresh acquires — INV-10); held for the whole op. + // anonymous reads cannot shed an authenticated push (#174). Taken after the + // per-source cap above so one source cannot occupy global slots it would be + // sub-cap-denied for; still before the Tigris acquire_write, bounding concurrent + // fresh acquires (INV-10); held for the whole op. let _permit = git_permit(&state.git_write_semaphore)?; - let name = smart_http_repo_name(&repo)?; tracing::info!(owner = %owner, repo = %name, "receive-pack request"); let record = state .db @@ -3606,6 +3623,82 @@ mod tests { } } + /// #174 U4 (P1-d, RED-before/GREEN-after): the authenticated receive-pack POST + /// carries a per-source WRITE sub-cap so one source IP cannot monopolize the write + /// pool with many slow pushes (owner enforcement defaults off, so disposable DIDs + /// are free). Global write pool has capacity; the source is pre-held at its single + /// write slot. A push from THAT source sheds (Overloaded/503) — which also proves + /// the PeerAddr+HeaderMap extractors resolve a key (without them the key is None and + /// the cap is inert, never shedding). A push from a DIFFERENT source is NOT shed by + /// the cap. Called directly so the test needs no signed request; the handler is + /// where the cap lives. Remove the `git_write_per_caller` acquire and the capped + /// source no longer sheds (RED). + #[sqlx::test] + async fn receive_pack_per_source_write_cap_sheds_capped_source_not_others(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + + let mut state = crate::test_support::test_state(pool).await; + // Global write pool has capacity; the per-source cap is 1. + state.git_write_semaphore = Arc::new(Semaphore::new(4)); + state.git_write_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6rp4wr", "rp4", "/tmp/rp4-nonexistent", None, false) + .await + .unwrap(); + + let did = "did:key:z6MkReceivePackWriteCapProofDidAAAAAAAAAA"; + let capped: SocketAddr = "203.0.113.44:5000".parse().unwrap(); + let other: SocketAddr = "203.0.113.45:5000".parse().unwrap(); + + // Pin the capped source at its single write slot. + let _slot = state + .git_write_per_caller + .try_acquire(&capped.ip().to_string()) + .expect("first write slot for the capped source IP"); + + // A push from the capped source must shed on the per-source write cap even with + // global write capacity free. The shed also proves the source-IP key resolved + // via the extractors (an inert None key would fall through to Ok(None)). + let capped_result = git_receive_pack( + State(state.clone()), + Path(("z6rp4wr".to_string(), "rp4".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(capped)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await; + assert!( + matches!(capped_result, Err(AppError::Overloaded(_))), + "a source at its per-source write cap must shed (Overloaded/503) with global \ + pool capacity free; got {capped_result:?}" + ); + + // A push from a DIFFERENT source must NOT be shed by the per-source cap — it + // proceeds past admission (and fails later on the nonexistent repo, which is not + // an Overloaded error). + let other_result = git_receive_pack( + State(state.clone()), + Path(("z6rp4wr".to_string(), "rp4".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(other)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await; + assert!( + !matches!(other_result, Err(AppError::Overloaded(_))), + "a different source must not be shed by the per-source write cap while the \ + capped source holds its slot; got {other_result:?}" + ); + } + /// #174 SC2 (per-source key, U1): the per-caller read sub-cap keys on the /// resolved source IP, NOT the signed DID, so a disposable-DID farm cannot /// multiply its budget. Fill the source IP's single read slot, then drive two diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index 5d827794..5597cd8d 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -526,6 +526,7 @@ mod tests { git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), + git_write_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), git_bin: "git".to_string(), } } diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 18019395..74504f90 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -399,6 +399,13 @@ async fn main() -> Result<()> { git_push_advert_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( (config.max_concurrent_git_pushes / 8).max(1), ), + // Per-source cap on the authenticated receive-pack POST, sized like the advert + // cap: one source IP can hold at most this many write-pool slots, so + // monopolizing the pool takes ~8 distinct source IPs, each also rate-limited + // (#174 P1-d). + git_write_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( + (config.max_concurrent_git_pushes / 8).max(1), + ), git_bin: "git".to_string(), }; diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index eb07a3e4..6ce158e3 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -123,6 +123,13 @@ pub struct AppState { /// a fraction of `max_concurrent_git_pushes`, so filling the write pool takes many /// distinct source IPs (each also braked by the per-IP push rate limiter). pub git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency, + /// Per-source concurrency sub-cap on the authenticated `git-receive-pack` POST: + /// each source IP may hold at most a small share of `git_write_semaphore`, so one + /// host minting disposable `did:key` identities cannot open enough slow pushes to + /// monopolize the write pool and 503 every other source's push (#174 P1-d). Keyed + /// on the resolved source IP (never the DID — a DID farm defeats a DID key). Sized + /// like `git_push_advert_per_caller`, a fraction of `max_concurrent_git_pushes`. + pub git_write_per_caller: crate::rate_limit::PerCallerConcurrency, /// The `git` executable the served-git withheld-blob walk spawns. Production is /// `"git"` (resolved via PATH); injectable so a fake `git` can drive the walk's /// process-group teardown in handler tests without mutating the process-global diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 0805fec1..9241b518 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -90,6 +90,7 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys( 8, ), + git_write_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), git_bin: "git".to_string(), } } From 2a54c159e6984032205545816e0e44e0881f996c Mon Sep 17 00:00:00 2001 From: t Date: Tue, 14 Jul 2026 20:28:52 -0500 Subject: [PATCH 31/58] fix(node): bound concurrent post-push encryption walks with an admission pool (#174) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After a successful path-scoped push the handler released its write permit and then ran a DETACHED tokio task whose spawn_blocking(withheld_blob_recipients_bounded) performed another full-history git walk under no admission. N fast completed pushes spawned N concurrent full-history walks past GITLAWB_MAX_CONCURRENT_GIT_PUSHES (which bounds only the in-handler phase), and the per-child bounding from 71389e6 caps each walk's duration but not their count (P1-e). Add git_encrypt_semaphore, a pool of its own (sized from max_concurrent_git_pushes, no new knob — Q1) so a long background walk never holds a foreground write slot and a handler holding a write permit can't self-deadlock. Route the walk through a new withheld_recipients_gated helper that acquires the pool before the walk and DEFERS (blocks) when full rather than shedding — dropping the walk would lose the withheld-blob recovery copy, so durability stays fail-closed. Regression: encrypt_walk_defers_when_pool_exhausted drives the gated helper with the pool exhausted and asserts the walk blocks and does not run its rev-list, then runs once a permit frees. RED before (the walk runs regardless of the pool), GREEN after. The detached push task calls this exact helper. --- crates/gitlawb-node/src/api/repos.rs | 138 +++++++++++++++++++++--- crates/gitlawb-node/src/auth/mod.rs | 1 + crates/gitlawb-node/src/main.rs | 6 ++ crates/gitlawb-node/src/state.rs | 11 ++ crates/gitlawb-node/src/test_support.rs | 1 + 5 files changed, 144 insertions(+), 13 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 67c6c82a..430b2f7f 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -727,6 +727,37 @@ fn acquire_read_caller_permit( } } +/// Acquire an encryption-walk admission permit, then run the bounded withheld-blob +/// recipients walk. Blocks (defers) when `git_encrypt_semaphore` is full rather than +/// shedding — the walk is background so added latency is fine, and dropping it would +/// lose the withheld-blob recovery copy (#174 P1-e). Bounds the number of concurrent +/// post-push encryption walks so N fast completed pushes cannot spawn N concurrent +/// full-history git walks. Mirrors the original `spawn_blocking(...).await` return +/// shape so the caller's `Ok(Ok(recipients))` match is unchanged. +async fn withheld_recipients_gated( + encrypt_sem: std::sync::Arc, + repo_path: std::path::PathBuf, + git_bin: String, + timeout: std::time::Duration, + rules: Vec, + is_public: bool, + owner_did: String, +) -> std::result::Result< + anyhow::Result>>, + tokio::task::JoinError, +> { + let _permit = encrypt_sem + .acquire_owned() + .await + .expect("git_encrypt_semaphore is never closed"); + tokio::task::spawn_blocking(move || { + crate::git::visibility_pack::withheld_blob_recipients_bounded( + &repo_path, &git_bin, timeout, &rules, is_public, &owner_did, + ) + }) + .await +} + /// Map an error from a `smart_http` git service call to the right `AppError`: /// [`smart_http::GitServiceTimeout`] to 504, a malformed client request to 400, /// anything else to a 500 git error. Pure (no logging) so it is unit-testable; @@ -1329,6 +1360,7 @@ pub async fn git_receive_pack( let repo_name = record.name.clone(); let enc_git_bin = state.git_bin.clone(); let enc_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let encrypt_sem = state.git_encrypt_semaphore.clone(); tokio::spawn(async move { let pinned = crate::ipfs_pin::pin_new_objects( &ipfs_api, @@ -1351,19 +1383,18 @@ pub async fn git_receive_pack( // the has_path_scoped_rule gate on the other two withheld-walk sites. if let Some(rules) = rules_for_enc.filter(|r| visibility_pack::has_path_scoped_rule(r)) { - let p = repo_path_clone.clone(); - let owner = owner_did.clone(); - let git_bin = enc_git_bin.clone(); - let recip = tokio::task::spawn_blocking(move || { - crate::git::visibility_pack::withheld_blob_recipients_bounded( - &p, - &git_bin, - enc_timeout, - &rules, - is_public, - &owner, - ) - }) + // Bound the number of concurrent post-push encryption walks (#174 P1-e): + // acquire an admission permit before the full-history walk, deferring + // when the pool is full rather than shedding the recovery pin. + let recip = withheld_recipients_gated( + encrypt_sem.clone(), + repo_path_clone.clone(), + enc_git_bin.clone(), + enc_timeout, + rules, + is_public, + owner_did.clone(), + ) .await; if let Ok(Ok(recipients)) = recip { let delta = crate::encrypted_pin::encrypt_and_pin( @@ -3699,6 +3730,87 @@ mod tests { ); } + /// #174 U5 (P1-e, RED-before/GREEN-after): the post-push encryption walk acquires a + /// `git_encrypt_semaphore` permit before running, so completed pushes cannot spawn + /// unbounded concurrent full-history walks. With the pool exhausted the gated walk + /// must DEFER (block on admission) and NOT run its rev-list; on the pre-fix code + /// (no acquire) the walk runs regardless of the pool (RED). It defers rather than + /// sheds — releasing the permit lets the SAME walk run and pin (durability stays + /// fail-closed). Exercises the gating seam directly; the detached push task calls + /// this exact helper. + #[tokio::test] + async fn encrypt_walk_defers_when_pool_exhausted() { + use std::sync::Arc; + use std::time::Duration; + use tokio::sync::Semaphore; + + let tmp = tempfile::TempDir::new().unwrap(); + let marker = tmp.path().join("revlist.ran"); + // Fake git records when rev-list runs (the walk's first git call). + let body = format!( + "#!/bin/sh\ncase \"$1\" in\n rev-list) echo ran > \"{}\" ;;\n *) : ;;\nesac\nexit 0\n", + marker.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + let git_bin = git_path.to_str().unwrap().to_string(); + let owner = "did:key:z6MkEncWalkOwnerAAAAAAAAAAAAAAAAAAAAAAAA".to_string(); + + // Exhaust the pool: hold its only permit so a gated walk must defer. + let sem = Arc::new(Semaphore::new(1)); + let held = sem.clone().acquire_owned().await.unwrap(); + + // Blocked: the gated walk must NOT complete or run rev-list while exhausted. + let blocked = tokio::time::timeout( + Duration::from_millis(500), + withheld_recipients_gated( + sem.clone(), + tmp.path().to_path_buf(), + git_bin.clone(), + Duration::from_secs(5), + Vec::new(), + true, + owner.clone(), + ), + ) + .await; + assert!( + blocked.is_err(), + "the encryption walk must defer (block on admission) when the pool is exhausted" + ); + assert!( + !marker.exists(), + "the walk's rev-list must not run while its admission permit is unavailable (P1-e)" + ); + + // Release admission: the SAME walk now runs (defer, not shed) — rev-list fires. + drop(held); + let ran = withheld_recipients_gated( + sem, + tmp.path().to_path_buf(), + git_bin, + Duration::from_secs(5), + Vec::new(), + true, + owner, + ) + .await; + assert!( + ran.is_ok(), + "with a permit the walk runs and joins: {ran:?}" + ); + assert!( + marker.exists(), + "once admission is available the deferred walk runs its rev-list" + ); + } + /// #174 SC2 (per-source key, U1): the per-caller read sub-cap keys on the /// resolved source IP, NOT the signed DID, so a disposable-DID farm cannot /// multiply its budget. Fill the source IP's single read slot, then drive two diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index 5597cd8d..f18b4963 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -523,6 +523,7 @@ mod tests { git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 74504f90..380ab5f4 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -389,6 +389,12 @@ async fn main() -> Result<()> { git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new( config.max_concurrent_git_pushes, )), + // Bounds concurrent detached post-push encryption walks, sized from the push + // pool (no separate knob — Q1): completed pushes cannot outnumber active + // encryption walks past this (#174 P1-e). + git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new( + config.max_concurrent_git_pushes, + )), git_read_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( config.max_concurrent_reads_per_caller, ), diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 6ce158e3..3ff86923 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -110,6 +110,17 @@ pub struct AppState { /// advert pool (each source also capped by `git_push_advert_per_caller` and the /// per-IP push rate limiter), and the reserved POST pool is untouched (#174). pub git_push_advert_semaphore: Arc, + /// Bounds concurrent post-push encrypt-then-pin history walks. Each successful + /// path-scoped push releases its handler write permit and then runs a DETACHED + /// full-history walk (`withheld_blob_recipients_bounded`) to seal withheld blobs; + /// without a cap, N fast pushes spawn N concurrent full-history git walks past + /// `max_concurrent_git_pushes` (which only bounds the in-handler phase). The walk + /// acquires a permit here and DEFERS (blocks) when the pool is full rather than + /// shedding — the work is background and dropping it would lose the recovery copy + /// (#174 P1-e). A pool of its own, not `git_write_semaphore`: a long background + /// walk must not hold a foreground write slot, and a handler already holding a + /// write permit that needed a second would self-deadlock at pool size 1. + pub git_encrypt_semaphore: Arc, /// Per-caller concurrency sub-cap on the read pool: each caller (keyed on the /// resolved source IP, #174 U1) may hold at most `max_concurrent_reads_per_caller` /// in-flight read ops, so one caller cannot monopolize `git_read_semaphore` diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 9241b518..a27be946 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -86,6 +86,7 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys( 8, From 5749df6e67be0d06fd99b69f0a64156ba3df29ca Mon Sep 17 00:00:00 2001 From: t Date: Tue, 14 Jul 2026 20:33:01 -0500 Subject: [PATCH 32/58] test(node): INV-22 completeness guard for the served-git concurrency surface (#174) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A source-scan tripwire that fails when a new site reintroduces the concurrency-cap class PR #174 fixed, or when one of the five gates (U1-U5) is removed. INV-22: a permit held per op recovers only if every path is duration-bounded and reaps the group before releasing admission, and every detached git task carries admission. Lives in tests/ (a separate crate) on purpose — a guard scanning the file it lives in would match its own identifier literals and pass vacuously; scanning src/ from here keeps every check load-bearing. Checks: run_bounded_git confirms child exit via child_terminated_without_reaping (U1); KillGroupOnDrop launches the reaper via Handle::try_current (U2); git_receive_pack acquires state.git_write_per_caller (U4); the encryption walk runs through withheld_recipients_gated over git_encrypt_semaphore (U5); and the bounded recipients walk is invoked exactly once — only inside the gated helper — so a new spawn_blocking bypass (count > 1) fails (P1-e non-bypass). Proven load-bearing: a synthetic second call site turns the count tripwire RED, and each named gate maps to a fix whose own regression already goes RED on revert. --- crates/gitlawb-node/tests/inv22_gates.rs | 80 ++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 crates/gitlawb-node/tests/inv22_gates.rs diff --git a/crates/gitlawb-node/tests/inv22_gates.rs b/crates/gitlawb-node/tests/inv22_gates.rs new file mode 100644 index 00000000..5f576300 --- /dev/null +++ b/crates/gitlawb-node/tests/inv22_gates.rs @@ -0,0 +1,80 @@ +//! #174 U6 — INV-22 completeness guard (rung-raising). +//! +//! INV-22: a permit held per op recovers only if every path that holds it is also +//! duration-bounded and reaps the process group before releasing admission, and every +//! detached git/blocking task carries its own admission. PR #174 fixed five paths +//! (U1-U5) that violated this. Each fix has a per-unit RED/GREEN regression; together +//! those form the five-revert matrix. This guard adds the missing piece: a source-scan +//! tripwire that fails when a NEW site reintroduces the class, or when one of the five +//! gates is removed. +//! +//! It lives in `tests/` (a separate crate) on purpose: a guard that scanned the same +//! file it lives in would match its own identifier literals and pass vacuously. Here +//! the scanned `src/` files never contain this file's literals, so each check is +//! load-bearing — reverting the named gate turns the assertion red. +//! +//! These are deliberately coarse structural checks, not a parser. They cannot prove a +//! gate is *correct* (the per-unit tests do that); they prove a gate is *present and +//! not bypassed*, which is what stops the class from silently regressing. + +use std::path::Path; + +fn src(rel: &str) -> String { + let p = Path::new(env!("CARGO_MANIFEST_DIR")).join("src").join(rel); + std::fs::read_to_string(&p).unwrap_or_else(|e| panic!("read {}: {e}", p.display())) +} + +#[test] +fn inv22_concurrency_gates_present_and_not_bypassed() { + let repos = src("api/repos.rs"); + let smart_http = src("git/smart_http.rs"); + let vis = src("git/visibility_pack.rs"); + + // U1 / P1-a: run_bounded_git stands the watchdog down only after confirming the + // child actually terminated (WNOWAIT), not on the raw stdout-drain EOF — otherwise + // a child that closes stdout then hangs pins the permit past the deadline. The + // probe is defined and called, so >= 2 occurrences; reverting the fix removes both. + assert!( + vis.matches("child_terminated_without_reaping").count() >= 2, + "U1/P1-a gate missing: run_bounded_git must confirm child exit via \ + child_terminated_without_reaping before signalling the watchdog" + ); + + // U2 / P1-c: on client disconnect KillGroupOnDrop must launch a detached reaper + // that runs the full TERM/grace/KILL/reap, not a lone SIGTERM. The reaper is spawned + // via a runtime handle in Drop; `Handle::try_current` is unique to that launch (the + // timeout path already has an async context and never calls it). + assert!( + smart_http.contains("Handle::try_current"), + "U2/P1-c gate missing: KillGroupOnDrop::drop must launch the full \ + TERM/grace/KILL reaper on disconnect (Handle::try_current), not a lone SIGTERM" + ); + + // U4 / P1-d: git_receive_pack must acquire the per-source write sub-cap before the + // global write permit. The acquire reads `state.git_write_per_caller`; comments name + // the field without the `state.` prefix, so this targets the real acquire site. + assert!( + repos.contains("state.git_write_per_caller"), + "U4/P1-d gate missing: git_receive_pack must acquire the per-source write cap \ + (state.git_write_per_caller) before the global write permit" + ); + + // U5 / P1-e: the detached post-push encryption walk must run through the + // admission-gated helper, which is wired to the shared encrypt pool. + assert!( + repos.contains("fn withheld_recipients_gated") + && repos.contains("state.git_encrypt_semaphore"), + "U5/P1-e gate missing: the encryption walk must run through \ + withheld_recipients_gated, which acquires git_encrypt_semaphore" + ); + + // P1-e non-bypass tripwire: the bounded recipients walk is spawn_blocking'd nowhere + // but inside withheld_recipients_gated. A second call site (count > 1) is a new + // detached git walk that skips the admission gate — exactly the class U5 closed. + assert_eq!( + repos.matches("withheld_blob_recipients_bounded").count(), + 1, + "P1-e bypass: the bounded recipients walk must be invoked only inside \ + withheld_recipients_gated; a new call site bypasses the encrypt-walk admission cap" + ); +} From 72e899dae022dbc757be4198161629dad5245a92 Mon Sep 17 00:00:00 2001 From: t Date: Wed, 15 Jul 2026 11:33:20 -0500 Subject: [PATCH 33/58] fix(node): hold served-git admission until the process group is reaped on the plain spawn paths (#174) On the plain (non-path-scoped) info_refs / upload-pack / receive-pack paths the global + per-source admission permits were handler-locals that dropped the instant the handler future dropped on a client disconnect, while KillGroupOnDrop reaps the git process group in a DETACHED task (SIGTERM -> grace -> SIGKILL -> reap). So a disconnect-spammer admitted replacements while prior groups were still alive, defeating the process cap and the per-source cap during teardown. be0cdd6 already fixed this for the path-scoped upload-pack walk; this closes the residual plain paths (P1-a). Introduce AdmissionGuard, a move-only holder of the permits, threaded through info_refs/upload_pack/receive_pack/run_git_service into drive_git_child's KillGroupOnDrop. On disconnect the guard moves into the detached reaper and drops only after the group is ESRCH-confirmed reaped; on success/timeout disarm() returns it for the earliest provably-free drop. The rev-list/pack-objects visibility-walk callers hold no admission and pass None. Thread git_bin through upload_pack/receive_pack (they hardcoded "git") so the plain POST path is drivable by an injected fake git, matching the existing injectable seam on info_refs/run_git_service/upload_pack_excluding. Handler-layer regression upload_pack_plain_permit_held_through_group_reap_after_disconnect: isolates the global read pool (size 1, per-source + rate limiter permissive), a SIGTERM-ignoring descendant keeps the group alive across the reap window, and asserts available_permits()==0 held on disconnect + a cross-source replacement sheds 503, then frees after ESRCH. Reverting the guard-into-reaper move (release immediately) turns it RED at the held-through-reap assertion. Plus a None-key arm. --- crates/gitlawb-node/src/api/repos.rs | 269 +++++++++++++++++++++- crates/gitlawb-node/src/git/smart_http.rs | 130 +++++++++-- 2 files changed, 382 insertions(+), 17 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 430b2f7f..16a8ed0d 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -653,8 +653,14 @@ pub async fn git_info_refs( AppError::Git(e.to_string()) })?; + // Move the admission permits into the guard so they release only after the spawned + // git process group is confirmed reaped, on complete/timeout/disconnect — not the + // instant a disconnect drops this future while the detached reaper is still tearing + // the group down (#174 P1-a). The handler keeps no copy: `_permit`/`_caller_permit` + // are moved in, so admission tracks the real process lifetime. + let admission = smart_http::AdmissionGuard::new(_permit, _caller_permit); let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); - smart_http::info_refs("git", &service, &disk_path, git_timeout) + smart_http::info_refs("git", &service, &disk_path, git_timeout, Some(admission)) .await .map_err(|e| { let app = git_service_app_error(&e); @@ -848,7 +854,12 @@ pub async fn git_upload_pack( // withheld walk and serve the pack directly. let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); let resp = if !visibility_pack::has_path_scoped_rule(&rules) { - smart_http::upload_pack(&disk_path, body, git_timeout).await + // Plain (non-path-scoped) serve: move both admission permits into the guard so + // they release only after the spawned git group is reaped, on + // complete/timeout/disconnect — not the instant a disconnect drops this future + // (#174 P1-a). The handler keeps no copy. + let admission = smart_http::AdmissionGuard::new(_permit, _caller_permit); + smart_http::upload_pack(&state.git_bin, &disk_path, body, git_timeout, Some(admission)).await } else { // withheld_blob_oids walks every ref with blocking `git ls-tree`; keep that // off the async worker thread. Move BOTH admission permits INTO the blocking @@ -886,9 +897,17 @@ pub async fn git_upload_pack( let withheld = withheld.map_err(|e| git_service_app_error(&e))?; if withheld.is_empty() { - smart_http::upload_pack(&disk_path, body, git_timeout).await + // No blobs to withhold: serve the plain pack, moving the permits returned by + // the walk into the guard so admission tracks the served git group's reap + // (the walk already held them per be0cdd6; this hands them to the serve). + let admission = smart_http::AdmissionGuard::new(_permit, _caller_permit); + smart_http::upload_pack(&state.git_bin, &disk_path, body, git_timeout, Some(admission)).await } else { tracing::info!(repo = %name, caller = ?caller, withheld = withheld.len(), "serving filtered pack"); + // upload_pack_excluding runs its own rev-list/pack-objects (both pass `None` + // admission internally); the walk's permits stay handler-locals held across + // this serve, as be0cdd6 established, and drop when the handler returns. + let _hold = (_permit, _caller_permit); smart_http::upload_pack_excluding(&disk_path, body, &withheld, git_timeout).await } } @@ -1172,7 +1191,14 @@ pub async fn git_receive_pack( tracing::debug!(repo = %name, path = %disk_path.display(), "running git receive-pack"); let body_len = body.len(); let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); - let receive_result = smart_http::receive_pack(&disk_path, body, git_timeout).await; + // Move both admission permits into the guard so they release only after the spawned + // receive-pack process group is reaped, on complete/timeout/disconnect — not the + // instant a disconnect drops this future while the detached reaper runs (#174 P1-a). + // The handler keeps no copy. This is independent of the write-lock `guard.release` + // below: admission tracks the git process lifetime, the write lock tracks the repo. + let admission = smart_http::AdmissionGuard::new(_permit, _caller_permit); + let receive_result = + smart_http::receive_pack(&state.git_bin, &disk_path, body, git_timeout, Some(admission)).await; // Always release the advisory lock — even on error — to prevent stale locks // from blocking subsequent pushes. Only upload to Tigris when the push @@ -3654,6 +3680,241 @@ mod tests { } } + /// #174 U1 (P1-a, plain-spawn residual, RED-before/GREEN-after): on the PLAIN + /// (non-path-scoped) upload-pack path a client disconnect must NOT release the + /// global read admission while the detached process-group reaper is still tearing + /// down a git group that ignores SIGTERM. The `be0cdd6` fix moved permits into the + /// path-scoped `spawn_blocking` walk; this closes the residual plain path, where the + /// permits were handler-locals that dropped the instant the future was dropped. + /// + /// Isolate the GLOBAL pool: read pool = 1, per-source cap + rate limiter permissive, + /// so the only thing that can shed a replacement is the leaked global permit. Drive + /// the handler until git spawns, disconnect, then assert the global slot stays held + /// (`available_permits() == 0`) AND a replacement sheds 503 while the group is alive; + /// after the reaper SIGKILLs+reaps the group the slot frees and a replacement is no + /// longer shed by the global cap. On the pre-fix code the handler-local permit drops + /// on future-drop and the slot frees at once (RED). + #[cfg(unix)] + #[sqlx::test] + async fn upload_pack_plain_permit_held_through_group_reap_after_disconnect(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + let tmp = tempfile::TempDir::new().unwrap(); + let descfile = tmp.path().join("desc.pid"); + // Fake git for the plain upload-pack path (invoked as `git upload-pack + // --stateless-rpc `). It forks a descendant that TRAPS SIGTERM, records its + // pid, and loops ~20s, then `wait`s — so on disconnect the group leader dies on + // the reaper's SIGTERM but the descendant survives until the reaper escalates to + // SIGKILL, keeping the group alive (ESRCH not reached) across the observation + // window. Bounded so a broken fix leaks no permanent orphan. + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + upload-pack)\n\ + sh -c 'trap \"\" TERM; echo $$ > \"{}\"; i=0; while [ $i -lt 20 ]; do sleep 1; i=$((i+1)); done' &\n\ + wait ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + descfile.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + // Isolate the global read pool: size 1; per-source cap + rate limiter permissive + // so only the leaked global permit can shed the replacement. + state.git_read_semaphore = Arc::new(Semaphore::new(1)); + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + state.git_bin = git_path.to_str().unwrap().to_string(); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let owner = "z6up1st"; + let name = "up1"; + state + .db + .upsert_mirror_repo(owner, name, "/unused", None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + // Real bare repo at the path acquire() computes, so the handler reaches the + // spawn. No path-scoped rule -> the PLAIN serve branch (this test's target). + state + .repo_store + .init(&rec.owner_did, &rec.name) + .await + .unwrap(); + + let sem = state.git_read_semaphore.clone(); + assert_eq!(sem.available_permits(), 1, "one read slot before the request"); + + let router = crate::server::build_router(state); + let make_req = |peer: SocketAddr| { + let mut req = Request::builder() + .method(Method::POST) + .uri(format!("/{owner}/{name}/git-upload-pack")) + .body(Body::from(&b"0000"[..])) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + req + }; + + let peer: SocketAddr = "203.0.113.71:5000".parse().unwrap(); + let mut fut = Box::pin(router.clone().oneshot(make_req(peer))); + // Drive until git spawns (the descendant records its pid) — the request is + // inside the plain serve, holding the global read permit. Stop polling the + // instant the future completes (re-polling a completed oneshot panics); read the + // descfile first so a spawn that recorded its pid then returned is still caught. + let mut spawned: Option = None; + let mut early = None; + for _ in 0..500 { + let done = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + if let Some(p) = std::fs::read_to_string(&descfile) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + spawned = Some(p); + break; + } + if let Ok(resp) = done { + early = Some(resp.map(|r| r.status())); + break; + } + } + let desc = spawned + .unwrap_or_else(|| panic!("the fake git must have spawned; early finish: {early:?}")); + // Kill the descendant regardless of outcome so a RED run leaks no orphan. + struct ReapOnDrop(i32); + impl Drop for ReapOnDrop { + fn drop(&mut self) { + unsafe { + libc::kill(self.0, libc::SIGKILL); + } + } + } + let _cleanup = ReapOnDrop(desc); + assert!( + unsafe { libc::kill(desc, 0) == 0 }, + "descendant should be running before the disconnect" + ); + assert_eq!( + sem.available_permits(), + 0, + "the read slot is held while the git op runs" + ); + + // Client disconnect: drop the request future. The detached reaper now owns the + // AdmissionGuard and will not drop it until the group is ESRCH-confirmed reaped. + drop(fut); + + // Load-bearing: the slot must STAY held while the SIGTERM-ignoring group is still + // alive. On the pre-fix code the handler-local permit drops here and the slot + // frees at once (RED). Check quickly (before the reaper's ~2s SIGKILL escalation). + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + assert!( + unsafe { libc::kill(desc, 0) == 0 }, + "the SIGTERM-ignoring descendant must still be alive during the hold window" + ); + assert_eq!( + sem.available_permits(), + 0, + "on disconnect the read admission must be HELD until the process group is \ + reaped, not released the instant the future drops (P1-a)" + ); + // A replacement request from a DIFFERENT source must shed 503 — the only pool + // that can shed it is the leaked global permit (per-source cap is permissive). + let peer2: SocketAddr = "203.0.113.72:5000".parse().unwrap(); + let resp = router.clone().oneshot(make_req(peer2)).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "while the prior group is still alive the held global permit must shed a \ + replacement with 503" + ); + + // After the reaper SIGKILLs + reaps the group the AdmissionGuard drops and the + // slot frees. Poll for recovery. + let mut freed = false; + for _ in 0..400 { + if sem.available_permits() == 1 { + freed = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + assert!( + freed, + "once the reaper confirms the group gone the admission guard must drop and \ + free the global slot" + ); + // A replacement is now no longer shed by the global cap (it proceeds past + // admission; it then fails downstream on the fake git, which is not a 503). + let peer3: SocketAddr = "203.0.113.73:5000".parse().unwrap(); + let resp = router.oneshot(make_req(peer3)).await.unwrap(); + assert_ne!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "after the group is reaped the freed slot must admit a replacement" + ); + } + + /// #174 U1 (P1-a): the `None`-key arm — a request with no resolvable source key + /// (no trusted-proxy header, no peer) is bounded by the GLOBAL read pool only, never + /// a per-source cap. With the global read pool exhausted such a request still sheds + /// 503, proving the plain path admits/sheds on the global pool for the `None` arm + /// (the counterpart to the `Some(ip)` arm above). Complements the resolver-arm rule: + /// neither arm is vacuous. + #[tokio::test] + async fn upload_pack_plain_none_key_arm_sheds_on_global_pool() { + use axum::body::Body; + use axum::http::{Method, Request, StatusCode}; + use axum::Router; + use std::sync::Arc; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state_lazy(); + // Global read pool exhausted; per-source cap permissive so only the global pool + // can shed. No ConnectInfo + no trusted header -> read_caller_key resolves None. + state.git_read_semaphore = Arc::new(Semaphore::new(0)); + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let router = Router::new() + .route( + "/{owner}/{repo}/git-upload-pack", + axum::routing::post(crate::api::repos::git_upload_pack), + ) + .with_state(state); + // No ConnectInfo extension and no XFF header: the caller key is None. + let req = Request::builder() + .method(Method::POST) + .uri("/alice/repo.git/git-upload-pack") + .body(Body::from(&b"0000"[..])) + .unwrap(); + let resp = router.oneshot(req).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a None-key request must still shed 503 on the exhausted GLOBAL read pool" + ); + } + /// #174 U4 (P1-d, RED-before/GREEN-after): the authenticated receive-pack POST /// carries a per-source WRITE sub-cap so one source IP cannot monopolize the write /// pool with many slow pushes (owner enforcement defaults off, so disposable DIDs diff --git a/crates/gitlawb-node/src/git/smart_http.rs b/crates/gitlawb-node/src/git/smart_http.rs index a5649a42..64d3cb7f 100644 --- a/crates/gitlawb-node/src/git/smart_http.rs +++ b/crates/gitlawb-node/src/git/smart_http.rs @@ -10,6 +10,40 @@ use std::time::{Duration, Instant}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::process::Command; +/// Owns the served-git admission permits (global + per-source) for the lifetime of +/// the work they admitted, so admission is released only when that work is truly +/// done — not the instant the handler future drops on a client disconnect. +/// +/// A move-only wrapper: no methods beyond construction and `Drop`. The handler +/// MOVEs its permits in and keeps no copy (a retained copy would drop early and +/// release admission the moment the future is dropped, defeating the guard). It is +/// threaded into `drive_git_child`, whose [`KillGroupOnDrop`] moves it into the +/// detached reaper on disconnect, so both permits drop only after the process group +/// is confirmed reaped (`kill(-pgid,0)==ESRCH`) rather than while the group is still +/// alive holding PIDs past the concurrency cap (#174 P1-a, plain-spawn residual). +/// +/// The `be0cdd6` path-scoped upload-pack walk already applies this discipline by +/// moving its permits into the `spawn_blocking`; this generalizes it to the plain +/// (non-path-scoped) `info_refs` / `run_git_service` spawn paths. +pub struct AdmissionGuard { + // Boxed so the concrete permit types stay private to the caller (repos.rs) — the + // guard only needs to hold them and drop them, never inspect them. `Send + + // 'static` so the guard can move into the detached reaper task. + _global: Option>, + _caller: Option>, +} + +impl AdmissionGuard { + /// Take ownership of the global permit and an optional per-caller permit. Both are + /// erased to `Box` — the guard's only job is to hold them until it drops. + pub fn new(global: impl Send + 'static, caller: Option) -> Self { + Self { + _global: Some(Box::new(global)), + _caller: caller.map(|c| Box::new(c) as Box), + } + } +} + /// Handle `GET /:owner/:repo/info/refs?service=git-upload-pack` /// or `?service=git-receive-pack` /// @@ -26,6 +60,7 @@ pub async fn info_refs( service: &str, repo_path: &Path, timeout: Duration, + admission: Option, ) -> Result { validate_service(service)?; @@ -36,7 +71,8 @@ pub async fn info_refs( .arg("--advertise-refs") .arg(repo_path); // No request body — advertise-refs does not read stdin. - let stdout = drive_git_child(command, Bytes::new(), timeout, "advertise-refs").await?; + let stdout = + drive_git_child(command, Bytes::new(), timeout, "advertise-refs", admission).await?; let content_type = format!("application/x-{service}-advertisement"); @@ -61,12 +97,21 @@ pub async fn info_refs( /// Serves pack data for a clone or fetch. This is stateless — the entire /// negotiation happens in a single request/response. pub async fn upload_pack( + git_bin: &str, repo_path: &Path, request_body: Bytes, timeout: Duration, + admission: Option, ) -> Result { - let output = - run_git_service("git", "git-upload-pack", repo_path, request_body, timeout).await?; + let output = run_git_service( + git_bin, + "git-upload-pack", + repo_path, + request_body, + timeout, + admission, + ) + .await?; Ok(Response::builder() .status(StatusCode::OK) @@ -80,12 +125,21 @@ pub async fn upload_pack( /// Accepts a push. The caller MUST verify HTTP Signature auth before /// calling this function. pub async fn receive_pack( + git_bin: &str, repo_path: &Path, request_body: Bytes, timeout: Duration, + admission: Option, ) -> Result { - let output = - run_git_service("git", "git-receive-pack", repo_path, request_body, timeout).await?; + let output = run_git_service( + git_bin, + "git-receive-pack", + repo_path, + request_body, + timeout, + admission, + ) + .await?; Ok(Response::builder() .status(StatusCode::OK) @@ -120,6 +174,12 @@ struct KillGroupOnDrop { // own SIGCHLD-driven orphan reaper. child: Option, pgid: Option, + // The global + per-source admission permits for this op, if any. On the plain + // (non-path-scoped) spawn paths repos.rs moves its permits in here so admission is + // released only when the group is confirmed reaped, on every exit — complete, + // timeout, or client-disconnect (#174 P1-a). The rev-list/pack-objects + // visibility-walk callers hold no admission and pass `None`. + admission: Option, } #[cfg(unix)] @@ -131,17 +191,25 @@ impl KillGroupOnDrop { /// Disarm on the success/timeout path: the body has already reaped the child (its /// `wait()` returned, or `reap_group_on_timeout` ran), so drop the handle and clear - /// the pgid, leaving the guard's Drop a no-op. - fn disarm(&mut self) { + /// the pgid, leaving the guard's Drop a no-op. Returns the admission guard so the + /// caller drops it at the earliest provably-free point (the group is already reaped + /// on both callers of this) rather than at end-of-bookkeeping. + fn disarm(&mut self) -> Option { self.pgid = None; self.child = None; + self.admission.take() } } #[cfg(unix)] impl Drop for KillGroupOnDrop { fn drop(&mut self) { + // Move any admission guard out first so it travels with the reaper (or drops + // here on the no-child / no-runtime fallback), never before the group is gone. + let admission = self.admission.take(); let (Some(mut child), Some(pgid)) = (self.child.take(), self.pgid) else { + // Nothing to reap (already disarmed); dropping `admission` here is correct — + // the group is already gone. return; }; // A sync Drop cannot await, so launch a detached reaper that owns the child and @@ -154,6 +222,13 @@ impl Drop for KillGroupOnDrop { Ok(handle) => { handle.spawn(async move { reap_group_on_timeout(&mut child).await; + // Release admission only now: the group is ESRCH-confirmed gone (or + // hit the ~4s D-state hard cap, past which nothing in userspace can + // free the PIDs anyway). Holding the permits until here is what + // stops disconnect-spam from admitting replacements while the prior + // group is still alive (#174 P1-a). `admission` is moved in and + // dropped when this closure ends. + drop(admission); }); } Err(_) => { @@ -162,6 +237,9 @@ impl Drop for KillGroupOnDrop { unsafe { libc::kill(-pgid, libc::SIGTERM); } + // No runtime to await the reap; drop admission best-effort after the + // synchronous signal (the fallback path is not on the hot request path). + drop(admission); } } } @@ -246,13 +324,14 @@ async fn run_git_service( repo_path: &Path, input: Bytes, timeout: Duration, + admission: Option, ) -> Result> { let mut command = Command::new(git_bin); command .arg(service_to_command(service)) .arg("--stateless-rpc") .arg(repo_path); - drive_git_child(command, input, timeout, service).await + drive_git_child(command, input, timeout, service, admission).await } /// Drive a spawned git child under `timeout` with process-group teardown, returning @@ -267,6 +346,7 @@ async fn drive_git_child( input: Bytes, timeout: Duration, what: &str, + admission: Option, ) -> Result> { command .stdin(Stdio::piped()) @@ -300,7 +380,12 @@ async fn drive_git_child( let mut group_guard = KillGroupOnDrop { child: Some(child), pgid, + admission, }; + // On non-unix there is no process-group teardown, so hold the admission guard here + // for the child's whole interaction; it drops when this function returns. + #[cfg(not(unix))] + let _admission = admission; let mut out = Vec::new(); let mut err = Vec::new(); @@ -339,18 +424,21 @@ async fn drive_git_child( // The join runs all arms to completion, so the child is reaped: disarm // before surfacing any interaction error (a read/wait error), else the // guard's drop would reap an already-reaped child / signal a reused pgid. + // Dropping the returned admission guard here releases the permits at the + // earliest provably-free point on the success path (the op finished). #[cfg(unix)] - group_guard.disarm(); + drop(group_guard.disarm()); result? } Err(_elapsed) => { // Timeout: tear the whole group down and reap it before returning so a // caller releasing a write lock can't race a still-live git. Then disarm - // the (now redundant) guard so its drop can't hit a reused pgid. + // the (now redundant) guard so its drop can't hit a reused pgid. The + // returned admission guard drops here, AFTER the group is confirmed reaped. #[cfg(unix)] { reap_group_on_timeout(group_guard.child_mut()).await; - group_guard.disarm(); + drop(group_guard.disarm()); } #[cfg(not(unix))] { @@ -415,7 +503,9 @@ async fn rev_list_keep( command .args(["rev-list", "--objects", "--all"]) .current_dir(repo_path); - let stdout = drive_git_child(command, Bytes::new(), timeout, "rev-list").await?; + // The visibility-walk callers intentionally hold no admission permit (their + // admission is governed elsewhere), so pass `None` (#174 KTD2). + let stdout = drive_git_child(command, Bytes::new(), timeout, "rev-list", None).await?; let mut keep = Vec::new(); for line in String::from_utf8_lossy(&stdout).lines() { let oid = line.split_whitespace().next().unwrap_or(""); @@ -464,6 +554,8 @@ pub async fn build_filtered_pack( Bytes::from(data), deadline.saturating_duration_since(Instant::now()), "pack-objects", + // Visibility-walk pack build: no admission permit here (#174 KTD2). + None, ) .await } @@ -736,7 +828,7 @@ mod tests { axum::extract::Query(q): axum::extract::Query>, ) -> Response { let service = q.get("service").cloned().unwrap_or_default(); - info_refs("git", &service, &st.repo, Duration::from_secs(30)) + info_refs("git", &service, &st.repo, Duration::from_secs(30), None) .await .unwrap() } @@ -1021,6 +1113,7 @@ mod tests { let _guard = KillGroupOnDrop { child: Some(child), pgid: Some(pid), + admission: None, }; } @@ -1078,6 +1171,7 @@ mod tests { let _guard = KillGroupOnDrop { child: Some(child), pgid: Some(pid), + admission: None, }; } @@ -1114,6 +1208,7 @@ mod tests { let mut guard = KillGroupOnDrop { child: Some(child), pgid: Some(pid), + admission: None, }; guard.disarm(); } // disarmed -> no reaper, no kill @@ -1320,6 +1415,7 @@ mod tests { tmp.path(), Bytes::new(), Duration::from_secs(60), + None, )); // Advance the future a slice at a time until the fake records its @@ -1421,6 +1517,7 @@ mod tests { tmp.path(), Bytes::new(), Duration::from_secs(60), + None, )); // Drive the future a slice at a time until the descendant records its pid. let mut desc: Option = None; @@ -1482,6 +1579,7 @@ mod tests { "git-upload-pack", tmp.path(), Duration::from_millis(200), + None, ), ) .await @@ -1589,6 +1687,7 @@ mod tests { tmp.path(), Bytes::new(), Duration::from_secs(60), + None, ) }) .await; @@ -1633,6 +1732,7 @@ mod tests { tmp.path(), Bytes::new(), Duration::from_secs(60), + None, ) }) .await; @@ -1683,6 +1783,7 @@ mod tests { tmp.path(), Bytes::new(), git_timeout, + None, ), ) }) @@ -1757,6 +1858,7 @@ mod tests { tmp.path(), Bytes::new(), git_timeout, + None, ), ) }) @@ -1794,6 +1896,7 @@ mod tests { tmp.path(), Bytes::new(), Duration::from_millis(200), + None, ), ) .await; @@ -1834,6 +1937,7 @@ mod tests { tmp.path(), big, Duration::from_secs(60), + None, ) .await; From 8d17038e5bc1e37cc76c931288c4d993a17b39c3 Mon Sep 17 00:00:00 2001 From: t Date: Wed, 15 Jul 2026 12:00:02 -0500 Subject: [PATCH 34/58] fix(node): bound the served-git storage-acquisition phase with a release-on-expiry deadline (#174) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The served-git admission permit is acquired, then RepoStore::acquire*/advisory-lock runs before the bounded git runner starts — with no local deadline. acquire/acquire_fresh await Tigris HEAD/GET, and acquire_write's advisory-lock loop awaits a per-iteration pg_try_advisory_lock that can block indefinitely on a hung Postgres pool. Since GITLAWB_GIT_SERVICE_TIMEOUT_SECS only starts once git spawns, a stalled backend pins the permit and drains the pool until every later request 503s (P1-2). Wrap each git-path acquire (git_info_refs read/advert, git_upload_pack, git_receive_pack) and the /ipfs per-repo acquire loop in tokio::time::timeout(git_acquire_timeout_secs, ..). On expiry the git paths return AppError::Overloaded (503 + Retry-After); the permit is a handler-local at that point (moved into the AdmissionGuard only after acquire) so the early return frees the slot. The /ipfs per-repo acquire keeps its fail-closed `continue` on timeout (never serves an un-acquired repo). No repo_store.rs signature change: the outer timeout cancels a mid-sleep/mid-fetch_one future, so no in-loop deadline is needed. New knob GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS (default 30, range 1..), modeled on git_service_timeout_secs, kept separate because acquisition and git execution are distinct cost centers. Handler-layer regression receive_pack_acquire_deadline_sheds_and_releases_permit: holds the same pg advisory lock acquire_write derives on a second connection so the real loop must retry, drives git-receive-pack with the deadline at 2s and write pool size 1, and asserts 503 at ~2s + the permit recovers + a follow-up admits once the lock frees. Removing the timeout wrapper turns it RED (blocks ~31s to the test ceiling on the ~59s advisory loop). --- crates/gitlawb-node/src/api/ipfs.rs | 21 +- crates/gitlawb-node/src/api/repos.rs | 307 ++++++++++++++++++++++++--- crates/gitlawb-node/src/config.rs | 21 ++ 3 files changed, 315 insertions(+), 34 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index e921c39f..284c9398 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -114,9 +114,24 @@ pub async fn get_by_cid( continue; } - let repo_path = match state.repo_store.acquire(&repo.owner_did, &repo.name).await { - Ok(p) => p, - Err(_) => continue, + // Bound the per-repo acquire under `git_acquire_timeout_secs`: this loop shares + // the P1-2 stall vector (a hung Tigris HEAD/GET on one repo would otherwise + // block the whole /ipfs request). On expiry keep the existing fail-closed skip — + // never serve an un-acquired repo; a public copy (if any) still gets its turn. + let acquire_deadline = + std::time::Duration::from_secs(state.config.git_acquire_timeout_secs); + let repo_path = match tokio::time::timeout( + acquire_deadline, + state.repo_store.acquire(&repo.owner_did, &repo.name), + ) + .await + { + Ok(Ok(p)) => p, + Ok(Err(_)) => continue, + Err(_elapsed) => { + tracing::warn!(repo = %repo.name, "repo acquire timed out during /ipfs walk; skipping repo"); + continue; + } }; // Check whether the object exists in this repo before any expensive diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 16a8ed0d..5b11404d 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -637,21 +637,37 @@ pub async fn git_info_refs( // For receive-pack (push), download the latest from Tigris so the client // sees the same refs that acquire_write() will operate on. - let disk_path = if service == "git-receive-pack" { - state - .repo_store - .acquire_fresh(&record.owner_did, &record.name) - .await - } else { - state - .repo_store - .acquire(&record.owner_did, &record.name) - .await - } - .map_err(|e| { - tracing::error!(repo = %name, service = %service, err = %e, "repo acquire failed"); - AppError::Git(e.to_string()) - })?; + // + // Bound the acquire under `git_acquire_timeout_secs`: the concurrency permit is + // already held above, and `git_service_timeout_secs` only starts once git spawns, + // so an un-deadlined acquire (a hung Tigris HEAD/GET here) pins the permit until + // the pool drains (#174 P1-2). On expiry the handler-local `_permit`/`_caller_permit` + // drop on the early return (the AdmissionGuard is not built until after acquire), + // so the shed frees the slot; return a bounded 503. + let acquire_deadline = std::time::Duration::from_secs(state.config.git_acquire_timeout_secs); + let acquire_fut = async { + if service == "git-receive-pack" { + state + .repo_store + .acquire_fresh(&record.owner_did, &record.name) + .await + } else { + state + .repo_store + .acquire(&record.owner_did, &record.name) + .await + } + }; + let disk_path = tokio::time::timeout(acquire_deadline, acquire_fut) + .await + .map_err(|_elapsed| { + tracing::warn!(repo = %name, service = %service, "repo acquire timed out; shedding with 503"); + AppError::Overloaded("git service acquisition timed out, retry shortly".into()) + })? + .map_err(|e| { + tracing::error!(repo = %name, service = %service, err = %e, "repo acquire failed"); + AppError::Git(e.to_string()) + })?; // Move the admission permits into the guard so they release only after the spawned // git process group is confirmed reaped, on complete/timeout/disconnect — not the @@ -842,11 +858,21 @@ pub async fn git_upload_pack( // exec (INV-10). let _permit = git_permit(&state.git_read_semaphore)?; - let disk_path = state - .repo_store - .acquire(&record.owner_did, &record.name) - .await - .map_err(|e| AppError::Git(e.to_string()))?; + // Bound the acquire under `git_acquire_timeout_secs` so a hung Tigris HEAD/GET + // cannot pin the read permit indefinitely (#174 P1-2). The permit is a handler + // local here (moved into the AdmissionGuard only below, once git is spawned), so + // the early return on timeout drops it and frees the slot; shed a bounded 503. + let acquire_deadline = std::time::Duration::from_secs(state.config.git_acquire_timeout_secs); + let disk_path = tokio::time::timeout( + acquire_deadline, + state.repo_store.acquire(&record.owner_did, &record.name), + ) + .await + .map_err(|_elapsed| { + tracing::warn!(repo = %name, "repo acquire timed out; shedding with 503"); + AppError::Overloaded("git service acquisition timed out, retry shortly".into()) + })? + .map_err(|e| AppError::Git(e.to_string()))?; let body_len = body.len(); // No path-scoped rule can withhold an individual blob, and the whole-repo @@ -1179,14 +1205,31 @@ pub async fn git_receive_pack( } tracing::debug!(repo = %name, "acquiring write lock"); - let guard = state - .repo_store - .acquire_write(&record.owner_did, &record.name) - .await - .map_err(|e| { - tracing::error!(repo = %name, err = %e, "acquire_write failed"); - AppError::Git(e.to_string()) - })?; + // Bound the write acquire under `git_acquire_timeout_secs`. acquire_write's + // advisory-lock loop already caps at ~60s, but its per-iteration + // `pg_try_advisory_lock().fetch_one(&pool)` can block indefinitely on a hung / + // exhausted Postgres pool (so the 60-count never advances) — and the write permit + // is held the whole time, draining the pool (#174 P1-2). The outer + // `tokio::time::timeout` cancels a mid-sleep/mid-`fetch_one` future, so it bounds + // both the loop and a hung iteration without any repo_store.rs change (KTD3). The + // permit is a handler local here (moved into the AdmissionGuard only after this), + // so the early return on timeout drops it and frees the slot; shed a bounded 503. + let acquire_deadline = std::time::Duration::from_secs(state.config.git_acquire_timeout_secs); + let guard = tokio::time::timeout( + acquire_deadline, + state + .repo_store + .acquire_write(&record.owner_did, &record.name), + ) + .await + .map_err(|_elapsed| { + tracing::warn!(repo = %name, "acquire_write timed out; shedding with 503"); + AppError::Overloaded("git service acquisition timed out, retry shortly".into()) + })? + .map_err(|e| { + tracing::error!(repo = %name, err = %e, "acquire_write failed"); + AppError::Git(e.to_string()) + })?; let disk_path = guard.path().to_path_buf(); tracing::debug!(repo = %name, path = %disk_path.display(), "running git receive-pack"); let body_len = body.len(); @@ -1197,8 +1240,14 @@ pub async fn git_receive_pack( // The handler keeps no copy. This is independent of the write-lock `guard.release` // below: admission tracks the git process lifetime, the write lock tracks the repo. let admission = smart_http::AdmissionGuard::new(_permit, _caller_permit); - let receive_result = - smart_http::receive_pack(&state.git_bin, &disk_path, body, git_timeout, Some(admission)).await; + let receive_result = smart_http::receive_pack( + &state.git_bin, + &disk_path, + body, + git_timeout, + Some(admission), + ) + .await; // Always release the advisory lock — even on error — to prevent stale locks // from blocking subsequent pushes. Only upload to Tigris when the push @@ -3761,7 +3810,11 @@ mod tests { .unwrap(); let sem = state.git_read_semaphore.clone(); - assert_eq!(sem.available_permits(), 1, "one read slot before the request"); + assert_eq!( + sem.available_permits(), + 1, + "one read slot before the request" + ); let router = crate::server::build_router(state); let make_req = |peer: SocketAddr| { @@ -3991,6 +4044,198 @@ mod tests { ); } + /// #174 U2 (P1-2, RED-before/GREEN-after): the storage-acquisition phase is bounded + /// by `git_acquire_timeout_secs`, so a stalled backend releases the admission permit + /// and sheds a 503 instead of pinning the pool. The permit is taken BEFORE + /// `acquire_write`, whose advisory-lock loop can spin ~60s (and whose per-iteration + /// `pg_try_advisory_lock` can block indefinitely on a hung pool), so without the + /// `tokio::time::timeout` wrapper the permit is held far past the deadline. + /// + /// Real stall (no `RepoStore` trait to fake): hold the SAME session-level advisory + /// lock `acquire_write` derives (`advisory_lock_key(owner_slug, repo_name)`, where + /// `owner_slug = owner_did.replace([':','/'], "_")`) on a second pooled connection, + /// so the handler's `pg_try_advisory_lock` returns false every iteration and the loop + /// must retry against the deadline. `git_acquire_timeout_secs = 2`; the request must + /// return 503 (Overloaded) at ~2s (NOT ~59s), and the write permit must be released + /// (`available_permits()` recovers to full once the shed returns). Covers R2. + /// + /// Load-bearing / mutation: remove the `tokio::time::timeout` wrapper on + /// `acquire_write` and the loop runs to ~59s with the permit held the whole time — + /// the `< DEADLINE_CEILING` timing assertion goes RED (observed ~59s) and the permit + /// stays pinned past the deadline. Restore to return GREEN. + #[sqlx::test] + async fn receive_pack_acquire_deadline_sheds_and_releases_permit(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + + // Reproduce acquire_write's session-level advisory-lock key exactly so the + // second-connection lock collides with the handler's pg_try_advisory_lock + // (repo_store.rs: advisory_lock_key over owner_slug then repo_name). + fn advisory_lock_key(owner_slug: &str, repo_name: &str) -> i64 { + use std::hash::{Hash, Hasher}; + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + owner_slug.hash(&mut hasher); + repo_name.hash(&mut hasher); + hasher.finish() as i64 + } + + let owner = "z6acqdead"; + let name = "acq1"; + // owner_slug as local_path() computes it from the record's owner_did. The + // mirror row stores the short owner as owner_did, so slug == owner (no ':'/'/'). + let owner_slug = owner.replace([':', '/'], "_"); + let lock_key = advisory_lock_key(&owner_slug, name); + + let mut state = crate::test_support::test_state(pool.clone()).await; + // Isolate the write pool at size 1 so available_permits() cleanly reports + // held (0) vs released (1). Per-source cap + trust permissive so only the + // write pool / acquire path can gate. + state.git_write_semaphore = Arc::new(Semaphore::new(1)); + state.git_write_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // Short acquire deadline: the fix must shed here, well before acquire_write's + // ~59s advisory-lock loop would bail on its own. + const ACQUIRE_TIMEOUT_SECS: u64 = 2; + let mut cfg = (*state.config).clone(); + cfg.git_acquire_timeout_secs = ACQUIRE_TIMEOUT_SECS; + // Keep the git-service timeout large so the deadline under test is the acquire + // one, not git execution (which is never reached on the stalled path anyway). + cfg.git_service_timeout_secs = 600; + state.config = std::sync::Arc::new(cfg); + + state + .db + .upsert_mirror_repo(owner, name, "/tmp/z6acqdead-acq1", None, false) + .await + .unwrap(); + + // Hold the advisory lock on a dedicated pooled connection (a distinct session), + // so the handler's pg_try_advisory_lock($lock_key) returns false every iteration + // and acquire_write's real loop must retry against the deadline. Released when + // this connection drops at end of test. + let mut lock_conn = pool + .acquire() + .await + .expect("second connection for the lock"); + sqlx::query("SELECT pg_advisory_lock($1)") + .bind(lock_key) + .execute(&mut *lock_conn) + .await + .expect("hold the advisory lock on the second connection"); + + let did = "did:key:z6MkAcquireDeadlineProofDidAAAAAAAAAAAAAAAA"; + let peer: SocketAddr = "203.0.113.61:5000".parse().unwrap(); + + let sem = state.git_write_semaphore.clone(); + assert_eq!( + sem.available_permits(), + 1, + "one write slot before the request" + ); + + // Drive the authenticated push in the background so we can observe the permit is + // held while acquire_write stalls, then that it is released on the shed. + let state_for_task = state.clone(); + let start = std::time::Instant::now(); + let handle = tokio::spawn(async move { + git_receive_pack( + State(state_for_task), + Path((owner.to_string(), name.to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(peer)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await + }); + + // The handler takes the write permit BEFORE acquire_write, so once it is stalled + // in the advisory-lock loop the pool reports 0 available. Wait for that to prove + // the permit is genuinely held during the stall (and the request really reached + // acquire_write, not an earlier reject). + let mut held = false; + for _ in 0..200 { + if sem.available_permits() == 0 { + held = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + assert!( + held, + "the write permit must be held while acquire_write stalls on the advisory lock" + ); + + // The bounded acquire deadline must shed with 503 (Overloaded), NOT wait out the + // ~59s advisory-lock loop. Ceiling is comfortably above the 2s deadline + task + // scheduling but far below 59s, so a RED run (no wrapper -> ~59s) fails here. + const DEADLINE_CEILING: std::time::Duration = std::time::Duration::from_secs(20); + let result = tokio::time::timeout( + DEADLINE_CEILING + std::time::Duration::from_secs(10), + handle, + ) + .await + .expect("the handler must return within the ceiling — a hang means the acquire deadline is missing (RED)") + .expect("the receive-pack task must not panic"); + let elapsed = start.elapsed(); + + assert!( + matches!(result, Err(AppError::Overloaded(_))), + "a stalled acquire_write must shed with Overloaded/503 at the acquire deadline; \ + got {result:?}" + ); + assert!( + elapsed < DEADLINE_CEILING, + "the shed must land at ~{ACQUIRE_TIMEOUT_SECS}s (the acquire deadline), not ~59s \ + (the advisory-lock loop). Observed {elapsed:?}; without the timeout wrapper this \ + is ~59s (RED)" + ); + + // Permit release on expiry: the Overloaded return drops the handler-local permit, + // so the isolated write pool must recover to full. A leaked permit here means the + // pool drains under a stalled backend (the #174 P1-2 bug). + let mut freed = false; + for _ in 0..200 { + if sem.available_permits() == 1 { + freed = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + assert!( + freed, + "on the acquire-deadline shed the write permit must be released; the pool did \ + not recover to full (permit leaked)" + ); + + // Follow-up admits once the contended lock is released: release the second-conn + // lock, then a fresh push proceeds PAST admission (it fails later on the + // nonexistent on-disk repo, which is NOT an Overloaded/503). Proves the freed + // slot is usable, not merely counted. + sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(lock_key) + .execute(&mut *lock_conn) + .await + .expect("release the advisory lock"); + let followup = git_receive_pack( + State(state.clone()), + Path((owner.to_string(), name.to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some("203.0.113.62:5000".parse().unwrap())), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await; + assert!( + !matches!(followup, Err(AppError::Overloaded(_))), + "once the lock frees, a follow-up push must admit past the (recovered) write \ + pool and acquire; got {followup:?}" + ); + } + /// #174 U5 (P1-e, RED-before/GREEN-after): the post-push encryption walk acquires a /// `git_encrypt_semaphore` permit before running, so completed pushes cannot spawn /// unbounded concurrent full-history walks. With the pool exhausted the gated walk diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index d0a4c319..68125364 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -183,6 +183,27 @@ pub struct Config { )] pub git_service_timeout_secs: u64, + /// Maximum wall-clock time the storage-acquisition phase of a served git + /// operation may run before the request is shed with a 503, in seconds. This + /// bounds `RepoStore::{acquire,acquire_fresh,acquire_write}` — the Tigris + /// HEAD/GET on a read/advert acquire and the advisory-lock retry loop (incl. a + /// per-iteration `pg_try_advisory_lock` that can block on a hung Postgres pool) + /// on a write acquire. A concurrency permit is taken BEFORE this phase, and + /// `git_service_timeout_secs` only starts once git spawns, so without this the + /// acquire phase is unbounded: a stalled backend pins the permit and drains the + /// pool until every later request 503s. On expiry the permit is released and a + /// bounded 503 + Retry-After is returned (fail-closed). Kept separate from + /// `git_service_timeout_secs` because acquisition and git execution are distinct + /// cost centers — one shared budget would let a slow acquire starve git. Must be + /// positive; set it very large to effectively disable the bound. Default: 30s. + #[arg( + long, + env = "GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS", + default_value_t = 30, + value_parser = clap::value_parser!(u64).range(1..) + )] + pub git_acquire_timeout_secs: u64, + /// Maximum connections in the PostgreSQL pool. This is a cap, not a floor /// (connections open lazily). Size against the database server's /// max_connections, remembering admin tooling opens its own pool. From f424ccbbcde49260932d776f3e8b560ec2a674c8 Mon Sep 17 00:00:00 2001 From: t Date: Wed, 15 Jul 2026 12:42:55 -0500 Subject: [PATCH 35/58] fix(node): gate the /ipfs walk with bounded concurrency admission + per-source cap + route rate limit (#174) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GET /ipfs/{cid} (get_by_cid, publicly reachable via optional_signature) ran allowed_blob_set_for_caller_bounded in spawn_blocking inside a per-repo loop with NO global/per-source admission and NO route rate limit — the surface had zero concurrency or rate control. A permissionless caller could fan out concurrent full-history git walks (each up to the walk timeout) exhausting blocking-pool threads and PIDs outside every served-git pool (P1-3). Acquire a global git_ipfs_walk permit (try_acquire_owned, shed 503) plus a per-source sub-cap (with_default_max_keys, reject-before-insert, keyed via client_key/TrustedProxy so XFF can't spoof it) ONCE after CID validation, held as handler locals across the whole request — including every spawn_blocking walk, so the slot reflects real blocking-thread occupancy (a tokio timeout cannot cancel spawn_blocking). None key -> global pool only. Cap repos walked per request so one request can't serialize N full-history walks. Attach rate_limit_by_ip + IpRateLimiter to the /ipfs route (mirrors the git/create/peer routers; the extension is required or the middleware is a silent no-op). Visibility/authz semantics unchanged. New knobs: GITLAWB_MAX_CONCURRENT_IPFS_WALKS (32), GITLAWB_IPFS_WALK_PER_SOURCE (4), GITLAWB_IPFS_MAX_REPOS_WALKED (64), GITLAWB_IPFS_RATE_LIMIT (600/hr, 0 disables) — all range-validated and enforced on-path. Handler-layer regressions (all mutation-verified RED->GREEN): shed-at-capacity 503 (delete acquire -> falls through), per-source cap sheds same source / admits another, None-key arm sheds on the global pool, map self-bounds reject-before-insert, the walk permit is held while the spawn_blocking walk runs (available_permits()==0; dropping it before the loop -> RED), repos-walked cap (remove break -> 2 walks), and the route IP rate limit actually fires 429 (drop the Extension -> RED no-op). --- crates/gitlawb-node/src/api/ipfs.rs | 691 ++++++++++++++++++++++++ crates/gitlawb-node/src/auth/mod.rs | 4 + crates/gitlawb-node/src/config.rs | 120 ++++ crates/gitlawb-node/src/main.rs | 17 + crates/gitlawb-node/src/server.rs | 11 + crates/gitlawb-node/src/state.rs | 27 + crates/gitlawb-node/src/test_support.rs | 6 + 7 files changed, 876 insertions(+) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index 284c9398..ac8e615c 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -58,6 +58,11 @@ pub async fn get_by_cid( Path(cid_str): Path, State(state): State, auth: Option>, + // Per-source keying for the walk concurrency sub-cap. Infallible extractors + // (mirror the git handlers in `repos.rs`): `PeerAddr` yields `None` under + // `oneshot` with no `ConnectInfo`, and the header map falls back per `client_key`. + crate::rate_limit::PeerAddr(peer): crate::rate_limit::PeerAddr, + req_headers: HeaderMap, ) -> Result { // 1. Decode the CID and extract the SHA-256 digest let cid = CidGeneric::<64>::from_str(&cid_str) @@ -76,6 +81,36 @@ pub async fn get_by_cid( let caller = auth.as_ref().map(|e| e.0 .0.as_str()); let caller_owned = caller.map(|c| c.to_string()); + // Bounded walk admission (#174 P1-3), taken before any DB/git work so a flood sheds + // cheaply. The per-repo `spawn_blocking` walk below is a full-history git walk with + // no served-git admission of its own; a permissionless caller could otherwise fan + // out concurrent walks past every git pool, exhausting the blocking pool + PIDs. + // Acquire the global permit (and, for a resolvable source, the per-source + // sub-permit) ONCE here and hold BOTH for the whole request — across every + // `spawn_blocking` walk in the loop below — so the slot reflects real blocking-thread + // occupancy (a tokio walk-timeout cannot free it while the blocking work still runs) + // and one request cannot open more than its share of concurrent walks. On + // unavailability shed a clean 503. The per-source key is the resolved source IP + // (`client_key`), never the DID (`/ipfs` admits any `did:key` unthrottled, so a DID + // key would be free to mint around); a `None` key (no trusted header, no peer) is + // bounded by the global pool only, never the per-source sub-cap. + let _ipfs_walk_permit = state + .git_ipfs_walk_semaphore + .clone() + .try_acquire_owned() + .map_err(|_| { + tracing::warn!("/ipfs walk concurrency cap reached; shedding request with 503"); + AppError::Overloaded("ipfs service at capacity, retry shortly".into()) + })?; + let source_key = crate::rate_limit::client_key(&req_headers, peer, state.push_limiter_trust); + let _ipfs_caller_permit = match &source_key { + Some(ip) => Some(state.git_ipfs_walk_per_caller.try_acquire(ip).ok_or_else(|| { + tracing::warn!(key = %ip, "/ipfs per-source walk cap reached; shedding request with 503"); + AppError::Overloaded("ipfs service at capacity for this source, retry shortly".into()) + })?), + None => None, + }; + // 2. Search all repos for an object with this SHA-256 let repos = state .db @@ -104,6 +139,11 @@ pub async fn get_by_cid( // deny entry (#126). let mut allowed_memo: HashMap> = HashMap::new(); + // Cap the number of candidate repos one request walks (it already short-circuits on + // serve): a CID present in — or path-gated out of — many repos must not serialize an + // unbounded number of full-history walks inside the single held admission slot. + let mut repos_walked: usize = 0; + for repo in &repos { // Repo-level read gate against THIS row's own rules (KTD2a). let rules: &[crate::db::VisibilityRule] = rules_by_repo @@ -114,6 +154,19 @@ pub async fn get_by_cid( continue; } + // Loop bound (#174 P1-3): once this request has walked its cap of candidate + // repos (each a git subprocess, up to a full-history walk), stop and fall + // through to the opaque 404 rather than serialize an unbounded number under the + // single held admission slot. + if repos_walked >= state.config.ipfs_max_repos_walked { + tracing::warn!( + cap = state.config.ipfs_max_repos_walked, + "/ipfs request hit the per-request repo-walk cap; stopping the scan" + ); + break; + } + repos_walked += 1; + // Bound the per-repo acquire under `git_acquire_timeout_secs`: this loop shares // the P1-2 stall vector (a hung Tigris HEAD/GET on one repo would otherwise // block the whole /ipfs request). On expiry keep the existing fail-closed skip — @@ -249,3 +302,641 @@ pub async fn list_pins(State(state): State) -> Result Router { + Router::new() + .route( + "/ipfs/{cid}", + axum::routing::get(crate::api::ipfs::get_by_cid), + ) + .layer(axum::middleware::from_fn(crate::auth::optional_signature)) + .with_state(state) + } + + /// A syntactically valid CIDv1(raw, sha2-256) string the handler decodes past its + /// CID/hash-code validation, so the request reaches the walk admission (not a 400). + fn valid_cid() -> String { + gitlawb_core::cid::Cid::from_git_object_bytes(b"blob 5\0hello") + .as_str() + .to_string() + } + + fn get_cid(cid: &str, peer: Option) -> Request { + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap(); + if let Some(p) = peer { + req.extensions_mut().insert(ConnectInfo(p)); + } + req + } + + /// Shed at capacity: an exhausted `git_ipfs_walk_semaphore` sheds a `/ipfs/{cid}` + /// request with 503 BEFORE any DB/git walk (the acquire is the first thing after CID + /// validation), so a lazy DB-free state suffices — exactly like the served-git shed + /// tests. MUTATION (RED): delete the `git_ipfs_walk_semaphore` acquire in + /// `get_by_cid` and the request no longer sheds here (it falls through to the DB / + /// walk and returns something other than 503). + #[tokio::test] + async fn get_by_cid_sheds_with_503_when_walk_pool_exhausted() { + let mut state = crate::test_support::test_state_lazy(); + // Global /ipfs walk pool exhausted; per-source cap permissive so only the global + // pool can shed. Route rate limit is applied as a layer in production, not here. + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(0)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let peer: SocketAddr = "203.0.113.9:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&valid_cid(), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an exhausted /ipfs walk pool must shed the request with 503" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the 503 shed must carry Retry-After" + ); + } + + /// Per-source sub-cap, the `Some(ip)` arm: with per-source = 1 and the source pinned + /// at its single slot, a request from THAT source sheds 503 (global pool has room), + /// while a request from a DIFFERENT source is NOT shed by the cap (it proceeds past + /// admission). Pinning proves the `PeerAddr`/`HeaderMap` extractors resolved the key + /// — an inert `None` key would never shed on the per-source cap. MUTATION (RED): + /// delete the `git_ipfs_walk_per_caller` acquire and the capped source no longer + /// sheds. + #[tokio::test] + async fn get_by_cid_per_source_cap_sheds_same_source_admits_other() { + let mut state = crate::test_support::test_state_lazy(); + // Global pool has room; the per-source cap is 1. + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(8)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let capped: SocketAddr = "203.0.113.20:5000".parse().unwrap(); + let other: SocketAddr = "203.0.113.21:5000".parse().unwrap(); + + // Pin the capped source at its single walk slot. + let _slot = state + .git_ipfs_walk_per_caller + .try_acquire(&capped.ip().to_string()) + .expect("first walk slot for the capped source IP"); + + let cid = valid_cid(); + // The capped source sheds on the per-source cap even with global capacity free. + let resp = ipfs_router(state.clone()) + .oneshot(get_cid(&cid, Some(capped))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a source at its per-source /ipfs walk cap must shed 503 with global capacity free" + ); + + // A DIFFERENT source is NOT shed by the per-source cap: it clears admission and + // proceeds (then errors on the lazy DB, which is not a 503). + let resp = ipfs_router(state) + .oneshot(get_cid(&cid, Some(other))) + .await + .unwrap(); + assert_ne!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a different source must not be shed by the per-source cap" + ); + } + + /// The `None`-key arm: a request with no resolvable source key (no trusted-proxy + /// header, no `ConnectInfo`) is bounded by the GLOBAL pool only, never the per-source + /// sub-cap. With the global pool exhausted it still sheds 503 (the counterpart to the + /// `Some(ip)` arm above, so neither arm is vacuous). + #[tokio::test] + async fn get_by_cid_none_key_arm_sheds_on_global_pool() { + let mut state = crate::test_support::test_state_lazy(); + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(0)); + // Per-source cap permissive so only the global pool can shed. + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + // No ConnectInfo + no trusted header -> client_key resolves None. + let resp = ipfs_router(state) + .oneshot(get_cid(&valid_cid(), None)) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a None-key request must still shed 503 on the exhausted GLOBAL /ipfs walk pool" + ); + } + + /// Map self-bound (INV-15): the `/ipfs` per-source map is a `PerCallerConcurrency` + /// built via `with_default_max_keys`, so a distinct-source-key flood cannot grow it + /// past the cap and a rejected key never allocates (reject-before-insert). Mirrors + /// `per_caller_concurrency_map_is_self_bounding_and_reject_before_insert` for the + /// pool U3 adds. + #[tokio::test] + async fn ipfs_walk_per_caller_map_is_self_bounding_and_reject_before_insert() { + let lim = crate::rate_limit::PerCallerConcurrency::new(4, 3); + // Acquire+drop a flood of distinct keys — the map self-empties (a key is removed + // the instant its in-flight count hits zero). + for i in 0..50 { + let _p = lim.try_acquire(&format!("src{i}")); + } + assert_eq!( + lim.tracked_keys(), + 0, + "an acquire+drop flood of distinct sources leaves the /ipfs map empty" + ); + // Reject-before-insert: hold max_keys distinct sources, then a new one sheds + // without growing the map. + let held: Vec<_> = (0..3) + .map(|i| lim.try_acquire(&format!("h{i}")).unwrap()) + .collect(); + assert_eq!( + lim.tracked_keys(), + 3, + "three distinct sources held concurrently" + ); + assert!( + lim.try_acquire("h3").is_none(), + "a new source key at max_keys is rejected" + ); + assert_eq!( + lim.tracked_keys(), + 3, + "the rejected key did not allocate an entry (reject-before-insert)" + ); + drop(held); + } + + /// Retain-through-blocking (R3, the load-bearing async property): the walk + /// admission is held until the `spawn_blocking` walk actually RETURNS, not when a + /// tokio timeout fires. With the global pool at size 1, drive a request until its + /// walk (a fake git that hangs on `rev-list`) is in flight; the slot must stay held + /// (`available_permits() == 0`) and a replacement from a DIFFERENT source must shed + /// 503 for as long as the blocking walk runs — even though the request future is + /// only `.await`ing the blocking join. When the blocking walk ends the permit frees + /// and a replacement is admitted. The permit lives INSIDE the handler across the + /// blocking `.await`; move it out (drop before the walk) and the replacement would + /// be admitted while the walk still burns a blocking thread (the bug this guards). + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_walk_permit_held_through_blocking_walk(pool: sqlx::PgPool) { + use std::process::Command; + + let tmp = tempfile::TempDir::new().unwrap(); + let revlist_pid = tmp.path().join("revlist.pid"); + // Fake git for the /ipfs WALK only (object_type/read_object_content use the real + // `git`, so the object must genuinely exist below). Empty refs (so + // assert_all_refs_are_commits returns Ok without the peel), `rev-parse` resolves, + // and `rev-list` records its pid then sleeps ~6s so the walk BLOCKS + // deterministically. The sleep bounds the walk so a broken fix cannot wedge the + // suite. + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + for-each-ref) : ;;\n\ + rev-parse) echo deadbeef ;;\n\ + rev-list) echo $$ > \"{}\"; sleep 6 ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + revlist_pid.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + // Isolate the global walk pool at size 1; per-source cap permissive so only the + // held global permit can shed the replacement. + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(1)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + state.git_bin = git_path.to_str().unwrap().to_string(); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let owner = "z6ipfs1"; + let name = "ip1"; + state + .db + .upsert_mirror_repo(owner, name, "/unused", None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + // The exact bare path the handler's `acquire` resolves. Build a REAL SHA-256 bare + // repo there with a committed blob under `src/`, so real `git cat-file -t ` classifies it as a blob (the CID digest IS the sha256 object id in + // object-format=sha256) and the handler reaches the path-scoped walk branch. + let bare = state + .repo_store + .acquire(&rec.owner_did, &rec.name) + .await + .unwrap(); + let _ = std::fs::remove_dir_all(&bare); + std::fs::create_dir_all(&bare).unwrap(); + let run = |args: &[&str], cwd: &std::path::Path| { + let out = Command::new("git") + .args(args) + .current_dir(cwd) + .output() + .expect("git runs"); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + }; + let work = tmp.path().join("work"); + std::fs::create_dir_all(work.join("src")).unwrap(); + std::fs::write(work.join("src/secret.txt"), b"ipfs walk retain proof\n").unwrap(); + run( + &["init", "-q", "--object-format=sha256", "-b", "main"], + &work, + ); + run(&["config", "user.email", "t@t"], &work); + run(&["config", "user.name", "t"], &work); + run(&["add", "src/secret.txt"], &work); + run(&["commit", "-q", "-m", "seed"], &work); + run( + &[ + "clone", + "--bare", + "-q", + work.to_str().unwrap(), + bare.to_str().unwrap(), + ], + tmp.path(), + ); + // The blob's SHA-256 object id (= the CID's digest); build the CID from it. + let oid = { + let out = Command::new("git") + .args(["rev-parse", "HEAD:src/secret.txt"]) + .current_dir(&work) + .output() + .expect("git rev-parse runs"); + assert!(out.status.success(), "rev-parse failed"); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + let oid_bytes = gitlawb_core::cid::sha256_hex_to_bytes(&oid).unwrap(); + let cid = gitlawb_core::cid::Cid::from_sha256_bytes(&oid_bytes) + .as_str() + .to_string(); + // Precondition: real git classifies the object as a blob (so the handler reaches + // the walk branch, not an early `continue`). + assert_eq!( + crate::git::store::object_type(&bare, &oid) + .unwrap() + .as_deref(), + Some("blob"), + "the seeded sha256 blob must exist so the handler reaches the walk" + ); + // A path-scoped rule so has_path_scoped_rule() is true (the walk branch) without + // denying the "/" gate on the public repo. + state + .db + .set_visibility_rule( + &rec.id, + "src/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkU3IpfsReaderAAAAAAAAAAAAAAAAAAAAAAAA".to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + + let sem = state.git_ipfs_walk_semaphore.clone(); + assert_eq!( + sem.available_permits(), + 1, + "one walk slot before the request" + ); + + let router = ipfs_router(state); + let make_req = |peer: SocketAddr| { + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + req + }; + + let peer: SocketAddr = "203.0.113.81:5000".parse().unwrap(); + let mut fut = Box::pin(router.clone().oneshot(make_req(peer))); + // Drive until the fake git's rev-list records its pid — the walk is now in the + // blocking pool and the request future is `.await`ing its join, holding the walk + // permit. Stop polling the instant the future completes (re-polling a completed + // oneshot panics). + let mut walk_pid: Option = None; + let mut early = None; + for _ in 0..500 { + let done = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + if let Some(p) = std::fs::read_to_string(&revlist_pid) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + walk_pid = Some(p); + break; + } + if let Ok(resp) = done { + early = Some(resp.map(|r| r.status())); + break; + } + } + let pid = walk_pid + .unwrap_or_else(|| panic!("the fake git rev-list must have spawned; early: {early:?}")); + // Reap the sleeping child on drop so a RED run leaks no orphan. + struct ReapOnDrop(i32); + impl Drop for ReapOnDrop { + fn drop(&mut self) { + unsafe { + libc::kill(self.0, libc::SIGKILL); + } + } + } + let _cleanup = ReapOnDrop(pid); + + // Load-bearing: while the blocking walk runs, the slot is HELD and a replacement + // from a DIFFERENT source sheds 503 — proving the permit is retained across the + // spawn_blocking join, not freed by a tokio timeout. + assert_eq!( + sem.available_permits(), + 0, + "the walk slot must be held while the spawn_blocking walk runs" + ); + let peer2: SocketAddr = "203.0.113.82:5000".parse().unwrap(); + let resp = router.clone().oneshot(make_req(peer2)).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a replacement must shed 503 while the prior request's blocking walk still runs" + ); + + // Drop the in-flight request; the detached blocking walk keeps running (a + // spawn_blocking cannot be cancelled), but on the fix the permit is a handler + // local, so dropping the future releases it once the blocking join is abandoned. + // Either way, kill the sleeping child so the slot frees promptly and poll for + // recovery — the point already proven above is that the slot stayed held for the + // duration of the blocking work. + drop(fut); + unsafe { + libc::kill(pid, libc::SIGKILL); + } + let mut freed = false; + for _ in 0..400 { + if sem.available_permits() == 1 { + freed = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + assert!( + freed, + "once the blocking walk ends the walk permit must free the global slot" + ); + } + + /// Loop bound (cap N): one `/ipfs/{cid}` request against a CID present in many repos + /// must not serialize an unbounded number of full-history walks. With + /// `ipfs_max_repos_walked = 1` and TWO public, path-scoped repos both carrying the + /// blob at the requested CID, the handler walks only the FIRST candidate then stops + /// (the second is cut by the cap), so the fake git's `rev-list` (one per walk) runs + /// exactly once. MUTATION (RED): remove the `repos_walked >= cap` break and both + /// repos are walked (count 2). + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_caps_repos_walked_per_request(pool: sqlx::PgPool) { + use std::process::Command; + + let tmp = tempfile::TempDir::new().unwrap(); + let walk_log = tmp.path().join("walks.log"); + // Fake git for the WALK: empty refs, `rev-parse` resolves, and each `rev-list` + // appends one line to a log (so the number of walks == the line count) and exits + // with EMPTY output (the allowed-set is empty, so every repo path-gates to a + // `continue` and the request 404s after walking). object_type uses the REAL git, + // so the seeded blob below must genuinely exist. + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + for-each-ref) : ;;\n\ + rev-parse) echo deadbeef ;;\n\ + rev-list) echo walk >> \"{}\" ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + walk_log.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.git_bin = git_path.to_str().unwrap().to_string(); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // The bound under test: walk at most one candidate repo per request. + let mut cfg = (*state.config).clone(); + cfg.ipfs_max_repos_walked = 1; + state.config = Arc::new(cfg); + + // Seed TWO public repos, each with the SAME blob (same content -> same sha256 OID + // -> same CID) under a path-scoped rule, so both are walk candidates for one CID. + let run = |args: &[&str], cwd: &std::path::Path| { + let out = Command::new("git") + .args(args) + .current_dir(cwd) + .output() + .expect("git runs"); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + }; + let mut oid = String::new(); + for (i, name) in ["ipa", "ipb"].iter().enumerate() { + let owner = "z6ipfsN"; + state + .db + .upsert_mirror_repo(owner, name, &format!("/unused-{name}"), None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + let bare = state + .repo_store + .acquire(&rec.owner_did, &rec.name) + .await + .unwrap(); + let _ = std::fs::remove_dir_all(&bare); + std::fs::create_dir_all(&bare).unwrap(); + let work = tmp.path().join(format!("work{i}")); + std::fs::create_dir_all(work.join("src")).unwrap(); + // Identical content in both repos -> identical sha256 blob OID -> one CID. + std::fs::write(work.join("src/secret.txt"), b"loop bound proof\n").unwrap(); + run( + &["init", "-q", "--object-format=sha256", "-b", "main"], + &work, + ); + run(&["config", "user.email", "t@t"], &work); + run(&["config", "user.name", "t"], &work); + run(&["add", "src/secret.txt"], &work); + run(&["commit", "-q", "-m", "seed"], &work); + run( + &[ + "clone", + "--bare", + "-q", + work.to_str().unwrap(), + bare.to_str().unwrap(), + ], + tmp.path(), + ); + if oid.is_empty() { + let out = Command::new("git") + .args(["rev-parse", "HEAD:src/secret.txt"]) + .current_dir(&work) + .output() + .expect("git rev-parse runs"); + oid = String::from_utf8_lossy(&out.stdout).trim().to_string(); + } + state + .db + .set_visibility_rule( + &rec.id, + "src/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkU3IpfsReaderBBBBBBBBBBBBBBBBBBBBBBBB".to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + } + let oid_bytes = gitlawb_core::cid::sha256_hex_to_bytes(&oid).unwrap(); + let cid = gitlawb_core::cid::Cid::from_sha256_bytes(&oid_bytes) + .as_str() + .to_string(); + + let peer: SocketAddr = "203.0.113.90:5000".parse().unwrap(); + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + let resp = ipfs_router(state).oneshot(req).await.unwrap(); + // The empty allowed-set path-gates both repos to a `continue`, so a 404; the + // point is HOW MANY walks ran to get there. + assert_eq!(resp.status(), StatusCode::NOT_FOUND); + + let walks = std::fs::read_to_string(&walk_log) + .map(|s| s.lines().count()) + .unwrap_or(0); + assert_eq!( + walks, 1, + "with the per-request repo-walk cap at 1, only the first candidate repo is \ + walked (the second is cut by the cap), so exactly one walk runs; got {walks}" + ); + } + + /// Route rate limit is WIRED (not a silent no-op): the production `build_router` + /// attaches an `IpRateLimiter` extension to the `/ipfs/{cid}` route, so a per-IP + /// flood is braked with 429. A bare `rate_limit_by_ip` layer with no extension does + /// nothing, so this proves the extension is attached. Drive it through the real + /// router with a tight limiter (1/hr): the second request from the same IP is 429. + /// MUTATION (RED): drop the `axum::Extension(ipfs_limiter)` layer in `server.rs` and + /// the second request is no longer braked (it reaches the handler, 404, not 429). + #[sqlx::test] + async fn ipfs_route_ip_rate_limit_is_attached(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool).await; + // Tight per-IP /ipfs bucket so the second request from one IP trips 429. + state.ipfs_rate_limiter = + crate::rate_limit::RateLimiter::new(1, std::time::Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let router = crate::server::build_router(state); + let cid = valid_cid(); + let make = |peer: SocketAddr| { + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + req + }; + let peer: SocketAddr = "203.0.113.99:5000".parse().unwrap(); + + // First request from this IP passes the brake and reaches the handler (404 — no + // such object anywhere), debiting the single-slot bucket. + let resp = router.clone().oneshot(make(peer)).await.unwrap(); + assert_ne!( + resp.status(), + StatusCode::TOO_MANY_REQUESTS, + "the first /ipfs request from an IP must pass the rate brake" + ); + // Second request from the SAME IP is braked with 429 — proving the limiter + // extension is attached (a bare no-op layer would let it through to 404). + let resp = router.clone().oneshot(make(peer)).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::TOO_MANY_REQUESTS, + "an exhausted per-IP /ipfs bucket must brake with 429 — the IpRateLimiter \ + extension must be attached to the route" + ); + // A DIFFERENT IP still has its own budget (independent bucket). + let other: SocketAddr = "203.0.113.100:5000".parse().unwrap(); + let resp = router.oneshot(make(other)).await.unwrap(); + assert_ne!( + resp.status(), + StatusCode::TOO_MANY_REQUESTS, + "a different IP must not be braked by another IP's exhausted bucket" + ); + } +} diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index f18b4963..eed49253 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -528,6 +528,10 @@ mod tests { git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), git_write_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), + git_ipfs_walk_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_ipfs_walk_per_caller: + crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), + ipfs_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), git_bin: "git".to_string(), } } diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index 68125364..b3667024 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -335,6 +335,66 @@ pub struct Config { value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) )] pub max_concurrent_reads_per_caller: usize, + + /// Maximum number of concurrent `GET /ipfs/{cid}` requests that may run their + /// visibility walk at once. The publicly-reachable `/ipfs/{cid}` route runs + /// `allowed_blob_set_for_caller_bounded` in `spawn_blocking` — a full-history + /// git walk (up to `git_service_timeout_secs`) — for each candidate repo. It + /// draws from THIS pool, not any served-git pool: a distinct public cost center + /// on a distinct surface, so sharing a git pool would let anonymous /ipfs + /// traffic shed authenticated git ops (the auth-boundary trap). A permit is + /// held for the whole request (across the repo loop) so it reflects real + /// blocking-thread occupancy, not merely the tokio wait. Beyond this the request + /// sheds a clean 503 + Retry-After. Must be between 1 and 1_048_576; the ceiling + /// keeps the value under tokio's `Semaphore` permit limit so an oversized value + /// is a clean CLI error rather than a boot-time panic. Default: 32. + #[arg( + long, + env = "GITLAWB_MAX_CONCURRENT_IPFS_WALKS", + default_value_t = 32, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub max_concurrent_ipfs_walks: usize, + + /// Maximum concurrent `/ipfs/{cid}` walk requests a single source may hold at + /// once, so one source cannot monopolize `max_concurrent_ipfs_walks` (#174). + /// Callers are keyed on the RESOLVED SOURCE IP (`client_key`/`GITLAWB_TRUSTED_PROXY`), + /// never the DID — `/ipfs` accepts any `did:key` via `optional_signature` with no + /// admission step, so keying on the DID would let one host mint disposable DIDs to + /// multiply its budget. A request with no resolvable key (no trusted header, no + /// peer) is bounded by the global pool only, never this sub-cap. Over-cap sheds a + /// clean 503 + Retry-After. Must be between 1 and 1_048_576. Default: 4. + #[arg( + long, + env = "GITLAWB_IPFS_WALK_PER_SOURCE", + default_value_t = 4, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub ipfs_walk_per_source: usize, + + /// Upper bound on the number of candidate repos a single `/ipfs/{cid}` request + /// will walk before giving up (returning the opaque 404). The handler already + /// short-circuits the moment it serves the object, but a CID that is present in + /// (or path-gated out of) many repos could otherwise serialize one full-history + /// walk per repo inside a single held admission slot. Capping the count bounds + /// the worst-case work one request can pin its slot with. Must be between 1 and + /// 1_048_576. Default: 64. + #[arg( + long, + env = "GITLAWB_IPFS_MAX_REPOS_WALKED", + default_value_t = 64, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub ipfs_max_repos_walked: usize, + + /// Per-client-IP rate limit for `GET /ipfs/{cid}`, in requests per hour. The + /// route is publicly reachable (`optional_signature`) and each request can drive + /// a full-history git walk, so it carries a per-IP flood brake in addition to the + /// concurrency cap above (a rate limit bounds request *rate*, the semaphore + /// bounds concurrent slow holds — different axes). Keyed on the resolved client + /// IP via `GITLAWB_TRUSTED_PROXY`. `0` disables. Default: 600. + #[arg(long, env = "GITLAWB_IPFS_RATE_LIMIT", default_value_t = 600)] + pub ipfs_rate_limit: usize, } impl Config { @@ -428,6 +488,66 @@ mod tests { ); } + #[test] + fn max_concurrent_ipfs_walks_defaults_and_rejects_out_of_range() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).max_concurrent_ipfs_walks, + 32 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--max-concurrent-ipfs-walks", "4"]) + .max_concurrent_ipfs_walks, + 4 + ); + // 0 permits would shed every /ipfs walk with a 503; clap must reject it. + assert!( + Config::try_parse_from(["gitlawb-node", "--max-concurrent-ipfs-walks", "0"]).is_err() + ); + // Above the ceiling would panic tokio's Semaphore::new at boot; clap rejects it. + assert!( + Config::try_parse_from(["gitlawb-node", "--max-concurrent-ipfs-walks", "1048577"]) + .is_err() + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--max-concurrent-ipfs-walks", "1048576"]) + .max_concurrent_ipfs_walks, + 1_048_576 + ); + } + + #[test] + fn ipfs_walk_per_source_defaults_and_rejects_out_of_range() { + assert_eq!(Config::parse_from(["gitlawb-node"]).ipfs_walk_per_source, 4); + assert_eq!( + Config::parse_from(["gitlawb-node", "--ipfs-walk-per-source", "2"]) + .ipfs_walk_per_source, + 2 + ); + // 0 would shed every /ipfs walk from a keyed source; clap must reject it. + assert!(Config::try_parse_from(["gitlawb-node", "--ipfs-walk-per-source", "0"]).is_err()); + assert!( + Config::try_parse_from(["gitlawb-node", "--ipfs-walk-per-source", "1048577"]).is_err() + ); + } + + #[test] + fn ipfs_max_repos_walked_defaults_and_rejects_out_of_range() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).ipfs_max_repos_walked, + 64 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--ipfs-max-repos-walked", "8"]) + .ipfs_max_repos_walked, + 8 + ); + // 0 would walk no repos (serve nothing); clap must reject it. + assert!(Config::try_parse_from(["gitlawb-node", "--ipfs-max-repos-walked", "0"]).is_err()); + assert!( + Config::try_parse_from(["gitlawb-node", "--ipfs-max-repos-walked", "1048577"]).is_err() + ); + } + #[test] fn max_concurrent_reads_per_caller_defaults_and_rejects_out_of_range() { assert_eq!( diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 380ab5f4..8727aae1 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -412,8 +412,25 @@ async fn main() -> Result<()> { git_write_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( (config.max_concurrent_git_pushes / 8).max(1), ), + // Bounds concurrent /ipfs visibility walks — a distinct public cost center, so + // its own pool + per-source sub-cap + per-IP rate limiter, never a git pool + // (#174 P1-3). The per-source map is bounded (reject-before-insert, INV-15). + git_ipfs_walk_semaphore: Arc::new(tokio::sync::Semaphore::new( + config.max_concurrent_ipfs_walks, + )), + git_ipfs_walk_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( + config.ipfs_walk_per_source, + ), + ipfs_rate_limiter: rate_limit::RateLimiter::new_bounded( + config.ipfs_rate_limit, + std::time::Duration::from_secs(3600), + 200_000, + ), git_bin: "git".to_string(), }; + if config.ipfs_rate_limit == 0 { + tracing::warn!("GITLAWB_IPFS_RATE_LIMIT=0 — per-IP /ipfs rate limiting disabled"); + } // Periodic peer-count poll for the metrics gauge. If p2p is disabled // we still set the gauge to 0 so dashboards don't show "no data". diff --git a/crates/gitlawb-node/src/server.rs b/crates/gitlawb-node/src/server.rs index f4c0d3e3..de61fcbe 100644 --- a/crates/gitlawb-node/src/server.rs +++ b/crates/gitlawb-node/src/server.rs @@ -214,9 +214,20 @@ pub fn build_router(state: AppState) -> Router { // identity and can apply per-repo visibility (#110); anonymous callers stay // anonymous and still read genuinely public content. `/api/v1/ipfs/pins` // stays unsigned — gating the pin index is tracked separately (#121). + // `/ipfs/{cid}` also carries a per-IP flood brake: it is anon-reachable and each + // request can drive a full-history git walk, so the per-IP rate limiter is the + // outermost layer (rejects a flood before the walk-admission work), mirroring the + // push/create routers. The extension MUST be attached or rate_limit_by_ip is a + // silent no-op. `/api/v1/ipfs/pins` (no walk) is merged in unbraked, as before. + let ipfs_limiter = rate_limit::IpRateLimiter { + limiter: state.ipfs_rate_limiter.clone(), + trust: state.push_limiter_trust, + }; let ipfs_routes = Router::new() .route("/ipfs/{cid}", get(ipfs::get_by_cid)) .layer(middleware::from_fn(auth::optional_signature)) + .layer(middleware::from_fn(rate_limit::rate_limit_by_ip)) + .layer(axum::Extension(ipfs_limiter)) .merge(Router::new().route("/api/v1/ipfs/pins", get(ipfs::list_pins))); // ── Arweave permanent anchors ────────────────────────────────────────── diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 3ff86923..3b8ba250 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -141,6 +141,33 @@ pub struct AppState { /// on the resolved source IP (never the DID — a DID farm defeats a DID key). Sized /// like `git_push_advert_per_caller`, a fraction of `max_concurrent_git_pushes`. pub git_write_per_caller: crate::rate_limit::PerCallerConcurrency, + /// Bounds concurrent `GET /ipfs/{cid}` visibility-walk requests. The public + /// `/ipfs/{cid}` route runs `allowed_blob_set_for_caller_bounded` in + /// `spawn_blocking` (a full-history git walk) with NO served-git admission of its + /// own; without this a permissionless caller fans out concurrent walks past every + /// git pool, exhausting the blocking pool + PIDs (#174 P1-3). A request acquires a + /// permit before the repo loop and holds it for the whole request (across every + /// `spawn_blocking` walk), so the slot reflects real thread occupancy — a tokio + /// walk-timeout cannot free it while the blocking work still runs. A pool of its + /// own (`max_concurrent_ipfs_walks`), NOT a git pool: distinct cost center + public + /// surface, so anonymous /ipfs traffic can never shed an authenticated git op. + pub git_ipfs_walk_semaphore: Arc, + /// Per-source concurrency sub-cap on the `/ipfs/{cid}` walk pool: each source + /// (keyed on the resolved source IP, never the DID — `/ipfs` admits any `did:key` + /// unthrottled, so a DID key would be free to mint around) may hold at most + /// `ipfs_walk_per_source` in-flight walk slots, so one source cannot monopolize + /// `git_ipfs_walk_semaphore` (#174 P1-3). A request with no resolvable key is + /// bounded by the global pool only, never this sub-cap. The key map is bounded + /// (`with_default_max_keys`, reject-before-insert) so a source-key farm cannot grow + /// it (INV-15). + pub git_ipfs_walk_per_caller: crate::rate_limit::PerCallerConcurrency, + /// Per-client-IP rate limiter for `GET /ipfs/{cid}`. The route is publicly + /// reachable and each request can drive a full-history git walk, so it carries a + /// per-IP flood brake in addition to the concurrency cap above — a rate limit + /// bounds request *rate*, the semaphore bounds concurrent slow holds (different + /// axes). Keyed on the resolved client IP via `push_limiter_trust`. Layered on the + /// `/ipfs` route via `rate_limit_by_ip`. + pub ipfs_rate_limiter: RateLimiter, /// The `git` executable the served-git withheld-blob walk spawns. Production is /// `"git"` (resolved via PATH); injectable so a fake `git` can drive the walk's /// process-group teardown in handler tests without mutating the process-global diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index a27be946..cfa676af 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -92,6 +92,12 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { 8, ), git_write_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), + // Generous — a test that drives the /ipfs walk shed overrides these directly. + git_ipfs_walk_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_ipfs_walk_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys( + 16, + ), + ipfs_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), git_bin: "git".to_string(), } } From 394768711ec6afa834d010734c34cafd1f6063ad Mon Sep 17 00:00:00 2001 From: t Date: Wed, 15 Jul 2026 13:10:04 -0500 Subject: [PATCH 36/58] fix(node): bound the post-push encryption task set by per-repo coalescing without dropping recovery copies (#174) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After a successful path-scoped push the handler spawns a DETACHED task that runs pin_new_objects then withheld_recipients_gated, which acquire_owned().await's on git_encrypt_semaphore — it DEFERS (blocks) when the pool is full. The semaphore caps active walks but nothing capped how many detached tasks were spawned and PARKED on that await: N rapid path-scoped pushes spawned N tasks, each holding cloned object lists/rules/paths/keys — an unbounded parked-waiter set (P2-2). 2a54c15 deliberately chose defer-not-shed because dropping the walk loses the withheld-blob recovery copy and there is no reconciliation sweep to rebuild it. So bound the OUTSTANDING-TASK set by per-repo coalescing rather than shedding: EncryptInflight (a bounded Arc>>) + try_begin — before the spawn, if a task for the repo is already in-flight skip the duplicate (the pending walk covers the newer objects); otherwise spawn and move an RAII EncryptInflightGuard into the task that removes the key on drop (completion, error, or panic-unwind). This bounds the set to <=1 pending task per repo and never drops work; withheld_recipients_gated's defer is unchanged. The second detached post-push spawn (Pinata replication, also runs pin_new_objects) is scoped out with rationale: it parks on no semaphore (no unbounded waiter set) and does per-push per-ref work (branch->CID, gossip, subscription, Arweave, peer-notify) keyed to this push's ref_updates — coalescing it would DROP a later push's announcements, a correctness regression. Regressions (mutation-verified RED->GREEN): the outstanding set is bounded to 1 per repo under saturation (never-coalesce -> 32 tasks RED); a coalesced repo's key is released when its task ends so it is reprocessed, never permanently skipped (guard-drop no-op -> repo locked out, recovery copy lost forever, RED — the exact durability regression the fix prevents); distinct repos each admit; cold-set first push admits; plus an inv22_gates structural tripwire that fails if the handler gate is removed. --- crates/gitlawb-node/src/api/repos.rs | 383 +++++++++++++++++------ crates/gitlawb-node/src/auth/mod.rs | 1 + crates/gitlawb-node/src/main.rs | 5 + crates/gitlawb-node/src/state.rs | 96 ++++++ crates/gitlawb-node/src/test_support.rs | 1 + crates/gitlawb-node/tests/inv22_gates.rs | 10 + 6 files changed, 407 insertions(+), 89 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 5b11404d..6b52273f 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1419,109 +1419,145 @@ pub async fn git_receive_pack( // Pin new git objects to the local IPFS node (no-op if ipfs_api is empty). // Skipped entirely when the public cannot read the repo (withheld == None). + // + // Coalesce per repo (#174 P2-2): this task parks on `git_encrypt_semaphore` + // (which DEFERS when the pool is full rather than dropping the recovery copy). To + // bound the OUTSTANDING parked-task set, only spawn if no encryption task for this + // repo is already in flight; otherwise skip — the pending/next walk over this + // repo's history already covers the newer push's objects, so the recovery copy is + // delayed, never lost. The guard removes the repo key when the task ends (success, + // error, or panic), so a later push is re-admitted (no permanent skip). if withheld.is_some() { - let object_list_ipfs = object_list.clone(); - let ipfs_api = state.config.ipfs_api.clone(); - let repo_path_clone = disk_path.clone(); - let db_clone = state.db.clone(); - let rules_for_enc = rules_opt.clone(); - let repo_id = record.id.clone(); - let owner_did = record.owner_did.clone(); - let is_public = record.is_public; - let irys_url = state.config.irys_url.clone(); - let http_client = std::sync::Arc::clone(&state.http_client); - let node_did_str = state.node_did.to_string(); - let node_seed = state.node_keypair.to_seed(); - let repo_name = record.name.clone(); - let enc_git_bin = state.git_bin.clone(); - let enc_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); - let encrypt_sem = state.git_encrypt_semaphore.clone(); - tokio::spawn(async move { - let pinned = crate::ipfs_pin::pin_new_objects( - &ipfs_api, - &repo_path_clone, - object_list_ipfs, - &db_clone, - ) - .await; - if !pinned.is_empty() { - tracing::info!(count = pinned.len(), "pinned git objects to IPFS"); - for (sha, cid) in &pinned { - tracing::info!(sha = %sha, %cid, "pinned"); - } + match state.encrypt_inflight.try_begin(&record.id) { + None => { + tracing::debug!( + repo = %record.id, + "post-push encryption task already in flight for this repo; coalescing \ + (the pending recovery-copy walk covers this push's objects)" + ); } - - // Option B1: encrypt-then-pin the withheld blobs so authorized - // readers can recover them when the origin cannot serve them. - // No path-scoped rule can withhold a blob, so withheld_blob_recipients - // would return an empty map after a full per-ref walk; skip it. Mirrors - // the has_path_scoped_rule gate on the other two withheld-walk sites. - if let Some(rules) = rules_for_enc.filter(|r| visibility_pack::has_path_scoped_rule(r)) - { - // Bound the number of concurrent post-push encryption walks (#174 P1-e): - // acquire an admission permit before the full-history walk, deferring - // when the pool is full rather than shedding the recovery pin. - let recip = withheld_recipients_gated( - encrypt_sem.clone(), - repo_path_clone.clone(), - enc_git_bin.clone(), - enc_timeout, - rules, - is_public, - owner_did.clone(), - ) - .await; - if let Ok(Ok(recipients)) = recip { - let delta = crate::encrypted_pin::encrypt_and_pin( + Some(inflight_guard) => { + let object_list_ipfs = object_list.clone(); + let ipfs_api = state.config.ipfs_api.clone(); + let repo_path_clone = disk_path.clone(); + let db_clone = state.db.clone(); + let rules_for_enc = rules_opt.clone(); + let repo_id = record.id.clone(); + let owner_did = record.owner_did.clone(); + let is_public = record.is_public; + let irys_url = state.config.irys_url.clone(); + let http_client = std::sync::Arc::clone(&state.http_client); + let node_did_str = state.node_did.to_string(); + let node_seed = state.node_keypair.to_seed(); + let repo_name = record.name.clone(); + let enc_git_bin = state.git_bin.clone(); + let enc_timeout = + std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let encrypt_sem = state.git_encrypt_semaphore.clone(); + tokio::spawn(async move { + // Held for the whole task; drop (on completion/error/panic) releases the + // repo's coalescing key so the next push for this repo can spawn again. + let _inflight_guard = inflight_guard; + let pinned = crate::ipfs_pin::pin_new_objects( &ipfs_api, &repo_path_clone, + object_list_ipfs, &db_clone, - &repo_id, - &node_seed, - &recipients, ) .await; + if !pinned.is_empty() { + tracing::info!(count = pinned.len(), "pinned git objects to IPFS"); + for (sha, cid) in &pinned { + tracing::info!(sha = %sha, %cid, "pinned"); + } + } - // Option B3: anchor a per-push manifest of the blobs sealed - // this push to Arweave, so the oid->cid index survives total - // node loss. Best-effort; never fails the push. - if !delta.is_empty() && !irys_url.is_empty() { - let owner_short = crate::db::normalize_owner_key(&owner_did); - let repo_slug = format!("{owner_short}/{repo_name}"); - let ts = chrono::Utc::now().to_rfc3339(); - let manifest = crate::arweave::EncryptedManifest { - repo: &repo_slug, - owner_did: &owner_did, - node_did: &node_did_str, - timestamp: &ts, - blobs: &delta, - }; - match crate::arweave::anchor_encrypted_manifest( - &http_client, - &irys_url, - &manifest, + // Option B1: encrypt-then-pin the withheld blobs so authorized + // readers can recover them when the origin cannot serve them. + // No path-scoped rule can withhold a blob, so withheld_blob_recipients + // would return an empty map after a full per-ref walk; skip it. Mirrors + // the has_path_scoped_rule gate on the other two withheld-walk sites. + if let Some(rules) = + rules_for_enc.filter(|r| visibility_pack::has_path_scoped_rule(r)) + { + // Bound the number of concurrent post-push encryption walks (#174 P1-e): + // acquire an admission permit before the full-history walk, deferring + // when the pool is full rather than shedding the recovery pin. + let recip = withheld_recipients_gated( + encrypt_sem.clone(), + repo_path_clone.clone(), + enc_git_bin.clone(), + enc_timeout, + rules, + is_public, + owner_did.clone(), ) - .await - { - Ok(tx) if !tx.is_empty() => tracing::info!( - repo = %repo_slug, - tx_id = %tx, - "anchored encrypted manifest to Arweave" - ), - Ok(_) => {} - Err(e) => tracing::warn!( - repo = %repo_slug, - err = %e, - "encrypted manifest anchor failed" - ), + .await; + if let Ok(Ok(recipients)) = recip { + let delta = crate::encrypted_pin::encrypt_and_pin( + &ipfs_api, + &repo_path_clone, + &db_clone, + &repo_id, + &node_seed, + &recipients, + ) + .await; + + // Option B3: anchor a per-push manifest of the blobs sealed + // this push to Arweave, so the oid->cid index survives total + // node loss. Best-effort; never fails the push. + if !delta.is_empty() && !irys_url.is_empty() { + let owner_short = crate::db::normalize_owner_key(&owner_did); + let repo_slug = format!("{owner_short}/{repo_name}"); + let ts = chrono::Utc::now().to_rfc3339(); + let manifest = crate::arweave::EncryptedManifest { + repo: &repo_slug, + owner_did: &owner_did, + node_did: &node_did_str, + timestamp: &ts, + blobs: &delta, + }; + match crate::arweave::anchor_encrypted_manifest( + &http_client, + &irys_url, + &manifest, + ) + .await + { + Ok(tx) if !tx.is_empty() => tracing::info!( + repo = %repo_slug, + tx_id = %tx, + "anchored encrypted manifest to Arweave" + ), + Ok(_) => {} + Err(e) => tracing::warn!( + repo = %repo_slug, + err = %e, + "encrypted manifest anchor failed" + ), + } + } } } - } + }); } - }); + } } - // Pin new git objects to Pinata, then record branch→CID and gossip + // Pin new git objects to Pinata, then record branch→CID and gossip. + // + // #174 P2-2 scope note: this SECOND detached spawn is deliberately NOT brought + // under the per-repo encryption coalescing above. Two reasons: (1) it does not + // park on `git_encrypt_semaphore` (or any semaphore) — the Pinata `pin_new_objects` + // is a bounded reqwest round-trip, so it does not form the unbounded PARKED-waiter + // set that is the P2-2 residual; it runs to completion under the HTTP client's + // network timeouts. (2) Unlike the idempotent recovery-copy walk, this task does + // PER-PUSH, PER-REF work — branch→CID upserts, gossip publish, GraphQL subscription + // broadcast, Arweave anchoring, and peer notify, each keyed to THIS push's + // ref_updates. Coalescing it against an in-flight task for the same repo would DROP + // a later push's ref-update announcements (a correctness regression), not merely + // delay a duplicate. So it is scoped out with rationale, not brought under the bound. { let pinata_jwt = state.config.pinata_jwt.clone(); let pinata_upload_url = state.config.pinata_upload_url.clone(); @@ -4317,6 +4353,175 @@ mod tests { ); } + // ---- #174 U4 (P2-2): post-push encryption task set bounded by per-repo coalescing ---- + // + // The residual jatmn found is not the WALK (bounded by `git_encrypt_semaphore`, + // proven by `encrypt_walk_defers_when_pool_exhausted` above) but the OUTER + // `tokio::spawn` + its parked `acquire_owned().await` waiters: N rapid pushes to a + // repo spawn N tasks that each park holding cloned object lists/rules/keys — an + // unbounded outstanding set. U4 bounds it by coalescing per repo: before spawning, + // if a task for the repo is in flight, skip the duplicate. Crucially this DEFERS a + // duplicate walk (the newer push's objects are covered by the pending one) and does + // NOT shed — there is no reconciliation sweep, so a dropped job would permanently + // lose the withheld-blob recovery copy (`2a54c15`'s fail-closed durability stance). + // + // These drive the coalescing seam (`EncryptInflight`) that the detached spawn at + // `repos.rs` consults directly (the try_begin gate on the in-flight set, guarded by + // `withheld.is_some()`). Observing `encrypt_and_pin`'s IPFS effect end-to-end needs a live IPFS node + // (`pin_git_object` hits the API), so the durability property is proven at this + // layer: a coalesced repo's key is released when its task ends, so a later push for + // that repo is processed once — NOT permanently skipped, which is exactly what a + // coalesce->shed mutation would break by dropping the job with no sweep to recover it. + + /// Bounded outstanding set under saturation (R4). Simulate K rapid path-scoped + /// pushes to the SAME repo while the encrypt pool is saturated (every spawned task + /// would park, so none has finished and removed its key): the first `try_begin` + /// admits (spawns), the rest coalesce (skip). The in-flight set holds at 1, not K. + /// + /// MUTATION (RED): removing the coalescing check makes every push spawn — modeled by + /// `simulate_without_coalescing`, which reaches K. If the coalesced count equaled the + /// un-coalesced one the gate would be a no-op; the strict inequality proves it bites. + #[test] + fn u4_outstanding_encrypt_set_is_bounded_to_one_per_repo_under_saturation() { + let inflight = crate::state::EncryptInflight::new(); + let repo = "did:key:z6MkRepoOwnerAAAAAAAAAAAAAAAAAAAAAAAAAAAA/proj"; + const K: usize = 32; + + // Hold every admitted guard so the tasks are "still in flight" (the saturated + // case: all parked on acquire_owned().await, none finished, none removed a key). + let mut admitted = Vec::new(); + let mut coalesced = 0usize; + for _ in 0..K { + match inflight.try_begin(repo) { + Some(g) => admitted.push(g), + None => coalesced += 1, + } + } + + assert_eq!( + admitted.len(), + 1, + "exactly ONE detached task may spawn per repo while one is in flight — the \ + outstanding set is bounded to 1, not K parked waiters" + ); + assert_eq!( + coalesced, + K - 1, + "the other K-1 rapid pushes to the same repo coalesce (skip spawning)" + ); + assert_eq!( + inflight.len(), + 1, + "the in-flight set holds at most one entry per repo under saturation" + ); + + let no_coalesce = simulate_without_coalescing(K); + assert_eq!( + no_coalesce, K, + "sanity: without the coalescing check all K pushes spawn (the unbounded set \ + the fix prevents) — proves the bound above is not vacuously 1" + ); + assert!( + admitted.len() < no_coalesce, + "coalesced set ({}) must be strictly smaller than the un-coalesced one ({})", + admitted.len(), + no_coalesce + ); + } + + /// Coalescing is PER-REPO: distinct repos are never coalesced against each other, so + /// one repo in flight cannot starve a second repo's recovery copy. + #[test] + fn u4_distinct_repos_each_admit_one_encrypt_task() { + let inflight = crate::state::EncryptInflight::new(); + let a = inflight.try_begin("owner/repo-a"); + let b = inflight.try_begin("owner/repo-b"); + let c = inflight.try_begin("owner/repo-c"); + assert!( + a.is_some() && b.is_some() && c.is_some(), + "three distinct repos each admit their own encryption task" + ); + assert_eq!(inflight.len(), 3, "one in-flight entry per distinct repo"); + } + + /// NO LOST RECOVERY COPY — the security guard (R4/R6). Coalescing must DELAY a + /// duplicate walk, never permanently drop a repo's recovery copy. Observable + /// property: once an in-flight task ENDS (its guard drops — completion, error, or + /// panic-unwind) the repo key is released, so the NEXT push for that repo is admitted + /// and processed again. A coalesce->shed mutation would drop the job AND never + /// re-admit — with no reconciliation sweep the copy is lost forever. Here re-admission + /// survives normal completion AND a panic, so no permanent skip / no leaked key. + #[test] + fn u4_coalesced_repo_is_reprocessed_after_task_ends_not_permanently_skipped() { + let inflight = crate::state::EncryptInflight::new(); + let repo = "did:key:z6MkDurableRepoBBBBBBBBBBBBBBBBBBBBBBBBB/repo"; + + // Push #1 admits and "spawns". A concurrent push #2 (task #1 still in flight) + // coalesces — no duplicate spawn. + let guard1 = inflight.try_begin(repo).expect("first push admits"); + assert!( + inflight.try_begin(repo).is_none(), + "while task #1 is in flight, push #2 to the same repo coalesces" + ); + + // Task #1 finishes (encrypt_and_pin ran or errored): guard drops, key released. + drop(guard1); + assert_eq!( + inflight.len(), + 0, + "when the in-flight task ends its repo key is released — the set does not leak" + ); + + // A LATER push for the SAME repo is admitted again (processed, not skipped + // forever). This is what coalesce->shed breaks: shed drops the job and no sweep + // re-derives the missing copy, so the recovery copy is permanently lost. + let guard2 = inflight.try_begin(repo).expect( + "a later push for a coalesced repo MUST be re-admitted — durability: the \ + deferred recovery copy is produced eventually, never dropped", + ); + drop(guard2); + assert_eq!(inflight.len(), 0); + + // Durability across PANIC: a task that panics mid-walk must still release its key + // (Drop runs on unwind), so one crashed walk never permanently locks a repo out + // of future recovery copies. + let panicked = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _g = inflight.try_begin(repo).expect("admit before the panic"); + assert_eq!(inflight.len(), 1); + panic!("simulate the detached encryption task panicking mid-walk"); + })); + assert!(panicked.is_err(), "the simulated task panicked"); + assert_eq!( + inflight.len(), + 0, + "a panicked encryption task still releases its repo key (Drop on unwind) — no \ + permanent leak that would block every future recovery copy for the repo" + ); + assert!( + inflight.try_begin(repo).is_some(), + "after a panicked task the repo can still be admitted — durability preserved" + ); + } + + /// Degenerate state: the first push on a cold/empty in-flight set always admits + /// (never a false coalesce on an empty set). + #[test] + fn u4_first_push_on_a_cold_set_always_admits() { + let inflight = crate::state::EncryptInflight::new(); + assert!(inflight.is_empty(), "cold set is empty"); + assert!( + inflight.try_begin("owner/first").is_some(), + "the first push on a cold in-flight set must admit (never falsely coalesce)" + ); + } + + /// Model of the pre-fix / mutated code: no coalescing check, so every push spawns. + /// Returns the count of tasks spawned (== the size of the unbounded outstanding set + /// the fix prevents), used as the RED comparison in the bound test above. + fn simulate_without_coalescing(pushes: usize) -> usize { + (0..pushes).count() + } + /// #174 SC2 (per-source key, U1): the per-caller read sub-cap keys on the /// resolved source IP, NOT the signed DID, so a disposable-DID farm cannot /// multiply its budget. Fill the source IP's single read slot, then drive two diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index eed49253..912108a0 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -524,6 +524,7 @@ mod tests { git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + encrypt_inflight: crate::state::EncryptInflight::new(), git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 8727aae1..ffac65a2 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -395,6 +395,11 @@ async fn main() -> Result<()> { git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new( config.max_concurrent_git_pushes, )), + // Coalesces the DETACHED post-push encryption tasks per repo so a rapid pusher + // cannot grow the outstanding parked-waiter set past one task per repo (#174 + // P2-2). No knob: it is a natural cap (one entry per distinct repo), not a + // sized pool. + encrypt_inflight: crate::state::EncryptInflight::new(), git_read_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( config.max_concurrent_reads_per_caller, ), diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 3b8ba250..deccf4c7 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -121,6 +121,16 @@ pub struct AppState { /// walk must not hold a foreground write slot, and a handler already holding a /// write permit that needed a second would self-deadlock at pool size 1. pub git_encrypt_semaphore: Arc, + /// Bounds the OUTSTANDING post-push encryption-task set by per-repo coalescing + /// (#174 P2-2). `git_encrypt_semaphore` caps *active* walks; this caps how many + /// detached tasks *spawn and park* on that semaphore's `acquire_owned().await`. + /// Before spawning a per-push encryption task, the receive-pack handler consults + /// this set: if the repo already has a task in flight it coalesces (skips the + /// duplicate spawn) rather than parking a new unbounded waiter. Coalescing only + /// delays a duplicate walk — it never drops the withheld-blob recovery copy, which + /// `2a54c15` deliberately kept fail-closed (there is no reconciliation sweep to + /// re-derive a dropped copy). See [`EncryptInflight`]. + pub encrypt_inflight: EncryptInflight, /// Per-caller concurrency sub-cap on the read pool: each caller (keyed on the /// resolved source IP, #174 U1) may hold at most `max_concurrent_reads_per_caller` /// in-flight read ops, so one caller cannot monopolize `git_read_semaphore` @@ -203,3 +213,89 @@ impl AppState { *self.shutdown_tx.borrow() } } + +/// Bounds the OUTSTANDING post-push encryption-task set by per-repo coalescing +/// (#174 P2-2). Each successful path-scoped push `tokio::spawn`s a DETACHED task that +/// parks on `git_encrypt_semaphore.acquire_owned().await` (which DEFERS when the pool +/// is full rather than shedding — `2a54c15` kept it fail-closed so the withheld-blob +/// recovery copy is never dropped). The semaphore caps *active* walks, but nothing +/// capped how many detached tasks *spawn and park* on that await: N rapid pushes to a +/// repo spawn N parked tasks, each holding cloned object lists/rules/paths/keys — an +/// unbounded outstanding set. +/// +/// This tracks the repo keys with an in-flight encryption task. Before spawning, the +/// handler calls [`try_begin`](Self::try_begin): if a task for the repo is already +/// in-flight it returns `None` and the handler SKIPS spawning a duplicate (coalesce — +/// the newer push's objects are covered by the pending/next walk over the same repo's +/// history). This bounds the outstanding set to <=1 pending task per repo WITHOUT +/// dropping work: coalescing only delays a duplicate walk, it never sheds the recovery +/// copy (there is no reconciliation sweep, so a *dropped* job would be lost forever). +/// +/// The returned [`EncryptInflightGuard`] is moved into the detached task and removes +/// the repo key on drop — on normal completion, error, OR panic (Drop runs on unwind) +/// — so one crashed walk can never permanently lock a repo out of future recovery +/// copies. +#[derive(Clone, Default)] +pub struct EncryptInflight { + // std::sync::Mutex: only ever held for O(1) HashSet insert/remove in a sync + // context (right before `tokio::spawn`, and in the guard's Drop) — never across + // an await, so a std Mutex is correct and cheaper than a tokio one. + repos: Arc>>, +} + +impl EncryptInflight { + pub fn new() -> Self { + Self::default() + } + + /// Try to begin an encryption task for `repo_id`. Returns `Some(guard)` if no task + /// for the repo was in-flight (the caller should spawn), or `None` if one already + /// is (the caller should COALESCE — skip spawning a duplicate). The guard releases + /// the repo key on drop. + pub fn try_begin(&self, repo_id: &str) -> Option { + let mut set = self.repos.lock().expect("encrypt_inflight mutex poisoned"); + if set.insert(repo_id.to_string()) { + Some(EncryptInflightGuard { + repos: Arc::clone(&self.repos), + repo_id: repo_id.to_string(), + }) + } else { + None + } + } + + /// Number of repos with an in-flight encryption task. Test/metrics observability; + /// the bound under saturation is `len() <= number of distinct repos`, i.e. at most + /// one task per repo. + #[allow(dead_code)] + pub fn len(&self) -> usize { + self.repos + .lock() + .expect("encrypt_inflight mutex poisoned") + .len() + } + + #[allow(dead_code)] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +/// RAII guard removing a repo key from [`EncryptInflight`] when the detached +/// encryption task finishes (drop on completion, error, or panic-unwind). Move-only — +/// there is no reason to clone a guard, and cloning would double-remove. +pub struct EncryptInflightGuard { + repos: Arc>>, + repo_id: String, +} + +impl Drop for EncryptInflightGuard { + fn drop(&mut self) { + // A poisoned lock (a prior panic while holding it) still lets us take the inner + // set via into_inner-on-guard; but poisoning here is not expected because the + // only critical sections are the O(1) ops above. Remove best-effort. + if let Ok(mut set) = self.repos.lock() { + set.remove(&self.repo_id); + } + } +} diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index cfa676af..92f41a8d 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -87,6 +87,7 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + encrypt_inflight: crate::state::EncryptInflight::new(), git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys( 8, diff --git a/crates/gitlawb-node/tests/inv22_gates.rs b/crates/gitlawb-node/tests/inv22_gates.rs index 5f576300..7d3fd1c8 100644 --- a/crates/gitlawb-node/tests/inv22_gates.rs +++ b/crates/gitlawb-node/tests/inv22_gates.rs @@ -77,4 +77,14 @@ fn inv22_concurrency_gates_present_and_not_bypassed() { "P1-e bypass: the bounded recipients walk must be invoked only inside \ withheld_recipients_gated; a new call site bypasses the encrypt-walk admission cap" ); + + // U4 / P2-2: the detached post-push encryption task must be gated by the per-repo + // coalescing set (`encrypt_inflight.try_begin`) so the OUTSTANDING parked-task set is + // bounded to <=1 per repo. Removing the gate lets N rapid pushes spawn N parked + // waiters (the unbounded set U4 closed); the semaphore only caps active walks. + assert!( + repos.contains("state.encrypt_inflight.try_begin"), + "U4/P2-2 gate missing: the detached post-push encryption spawn must consult \ + encrypt_inflight.try_begin to coalesce per repo (bound the outstanding-task set)" + ); } From 413d6cf0bcf1a8c587e01d6c9384de70e2876ecb Mon Sep 17 00:00:00 2001 From: t Date: Wed, 15 Jul 2026 13:16:40 -0500 Subject: [PATCH 37/58] fix(node): reject unsupported git service before the read slot; document the new admission knobs (#174) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit P2-1: git_info_refs treated every service other than git-receive-pack as a read op and did the DB/visibility/Tigris work under a read permit, validating the service string only downstream in smart_http. So an unauthenticated `?service=anything` to a public repo consumed a read slot + storage work before rejection. Validate the service is exactly git-upload-pack or git-receive-pack immediately after parsing, returning 400 before the pre-DB shed and the permit. Handler-layer regression: with the read pool exhausted, `?service=git-explode` returns 400 (validated first), not 503 — removing the validation makes it 503 (RED). U5/INV-24 docs: document the new knobs from U2/U3 in .env.example and the README env-var table — GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS, GITLAWB_MAX_CONCURRENT_IPFS_WALKS, GITLAWB_IPFS_WALK_PER_SOURCE, GITLAWB_IPFS_MAX_REPOS_WALKED, GITLAWB_IPFS_RATE_LIMIT. Each is already enforced on the path it names (U2/U3 tests), so the docs match the implemented routing. --- .env.example | 29 ++++++++++++++++ README.md | 5 +++ crates/gitlawb-node/src/api/repos.rs | 51 ++++++++++++++++++++++++++++ 3 files changed, 85 insertions(+) diff --git a/.env.example b/.env.example index 0ef193f8..149ea8b1 100644 --- a/.env.example +++ b/.env.example @@ -115,6 +115,16 @@ GITLAWB_MAX_PACK_BYTES=2147483648 # Default 600. GITLAWB_GIT_SERVICE_TIMEOUT_SECS=600 +# Max seconds the storage-ACQUISITION phase of a served git op may run before the +# request is shed with a 503, separate from the git-run timeout above. A +# concurrency permit is taken before this phase and GITLAWB_GIT_SERVICE_TIMEOUT_SECS +# only starts once git spawns, so without this a stalled backend (a hung Tigris +# HEAD/GET, or a hung pg advisory-lock iteration on push) pins the permit and drains +# the pool until every later request 503s. On expiry the permit is released +# (fail-closed). Kept separate because acquisition and git execution are distinct +# cost centers. Must be positive; set very large to effectively disable. Default 30. +GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS=30 + # Max concurrent git READ ops (upload-pack + the upload-pack info/refs # advertisement) served at once, a global pool separate from the push pool below. # The anon receive-pack info/refs advertisement has its OWN pool (see below), not @@ -146,6 +156,25 @@ GITLAWB_MAX_CONCURRENT_READS_PER_CALLER=16 # IP per hour. 0 disables. Default 600. GITLAWB_PUSH_RATE_LIMIT=600 +# ── /ipfs/{cid} visibility-walk admission (#174) ────────────────────────── +# GET /ipfs/{cid} runs a per-repo full-history git walk in a blocking thread to +# decide whether the caller may read a path-scoped blob. It is publicly reachable, +# so it is bounded to keep a permissionless caller from fanning out unbounded +# concurrent walks and exhausting blocking-pool threads + PIDs. +# Max concurrent /ipfs walks across all callers (a pool of its own, disjoint from +# the served-git pools). Over-cap sheds a 503. Default 32. +GITLAWB_MAX_CONCURRENT_IPFS_WALKS=32 +# Max concurrent /ipfs walks a single SOURCE IP may hold (keyed like the git +# per-caller caps via GITLAWB_TRUSTED_PROXY; reject-before-insert bounded map). +# Default 4. +GITLAWB_IPFS_WALK_PER_SOURCE=4 +# Max repos walked per single /ipfs request, so one request cannot serialize a +# full-history walk over every repo carrying the CID. Default 64. +GITLAWB_IPFS_MAX_REPOS_WALKED=64 +# Max /ipfs/{cid} requests per client IP per hour (route flood brake, distinct +# from the concurrency caps above). 0 disables. Default 600. +GITLAWB_IPFS_RATE_LIMIT=600 + # ── Creation rate limiting (repo/agent/issue/PR flood brake) ────────────── # Max creation requests (POST /api/v1/repos, /api/register, fork, issues, # pulls) per client IP per hour, in addition to the per-DID limit. The per-DID diff --git a/README.md b/README.md index 542a988e..e9765876 100644 --- a/README.md +++ b/README.md @@ -344,6 +344,11 @@ Important node settings: | `GITLAWB_AUTO_SYNC` | Enable automatic sync from known peers. | | `GITLAWB_MAX_PACK_BYTES` | Max git pack body size for smart-HTTP routes. | | `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack, receive-pack, or `info/refs` advertisement may run before it is aborted (504). Default 600. Also bounds the withheld-blob classification walk (on both the upload-pack serve and receive-pack replication paths) and the push-side pin-candidate discovery (`rev-list` / `cat-file`), each reaped via process-group teardown at the deadline. | +| `GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS` | Max seconds the storage-acquisition phase (Tigris HEAD/GET, push advisory-lock) of a served git op may run before the request is shed with a 503, separate from the git-run timeout. The concurrency permit is released on expiry so a stalled backend cannot pin the pool. Default 30. | +| `GITLAWB_MAX_CONCURRENT_IPFS_WALKS` | Max concurrent `GET /ipfs/{cid}` visibility walks across all callers (own pool, disjoint from the served-git pools); over-cap sheds 503. Default 32. | +| `GITLAWB_IPFS_WALK_PER_SOURCE` | Max concurrent `/ipfs` walks a single source IP may hold. Default 4. | +| `GITLAWB_IPFS_MAX_REPOS_WALKED` | Max repos walked per `/ipfs/{cid}` request, bounding one request's fan-out. Default 64. | +| `GITLAWB_IPFS_RATE_LIMIT` | Max `/ipfs/{cid}` requests per client IP per hour (route flood brake). 0 disables. Default 600. | | `GITLAWB_TIGRIS_BUCKET` | Optional S3/Tigris shared repo storage bucket. | | `GITLAWB_PINATA_JWT` | Optional Pinata/IPFS warm-storage pinning. | | `GITLAWB_IRYS_URL` | Optional Irys/Arweave permanent anchoring. | diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 6b52273f..9b05ece1 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -515,6 +515,16 @@ pub async fn git_info_refs( let service = query .service .ok_or_else(|| AppError::BadRequest("missing ?service= parameter".into()))?; + // Reject an unsupported service BEFORE taking a read slot or doing any DB/Tigris + // work (#174 P2-1). git_info_refs otherwise treats everything that is not + // git-receive-pack as a read op, so an unauthenticated `?service=anything` to a + // public repo would consume a read permit and the visibility/Tigris work before + // validate_service rejected it downstream in smart_http. + if service != "git-upload-pack" && service != "git-receive-pack" { + return Err(AppError::BadRequest(format!( + "unsupported git service: {service}" + ))); + } // #62 cheap pre-DB load shed: if the pool this service draws from is already // saturated, shed with 503 before any DB/disk work. Best-effort (holds no // permit); the authoritative hold is `git_permit` below, after the per-source @@ -3118,6 +3128,47 @@ mod tests { ); } + /// #174 P2-1: an unsupported `?service=` must be rejected with 400 BEFORE taking a + /// read slot or doing DB/Tigris work. Isolate it: exhaust the read pool so a read + /// op WOULD shed 503 at the pre-DB check — a garbage service must still return 400 + /// (validation runs first), proving `?service=anything` cannot consume the read + /// pool. Removing the validation makes this 503 (RED). + #[sqlx::test] + async fn info_refs_rejects_unsupported_service_before_the_read_slot(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + state + .db + .upsert_mirror_repo("z6svcowner", "svc", "/tmp/svc", None, false) + .await + .unwrap(); + // Exhaust the read pool: a read op would shed 503 at the pre-DB check. + state.git_read_semaphore = Arc::new(Semaphore::new(0)); + + let router = crate::server::build_router(state); + let peer: SocketAddr = "203.0.113.90:7000".parse().unwrap(); + let mut req = Request::builder() + .method(Method::GET) + .uri("/z6svcowner/svc/info/refs?service=git-explode") + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + + let status = router.oneshot(req).await.unwrap().status(); + assert_eq!( + status, + StatusCode::BAD_REQUEST, + "an unsupported ?service= must be 400 before the read-pool shed, not 503" + ); + } + /// #174 (jatmn P1): the anon-reachable receive-pack advertisement /// (`GET info/refs?service=git-receive-pack`) draws from a DEDICATED advert pool /// (`git_push_advert_semaphore`), NOT the write pool the authenticated POST uses. From 3af50a53040d7eef18444ecbedf1f663a31e992e Mon Sep 17 00:00:00 2001 From: t Date: Fri, 17 Jul 2026 17:05:28 -0500 Subject: [PATCH 38/58] fix(node): hold filtered-pack admission through disconnect teardown (#174) drive_git_child hands the AdmissionGuard back on success so upload_pack_excluding can thread it across the rev-list and pack-objects stages; on timeout or error it is dropped post-reap, and on future-drop it rides the detached reaper as before. The upload-pack handler builds the guard once ahead of the withheld/plain branch, so a client disconnect mid-filtered-serve no longer frees the read-pool and per-source permits while the git process group is still being reaped. --- crates/gitlawb-node/src/api/repos.rs | 249 +++++++++++- crates/gitlawb-node/src/git/smart_http.rs | 446 +++++++++++++++++++--- 2 files changed, 637 insertions(+), 58 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 9b05ece1..bb9e743b 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -932,19 +932,30 @@ pub async fn git_upload_pack( // the smart_http paths, not a generic 500 (#174 U3). let withheld = withheld.map_err(|e| git_service_app_error(&e))?; + // Move the permits returned by the walk into the guard, ONE construction + // site for both serve arms below, so admission tracks the served git group's + // reap (complete/timeout/disconnect) whether the pack is plain or filtered. + // The handler keeps no copy (F1: handler-local permits would drop the + // instant a disconnect drops this future, mid-reap). + let admission = smart_http::AdmissionGuard::new(_permit, _caller_permit); if withheld.is_empty() { - // No blobs to withhold: serve the plain pack, moving the permits returned by - // the walk into the guard so admission tracks the served git group's reap - // (the walk already held them per be0cdd6; this hands them to the serve). - let admission = smart_http::AdmissionGuard::new(_permit, _caller_permit); + // No blobs to withhold: serve the plain pack (the walk already held the + // permits per be0cdd6; the guard hands them to the serve). smart_http::upload_pack(&state.git_bin, &disk_path, body, git_timeout, Some(admission)).await } else { tracing::info!(repo = %name, caller = ?caller, withheld = withheld.len(), "serving filtered pack"); - // upload_pack_excluding runs its own rev-list/pack-objects (both pass `None` - // admission internally); the walk's permits stay handler-locals held across - // this serve, as be0cdd6 established, and drop when the handler returns. - let _hold = (_permit, _caller_permit); - smart_http::upload_pack_excluding(&disk_path, body, &withheld, git_timeout).await + // The guard threads through both filtered-pack stages (rev-list, then + // pack-objects), so a disconnect mid-stage keeps the permits held until + // that stage's process group is reaped (F1). + smart_http::upload_pack_excluding( + &state.git_bin, + &disk_path, + body, + &withheld, + git_timeout, + Some(admission), + ) + .await } } .map_err(|e| { @@ -4013,6 +4024,226 @@ mod tests { ); } + /// F1 (filtered-serve residual of #174 P1-a, RED-before/GREEN-after): on the + /// FILTERED (path-scoped, non-empty withheld) upload-pack branch a client + /// disconnect mid-pack-objects must NOT release the read admission while the + /// detached reaper is still tearing down the git group. Pre-fix the handler held + /// the permits as locals (`_hold`) across `upload_pack_excluding`, so dropping + /// the future released them instantly and disconnect-spam could exceed the read + /// caps during each reap window (RED). The fix threads the AdmissionGuard through + /// both filtered-pack stages so it rides `KillGroupOnDrop` into the reaper. + /// + /// Same isolation as the plain-path test above: read pool = 1, per-source cap + /// permissive, so only the global permit can shed a replacement. The fake git + /// serves the withheld walk (for-each-ref/rev-parse/rev-list/ls-tree) with a blob + /// under the denied `/src/**` subtree so the filtered branch is taken, answers + /// the pack build's rev-list fast, and hangs pack-objects in a SIGTERM-trapping + /// descendant. The descendant hang is first-invocation-only (keyed on the + /// pidfile's existence) so the post-reap replacement request completes fast. + #[cfg(unix)] + #[sqlx::test] + async fn upload_pack_filtered_permit_held_through_group_reap_after_disconnect( + pool: sqlx::PgPool, + ) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + let tmp = tempfile::TempDir::new().unwrap(); + let descfile = tmp.path().join("desc.pid"); + let commit = "1111111111111111111111111111111111111111"; + let blob = "2222222222222222222222222222222222222222"; + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + rev-parse) echo {commit} ;;\n\ + rev-list) echo {commit} ;;\n\ + ls-tree) printf '100644 blob {blob}\\tsrc/x.txt' ;;\n\ + pack-objects)\n\ + if [ ! -e \"{desc}\" ]; then\n\ + sh -c 'trap \"\" TERM; echo $$ > \"{desc}\"; i=0; while [ $i -lt 20 ]; do sleep 1; i=$((i+1)); done' &\n\ + wait\n\ + fi ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + desc = descfile.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + // Isolate the global read pool: size 1; per-source cap + rate limiter permissive + // so only the leaked global permit can shed the replacement. + state.git_read_semaphore = Arc::new(Semaphore::new(1)); + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + state.git_bin = git_path.to_str().unwrap().to_string(); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let owner = "z6upf1"; + let name = "upf"; + state + .db + .upsert_mirror_repo(owner, name, "/unused", None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + // Real bare repo at the path acquire() computes, so the handler reaches the + // walk and the filtered serve. + state + .repo_store + .init(&rec.owner_did, &rec.name) + .await + .unwrap(); + // Path-scoped rule denying the anonymous caller under /src, matching the + // fake ls-tree's blob path, so the withheld set is NON-EMPTY and the + // filtered (upload_pack_excluding) branch is taken. + state + .db + .set_visibility_rule( + &rec.id, + "/src/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkUF1ReaderAAAAAAAAAAAAAAAAAAAAAAAAAAA".to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + + let sem = state.git_read_semaphore.clone(); + assert_eq!( + sem.available_permits(), + 1, + "one read slot before the request" + ); + + let router = crate::server::build_router(state); + let make_req = |peer: SocketAddr| { + let mut req = Request::builder() + .method(Method::POST) + .uri(format!("/{owner}/{name}/git-upload-pack")) + .body(Body::from(&b"0000"[..])) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + req + }; + + let peer: SocketAddr = "203.0.113.81:5000".parse().unwrap(); + let mut fut = Box::pin(router.clone().oneshot(make_req(peer))); + // Drive until the pack-objects descendant records its pid: the request is + // past the walk, inside the filtered serve's stage 2, holding the read permit. + // Stop polling the instant the future completes (re-polling a completed + // oneshot panics); read the descfile first so a spawn that recorded its pid + // then returned is still caught. + let mut spawned: Option = None; + let mut early = None; + for _ in 0..500 { + let done = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + if let Some(p) = std::fs::read_to_string(&descfile) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + spawned = Some(p); + break; + } + if let Ok(resp) = done { + early = Some(resp.map(|r| r.status())); + break; + } + } + let desc = spawned.unwrap_or_else(|| { + panic!("the fake pack-objects must have spawned; early finish: {early:?}") + }); + // Kill the descendant regardless of outcome so a RED run leaks no orphan. + struct ReapOnDrop(i32); + impl Drop for ReapOnDrop { + fn drop(&mut self) { + unsafe { + libc::kill(self.0, libc::SIGKILL); + } + } + } + let _cleanup = ReapOnDrop(desc); + assert!( + unsafe { libc::kill(desc, 0) == 0 }, + "descendant should be running before the disconnect" + ); + assert_eq!( + sem.available_permits(), + 0, + "the read slot is held while the filtered serve runs" + ); + + // Client disconnect: drop the request future mid-pack-objects. The detached + // reaper must now own the AdmissionGuard and hold it until ESRCH. + drop(fut); + + // Load-bearing: the slot must STAY held while the SIGTERM-ignoring group is + // still alive. On the pre-fix code the handler-local `_hold` drops here and + // the slot frees at once (RED). Check quickly (before the reaper's ~2s + // SIGKILL escalation). + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + assert!( + unsafe { libc::kill(desc, 0) == 0 }, + "the SIGTERM-ignoring descendant must still be alive during the hold window" + ); + assert_eq!( + sem.available_permits(), + 0, + "on disconnect the read admission must be HELD until the filtered serve's \ + process group is reaped, not released the instant the future drops (F1)" + ); + // A replacement request from a DIFFERENT source must shed 503: the only pool + // that can shed it is the held global permit (per-source cap is permissive). + let peer2: SocketAddr = "203.0.113.82:5000".parse().unwrap(); + let resp = router.clone().oneshot(make_req(peer2)).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "while the prior group is still alive the held global permit must shed a \ + replacement with 503" + ); + + // After the reaper SIGKILLs + reaps the group the AdmissionGuard drops and + // the slot frees. Poll for recovery. + let mut freed = false; + for _ in 0..400 { + if sem.available_permits() == 1 { + freed = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + assert!( + freed, + "once the reaper confirms the group gone the admission guard must drop and \ + free the global slot" + ); + // A replacement is now admitted and completes: the fake pack-objects takes + // its fast path (the descfile exists), so the filtered serve returns instead + // of hanging. + let peer3: SocketAddr = "203.0.113.83:5000".parse().unwrap(); + let resp = router.oneshot(make_req(peer3)).await.unwrap(); + assert_ne!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "after the group is reaped the freed slot must admit a replacement" + ); + } + /// #174 U1 (P1-a): the `None`-key arm — a request with no resolvable source key /// (no trusted-proxy header, no peer) is bounded by the GLOBAL read pool only, never /// a per-source cap. With the global read pool exhausted such a request still sheds diff --git a/crates/gitlawb-node/src/git/smart_http.rs b/crates/gitlawb-node/src/git/smart_http.rs index 64d3cb7f..f8b46ce8 100644 --- a/crates/gitlawb-node/src/git/smart_http.rs +++ b/crates/gitlawb-node/src/git/smart_http.rs @@ -24,7 +24,9 @@ use tokio::process::Command; /// /// The `be0cdd6` path-scoped upload-pack walk already applies this discipline by /// moving its permits into the `spawn_blocking`; this generalizes it to the plain -/// (non-path-scoped) `info_refs` / `run_git_service` spawn paths. +/// (non-path-scoped) `info_refs` / `run_git_service` spawn paths and, via the +/// hand-back contract on `drive_git_child`, across both stages of the filtered +/// (`upload_pack_excluding`) pack build (F1). pub struct AdmissionGuard { // Boxed so the concrete permit types stay private to the caller (repos.rs) — the // guard only needs to hold them and drop them, never inspect them. `Send + @@ -71,8 +73,11 @@ pub async fn info_refs( .arg("--advertise-refs") .arg(repo_path); // No request body — advertise-refs does not read stdin. - let stdout = + let (stdout, admission) = drive_git_child(command, Bytes::new(), timeout, "advertise-refs", admission).await?; + // Single-stage op: the advertisement's group is reaped by now; release admission + // rather than holding it across the pure response framing below. + drop(admission); let content_type = format!("application/x-{service}-advertisement"); @@ -174,11 +179,12 @@ struct KillGroupOnDrop { // own SIGCHLD-driven orphan reaper. child: Option, pgid: Option, - // The global + per-source admission permits for this op, if any. On the plain - // (non-path-scoped) spawn paths repos.rs moves its permits in here so admission is - // released only when the group is confirmed reaped, on every exit — complete, - // timeout, or client-disconnect (#174 P1-a). The rev-list/pack-objects - // visibility-walk callers hold no admission and pass `None`. + // The global + per-source admission permits for this op, if any. repos.rs moves + // its permits in here (directly on the plain spawn paths, via the stage threading + // on the filtered rev-list/pack-objects ones) so admission is released only when + // the group is confirmed reaped, on every exit: complete, timeout, or + // client-disconnect (#174 P1-a, F1). `None` when the op carries no admission + // (e.g. the smart_http test harness). admission: Option, } @@ -191,9 +197,10 @@ impl KillGroupOnDrop { /// Disarm on the success/timeout path: the body has already reaped the child (its /// `wait()` returned, or `reap_group_on_timeout` ran), so drop the handle and clear - /// the pgid, leaving the guard's Drop a no-op. Returns the admission guard so the - /// caller drops it at the earliest provably-free point (the group is already reaped - /// on both callers of this) rather than at end-of-bookkeeping. + /// the pgid, leaving the guard's Drop a no-op. Returns the admission guard: the + /// group is already reaped on both callers of this, so the success path hands it + /// back to drive_git_child's caller (a multi-stage build carries it to the next + /// stage) and the timeout path drops it there and then. fn disarm(&mut self) -> Option { self.pgid = None; self.child = None; @@ -331,23 +338,36 @@ async fn run_git_service( .arg(service_to_command(service)) .arg("--stateless-rpc") .arg(repo_path); - drive_git_child(command, input, timeout, service, admission).await + let (out, admission) = drive_git_child(command, input, timeout, service, admission).await?; + // Single-stage op: the child group is already reaped when drive_git_child + // returns, so releasing admission here keeps the permits held exactly for the + // process lifetime, no longer. + drop(admission); + Ok(out) } /// Drive a spawned git child under `timeout` with process-group teardown, returning -/// its stdout. Shared core for [`run_git_service`] and [`info_refs`]: the caller +/// its stdout together with the admission guard. Shared core for +/// [`run_git_service`], [`info_refs`], and the filtered-pack stages: the caller /// passes a `Command` with its args set; this adds piped stdio and `process_group(0)`. /// On the deadline the whole group is torn down and reaped before returning /// [`GitServiceTimeout`]; on a dropped future (client disconnect) the /// [`KillGroupOnDrop`] guard fires. `input` is written to the child's stdin (empty /// for the advertise-refs path, which has no request body); `what` labels errors. +/// +/// Admission contract: on success the guard is handed BACK so a multi-stage caller +/// (the filtered-pack build) can carry one admission across consecutive children +/// instead of releasing it between stages. On every Err return (timeout, spawn +/// failure, interaction error, non-zero exit) the guard is dropped internally, and +/// only after the child is reaped: an Err cannot carry the guard, and by then the +/// group is confirmed gone, so releasing admission there is correct. async fn drive_git_child( mut command: Command, input: Bytes, timeout: Duration, what: &str, admission: Option, -) -> Result> { +) -> Result<(Vec, Option)> { command .stdin(Stdio::piped()) .stdout(Stdio::piped()) @@ -382,10 +402,10 @@ async fn drive_git_child( pgid, admission, }; - // On non-unix there is no process-group teardown, so hold the admission guard here - // for the child's whole interaction; it drops when this function returns. - #[cfg(not(unix))] - let _admission = admission; + // On non-unix there is no process-group teardown; the `admission` parameter + // itself stays live for the child's whole interaction (nothing moves it until + // the match below), so the success path hands it back like the unix path and + // the timeout path drops it after the child is waited on. let mut out = Vec::new(); let mut err = Vec::new(); @@ -419,16 +439,18 @@ async fn drive_git_child( }; let timed = tokio::time::timeout(timeout, interact).await; - let (write_result, status) = match timed { + let (write_result, status, admission) = match timed { Ok(result) => { // The join runs all arms to completion, so the child is reaped: disarm // before surfacing any interaction error (a read/wait error), else the // guard's drop would reap an already-reaped child / signal a reused pgid. - // Dropping the returned admission guard here releases the permits at the - // earliest provably-free point on the success path (the op finished). + // Disarming hands the admission guard back; it rides to the Ok return + // below, and every error return between here and there drops it, always + // AFTER the reap, the earliest provably-free point. #[cfg(unix)] - drop(group_guard.disarm()); - result? + let admission = group_guard.disarm(); + let (write_result, status) = result?; + (write_result, status, admission) } Err(_elapsed) => { // Timeout: tear the whole group down and reap it before returning so a @@ -444,6 +466,7 @@ async fn drive_git_child( { let _ = child.start_kill(); let _ = tokio::time::timeout(Duration::from_secs(2), child.wait()).await; + drop(admission); } return Err(GitServiceTimeout.into()); } @@ -452,7 +475,8 @@ async fn drive_git_child( // Surface git's own failure (its stderr, which the handler may classify as a // 400) before any stdin-write error: when git rejects a malformed body it // exits non-zero and closes stdin, so the write's EPIPE would otherwise mask - // the real cause. + // the real cause. `admission` drops on these error returns; the child is + // already reaped by the completed join above. if !status.success() { let stderr = String::from_utf8_lossy(&err); bail!("{what} failed: {stderr}"); @@ -460,7 +484,7 @@ async fn drive_git_child( write_result.context("failed to write to git stdin")?; - Ok(out) + Ok((out, admission)) } fn service_to_command(service: &str) -> &str { @@ -498,14 +522,17 @@ async fn rev_list_keep( repo_path: &Path, withheld: &HashSet, timeout: Duration, -) -> Result> { + admission: Option, +) -> Result<(Vec, Option)> { let mut command = Command::new(git_bin); command .args(["rev-list", "--objects", "--all"]) .current_dir(repo_path); - // The visibility-walk callers intentionally hold no admission permit (their - // admission is governed elsewhere), so pass `None` (#174 KTD2). - let stdout = drive_git_child(command, Bytes::new(), timeout, "rev-list", None).await?; + // The filtered-serve caller's admission rides through drive_git_child so a + // disconnect mid-enumeration keeps the permits held until the rev-list group is + // reaped; on success the guard comes back for the pack-objects stage (F1). + let (stdout, admission) = + drive_git_child(command, Bytes::new(), timeout, "rev-list", admission).await?; let mut keep = Vec::new(); for line in String::from_utf8_lossy(&stdout).lines() { let oid = line.split_whitespace().next().unwrap_or(""); @@ -514,7 +541,7 @@ async fn rev_list_keep( } keep.push(oid.to_string()); } - Ok(keep) + Ok((keep, admission)) } /// Build a packfile containing every object reachable from all refs EXCEPT the @@ -527,20 +554,30 @@ async fn rev_list_keep( /// disconnect. An outer `tokio::time::timeout` around a `spawn_blocking` cannot /// cancel the blocking thread, so neither stage may live off the async side /// (#174, KTD5). +/// +/// `admission` carries the filtered-serve permits across BOTH stages: stage 1 hands +/// it back on success, stage 2 takes it, and the caller receives it after stage 2 so +/// release happens only once the last child group is reaped. On a disconnect the +/// guard rides the active stage's detached reaper instead of dropping with the +/// future (F1). A stage that fails or times out (including a stage 2 whose +/// remaining budget has saturated to zero) drops the guard inside drive_git_child, +/// after that stage's reap. pub async fn build_filtered_pack( git_bin: &str, repo_path: &Path, withheld: &HashSet, timeout: Duration, -) -> Result> { + admission: Option, +) -> Result<(Vec, Option)> { // One deadline spans both git stages so a slow rev-list eats into the pack // budget rather than granting each stage a fresh `timeout` (2x the permit hold). let deadline = Instant::now() + timeout; - let keep = rev_list_keep( + let (keep, admission) = rev_list_keep( git_bin, repo_path, withheld, deadline.saturating_duration_since(Instant::now()), + admission, ) .await?; let mut data = keep.join("\n").into_bytes(); @@ -554,8 +591,7 @@ pub async fn build_filtered_pack( Bytes::from(data), deadline.saturating_duration_since(Instant::now()), "pack-objects", - // Visibility-walk pack build: no admission permit here (#174 KTD2). - None, + admission, ) .await } @@ -584,16 +620,24 @@ pub async fn build_filtered_pack( /// instead of a thin delta. Honoring negotiation for smaller fetch packs is an /// optimization follow-up, not a correctness requirement. pub async fn upload_pack_excluding( + git_bin: &str, repo_path: &Path, request_body: Bytes, withheld: &HashSet, timeout: Duration, + admission: Option, ) -> Result { - // The rev-list enumeration runs blocking off the runtime; the streaming - // pack-objects stage is duration-bounded and its process group is reaped on - // disconnect via drive_git_child (#174), so a hung build no longer pins its - // concurrency slot and a client disconnect no longer orphans the git child. - let pack = build_filtered_pack("git", repo_path, withheld, timeout).await?; + // Both filtered-pack stages run async under drive_git_child (duration-bounded, + // process group reaped on disconnect, #174), and `admission` threads through + // them so the caller's permits release only after the active stage's group is + // reaped, never the instant a disconnect drops this future (F1). `git_bin` is + // injectable for the same fake-git testing reason as `run_git_service` + // (production passes the configured git binary). + let (pack, admission) = + build_filtered_pack(git_bin, repo_path, withheld, timeout, admission).await?; + // Both git stages are done and their groups reaped; release admission before the + // pure in-memory response framing below. + drop(admission); // The client lists its capabilities on the first `want` line. Honor // side-band-64k when offered (every modern smart-HTTP client offers it); @@ -712,9 +756,10 @@ mod tests { let mut withheld = std::collections::HashSet::new(); withheld.insert(secret.clone()); - let pack = build_filtered_pack("git", &bare, &withheld, Duration::from_secs(30)) - .await - .unwrap(); + let (pack, _admission) = + build_filtered_pack("git", &bare, &withheld, Duration::from_secs(30), None) + .await + .unwrap(); let ids = pack_object_ids(&pack); assert!(ids.contains(&public), "public blob must be in the pack"); assert!( @@ -776,9 +821,10 @@ mod tests { b"0098want 0000000000000000000000000000000000000000 \ side-band-64k ofs-delta agent=git/2\n00000009done\n", ); - let resp = upload_pack_excluding(&bare, req, &withheld, Duration::from_secs(30)) - .await - .unwrap(); + let resp = + upload_pack_excluding("git", &bare, req, &withheld, Duration::from_secs(30), None) + .await + .unwrap(); let body = to_bytes(resp.into_body(), usize::MAX).await.unwrap(); let ids = pack_object_ids(&extract_pack(&body)); assert!( @@ -837,9 +883,16 @@ mod tests { axum::extract::State(st): axum::extract::State>, body: Bytes, ) -> Response { - upload_pack_excluding(&st.repo, body, &st.withheld, Duration::from_secs(30)) - .await - .unwrap() + upload_pack_excluding( + "git", + &st.repo, + body, + &st.withheld, + Duration::from_secs(30), + None, + ) + .await + .unwrap() } /// Spawn the server for `bare`, withholding `withheld`. Returns the clone URL @@ -1359,14 +1412,19 @@ mod tests { for i in 0..FAKE_GIT_RETRY_ATTEMPTS { let result = tokio::time::timeout( Duration::from_secs(10), - build_filtered_pack(git_bin, repo_path, withheld, stage_timeout), + build_filtered_pack(git_bin, repo_path, withheld, stage_timeout, None), ) .await .expect( "build_filtered_pack must return within the watchdog — the git stage \ must be timeout-bounded, not an uncancellable spawn_blocking", ); - let err = result.expect_err("a hung git stage must return an error, not hang"); + // `match` rather than `expect_err`: the Ok arm carries an + // AdmissionGuard, which has no Debug impl for expect_err to print. + let err = match result { + Ok(_) => panic!("a hung git stage must return an error, not hang"), + Err(e) => e, + }; if is_transient_exec_race(&err) { // Fresh-fake-git ETXTBSY: back off (growing) so a bursty fork-pressure // spike subsides before retrying, per fake_git_run_with_pids. @@ -1655,6 +1713,296 @@ mod tests { ); } + // ── F1: filtered-serve admission threaded through both pack stages ────── + // + // The handler-level disconnect regression lives in api/repos.rs + // (upload_pack_filtered_permit_held_through_group_reap_after_disconnect); these + // exercise the guard contract at the smart_http seam: held through a + // mid-rev-list disconnect, handed back on success, and recovered (post-reap) + // on every error return. + + /// F1: a client disconnect mid-REV-LIST (stage 1 of the filtered build) must + /// keep the threaded admission held until the rev-list group is reaped, and + /// stage 2 (pack-objects) must never spawn, since the dropped future cannot + /// advance to it. + #[cfg(unix)] + #[tokio::test] + async fn build_filtered_pack_holds_admission_through_rev_list_reap_on_disconnect() { + let tmp = tempfile::TempDir::new().unwrap(); + let descfile = tmp.path().join("desc.pid"); + let packfile = tmp.path().join("pack.ran"); + // rev-list: SIGTERM-trapping descendant records its pid and loops (bounded + // so a broken fix leaks no permanent orphan); the leader waits, keeping the + // interaction pending. pack-objects: records that it ran (it must never run). + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + rev-list)\n\ + sh -c 'trap \"\" TERM; echo $$ > \"{desc}\"; i=0; while [ $i -lt 20 ]; do sleep 1; i=$((i+1)); done' &\n\ + wait ;;\n\ + pack-objects) : > \"{pack}\" ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + desc = descfile.display(), + pack = packfile.display() + ); + let git_bin = write_fake_git(tmp.path(), &body); + let sem = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + let guard = AdmissionGuard::new(sem.clone().try_acquire_owned().unwrap(), None::<()>); + let withheld = HashSet::new(); + + let mut fut = Box::pin(build_filtered_pack( + git_bin.to_str().unwrap(), + tmp.path(), + &withheld, + Duration::from_secs(60), + Some(guard), + )); + // Drive until the rev-list descendant records its pid. + let mut desc: Option = None; + for _ in 0..500 { + let _ = tokio::time::timeout(Duration::from_millis(10), &mut fut).await; + if let Some(p) = std::fs::read_to_string(&descfile) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + desc = Some(p); + break; + } + } + let desc = desc.expect("the fake rev-list must have spawned its descendant"); + let _cleanup = ReapOnPanic(vec![desc]); + assert_eq!( + sem.available_permits(), + 0, + "admission held while rev-list runs" + ); + + // Client disconnect mid-enumeration. + drop(fut); + + // Load-bearing: the permit must stay held while the SIGTERM-ignoring group + // member is still alive (check before the reaper's ~2s SIGKILL escalation). + tokio::time::sleep(Duration::from_millis(100)).await; + assert!( + alive(desc), + "the SIGTERM-ignoring descendant must still be alive during the hold window" + ); + assert_eq!( + sem.available_permits(), + 0, + "on disconnect mid-rev-list the admission must be held until the group is \ + reaped, not released the instant the future drops (F1)" + ); + + let mut freed = false; + for _ in 0..400 { + if sem.available_permits() == 1 { + freed = true; + break; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + assert!( + freed, + "the reaper must release admission once the rev-list group is gone" + ); + assert!( + !packfile.exists(), + "pack-objects must never spawn after a disconnect mid-rev-list" + ); + } + + /// F1 success path: a filtered serve that completes hands the guard back + /// through both stages and releases admission when the serve returns. No leak, + /// no double-release. + #[tokio::test] + async fn upload_pack_excluding_releases_admission_after_success() { + let td = TempDir::new().unwrap(); + let (_work, bare, secret_oid, _public_oid) = fixture_with_secret(&td); + let sem = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + let guard = AdmissionGuard::new(sem.clone().try_acquire_owned().unwrap(), None::<()>); + let withheld = HashSet::from([secret_oid]); + let resp = upload_pack_excluding( + "git", + &bare, + Bytes::from_static(b"0000"), + &withheld, + Duration::from_secs(30), + Some(guard), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + assert_eq!( + sem.available_permits(), + 1, + "admission must be released once the filtered serve returns" + ); + } + + /// F1 error path: a rev-list that exits non-zero fails the build AND recovers + /// the threaded admission: drive_git_child drops the guard only after the + /// completed join has reaped the child, and the error return must not leak it. + #[cfg(unix)] + #[tokio::test] + async fn build_filtered_pack_releases_admission_after_rev_list_failure() { + let tmp = tempfile::TempDir::new().unwrap(); + let body = + "#!/bin/sh\ncase \"$1\" in\n rev-list) echo boom >&2; exit 2 ;;\n *) : ;;\nesac\nexit 0\n"; + let git_bin = write_fake_git(tmp.path(), body); + let withheld = HashSet::new(); + let sem = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + + for i in 0..FAKE_GIT_RETRY_ATTEMPTS { + let guard = AdmissionGuard::new(sem.clone().try_acquire_owned().unwrap(), None::<()>); + let result = build_filtered_pack( + git_bin.to_str().unwrap(), + tmp.path(), + &withheld, + Duration::from_secs(30), + Some(guard), + ) + .await; + let err = match result { + Ok(_) => panic!("a non-zero rev-list exit must fail the build"), + Err(e) => e, + }; + // Load-bearing on EVERY attempt: even a transient spawn-failure return + // must recover the permit (the guard drops before the Err surfaces). + assert_eq!( + sem.available_permits(), + 1, + "the admission permit must be recovered on the error return, not leaked" + ); + if is_transient_exec_race(&err) { + tokio::time::sleep(Duration::from_millis(FAKE_GIT_BACKOFF_STEP_MS * (i + 1))).await; + continue; + } + assert!( + err.to_string().contains("rev-list failed"), + "the failure must surface rev-list's own error, got: {err}" + ); + return; + } + panic!( + "fake git kept hitting ETXTBSY after {FAKE_GIT_RETRY_ATTEMPTS} attempts \ + (persistent exec failure, not a transient parallel-runner miss)" + ); + } + + /// F1: a pack-objects stage that times out (stage 2 of the filtered build, + /// guard threaded through) recovers the admission after the reap: the timeout + /// Err cannot carry the guard, so drive_git_child must drop it post-reap. + #[cfg(unix)] + #[tokio::test] + async fn build_filtered_pack_recovers_admission_after_pack_objects_timeout() { + let tmp = tempfile::TempDir::new().unwrap(); + // rev-list returns one oid fast (guard handed back); pack-objects hangs. + let body = "#!/bin/sh\ncase \"$1\" in\n rev-list) echo deadbeefdeadbeefdeadbeefdeadbeefdeadbeef ;;\n pack-objects) sleep 300 ;;\n *) exit 1 ;;\nesac\n"; + let git_bin = write_fake_git(tmp.path(), body); + let withheld = HashSet::new(); + let sem = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + + for i in 0..FAKE_GIT_RETRY_ATTEMPTS { + let guard = AdmissionGuard::new(sem.clone().try_acquire_owned().unwrap(), None::<()>); + let result = tokio::time::timeout( + Duration::from_secs(10), + build_filtered_pack( + git_bin.to_str().unwrap(), + tmp.path(), + &withheld, + Duration::from_millis(200), + Some(guard), + ), + ) + .await + .expect( + "build_filtered_pack must return within the watchdog; the git stage \ + must be timeout-bounded", + ); + let err = match result { + Ok(_) => panic!("a hung pack-objects must fail the build"), + Err(e) => e, + }; + assert_eq!( + sem.available_permits(), + 1, + "admission must be recovered after the stage-2 timeout reap, not leaked" + ); + if is_transient_exec_race(&err) { + tokio::time::sleep(Duration::from_millis(FAKE_GIT_BACKOFF_STEP_MS * (i + 1))).await; + continue; + } + assert!( + err.downcast_ref::().is_some(), + "a hung pack-objects must abort with GitServiceTimeout, got: {err}" + ); + return; + } + panic!( + "fake git kept hitting ETXTBSY after {FAKE_GIT_RETRY_ATTEMPTS} attempts \ + (persistent exec failure, not a transient parallel-runner miss)" + ); + } + + /// F1 zero-budget contract: build_filtered_pack hands pack-objects + /// `deadline.saturating_duration_since(now)`, which saturates to ZERO once + /// rev-list has consumed the whole budget. A zero-budget stage must still route + /// the guard through the timeout's reap-then-drop rather than leak it: the + /// child spawns, the deadline fires immediately, the group is reaped, and only + /// then does the guard drop. + #[cfg(unix)] + #[tokio::test] + async fn drive_git_child_zero_budget_drops_admission_after_reap() { + let tmp = tempfile::TempDir::new().unwrap(); + let git_bin = write_fake_git(tmp.path(), "#!/bin/sh\nsleep 300\n"); + let sem = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + + for i in 0..FAKE_GIT_RETRY_ATTEMPTS { + let guard = AdmissionGuard::new(sem.clone().try_acquire_owned().unwrap(), None::<()>); + let mut command = tokio::process::Command::new(git_bin.to_str().unwrap()); + command + .args(["pack-objects", "--stdout"]) + .current_dir(tmp.path()); + let result = tokio::time::timeout( + Duration::from_secs(10), + drive_git_child( + command, + Bytes::new(), + Duration::ZERO, + "pack-objects", + Some(guard), + ), + ) + .await + .expect("a zero-budget stage must abort via its own deadline, not hang"); + let err = match result { + Ok(_) => panic!("a zero-budget stage must return an error"), + Err(e) => e, + }; + assert_eq!( + sem.available_permits(), + 1, + "the guard must drop after the zero-budget timeout reap, not leak" + ); + if is_transient_exec_race(&err) { + tokio::time::sleep(Duration::from_millis(FAKE_GIT_BACKOFF_STEP_MS * (i + 1))).await; + continue; + } + assert!( + err.downcast_ref::().is_some(), + "a zero-budget stage must abort with GitServiceTimeout, got: {err}" + ); + return; + } + panic!( + "fake git kept hitting ETXTBSY after {FAKE_GIT_RETRY_ATTEMPTS} attempts \ + (persistent exec failure, not a transient parallel-runner miss)" + ); + } + // A request that runs to completion must DISARM the guard after reaping, so // no stray group SIGTERM fires. The fake exits non-zero (surfacing as Err) // but leaves a grandchild alive; the grandchild must survive. Goes RED if the From 138b48210a1ae949edc798a93afd1ee38412cfb9 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 17 Jul 2026 17:35:01 -0500 Subject: [PATCH 39/58] fix(node): make /ipfs scan verdicts honest: walk-scoped cap, visit ceiling, 503 on truncation (#174) A request-scoped taint list replaces the silent folds: any repo skipped without a verdict (acquire timeout, probe or read failure, walk failure, cap or ceiling exhaustion) turns the terminal into a retryable 503 with Retry-After, and 404 is reserved for a scan in which every candidate reached a verdict. The walk cap now counts only expensive allowed-set walks and skip-continues on exhaustion so a plain public copy past the cap still serves; a new GITLAWB_IPFS_MAX_REPO_VISITS knob (default 1024) bounds the acquire+probe cost class and stops the scan, documented with its worst-case object-store fetch count. --- .env.example | 12 +- README.md | 3 +- crates/gitlawb-node/src/api/ipfs.rs | 706 ++++++++++++++++++++++-- crates/gitlawb-node/src/config.rs | 54 +- crates/gitlawb-node/src/git/tigris.rs | 19 + crates/gitlawb-node/src/test_support.rs | 25 +- 6 files changed, 765 insertions(+), 54 deletions(-) diff --git a/.env.example b/.env.example index 149ea8b1..144108ba 100644 --- a/.env.example +++ b/.env.example @@ -168,9 +168,17 @@ GITLAWB_MAX_CONCURRENT_IPFS_WALKS=32 # per-caller caps via GITLAWB_TRUSTED_PROXY; reject-before-insert bounded map). # Default 4. GITLAWB_IPFS_WALK_PER_SOURCE=4 -# Max repos walked per single /ipfs request, so one request cannot serialize a -# full-history walk over every repo carrying the CID. Default 64. +# Max EXPENSIVE path-scope visibility walks per single /ipfs request (only a +# blob in a path-scoped repo costs a full-history walk). Over-cap repos are +# skipped without a verdict and the scan continues; if the object is then found +# nowhere the request sheds a retryable 503 instead of a false 404. Default 64. GITLAWB_IPFS_MAX_REPOS_WALKED=64 +# Ceiling on repos one /ipfs request may VISIT past the visibility gate. Each +# visit costs a repo acquire — on a Tigris cache miss a full archive download, +# so this is also the worst-case object-store fetch count per request — plus a +# cat-file probe. On exhaustion the scan stops and sheds a retryable 503. +# Default 1024. +GITLAWB_IPFS_MAX_REPO_VISITS=1024 # Max /ipfs/{cid} requests per client IP per hour (route flood brake, distinct # from the concurrency caps above). 0 disables. Default 600. GITLAWB_IPFS_RATE_LIMIT=600 diff --git a/README.md b/README.md index e9765876..f3fa361b 100644 --- a/README.md +++ b/README.md @@ -347,7 +347,8 @@ Important node settings: | `GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS` | Max seconds the storage-acquisition phase (Tigris HEAD/GET, push advisory-lock) of a served git op may run before the request is shed with a 503, separate from the git-run timeout. The concurrency permit is released on expiry so a stalled backend cannot pin the pool. Default 30. | | `GITLAWB_MAX_CONCURRENT_IPFS_WALKS` | Max concurrent `GET /ipfs/{cid}` visibility walks across all callers (own pool, disjoint from the served-git pools); over-cap sheds 503. Default 32. | | `GITLAWB_IPFS_WALK_PER_SOURCE` | Max concurrent `/ipfs` walks a single source IP may hold. Default 4. | -| `GITLAWB_IPFS_MAX_REPOS_WALKED` | Max repos walked per `/ipfs/{cid}` request, bounding one request's fan-out. Default 64. | +| `GITLAWB_IPFS_MAX_REPOS_WALKED` | Max expensive path-scope visibility walks per `/ipfs/{cid}` request; over-cap repos are skipped and the scan continues, shedding a retryable 503 (not a false 404) if the object is then found nowhere. Default 64. | +| `GITLAWB_IPFS_MAX_REPO_VISITS` | Ceiling on repos one `/ipfs/{cid}` request may visit (acquire + probe) past the visibility gate — also the worst-case per-request Tigris fetch count. On exhaustion the scan stops with a retryable 503. Default 1024. | | `GITLAWB_IPFS_RATE_LIMIT` | Max `/ipfs/{cid}` requests per client IP per hour (route flood brake). 0 disables. Default 600. | | `GITLAWB_TIGRIS_BUCKET` | Optional S3/Tigris shared repo storage bucket. | | `GITLAWB_PINATA_JWT` | Optional Pinata/IPFS warm-storage pinning. | diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index ac8e615c..d4a6c7a8 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -51,6 +51,15 @@ use crate::visibility::{visibility_check, Decision}; /// fail-closed 404'd under path-scoped rules (#126). Denial and genuine /// not-found both fall through to an opaque 404. /// +/// Scan completeness (F2): the 404 above is returned ONLY when every candidate +/// repo reached a VERDICT — visibility deny, probe-says-absent, walk-gate deny, +/// or served. A candidate skipped WITHOUT a verdict (acquire failure/timeout, +/// probe error, walk failure/panic, content-read error, or truncation by +/// `ipfs_max_repos_walked` / `ipfs_max_repo_visits`) taints the scan, and a +/// tainted scan that found nothing sheds a retryable 503 + Retry-After naming +/// the truncation sources — existing content is never misreported absent +/// because of unrelated repos or transient faults. +/// /// Scope: this closes the direct unauthenticated scan, including the dangling /// case. A stale-public mirror row still serves withheld content (tracked /// separately, #124). @@ -139,13 +148,35 @@ pub async fn get_by_cid( // deny entry (#126). let mut allowed_memo: HashMap> = HashMap::new(); - // Cap the number of candidate repos one request walks (it already short-circuits on - // serve): a CID present in — or path-gated out of — many repos must not serialize an - // unbounded number of full-history walks inside the single held admission slot. + // Verdict-or-taint bookkeeping (F2): a candidate repo the loop cannot bring to a + // VERDICT (visibility deny / probe-says-absent / walk-gate deny / served) marks + // the scan truncated with its source. A truncated scan that finds nothing must + // NOT report 404 — the object may sit in a repo we skipped — so the terminal arm + // sheds a retryable 503 naming the sources, keyed so the operator can tell which + // knob (or backend) to look at. + let mut truncated_by: Vec<&'static str> = Vec::new(); + fn taint(truncated_by: &mut Vec<&'static str>, source: &'static str) { + if !truncated_by.contains(&source) { + truncated_by.push(source); + } + } + + // Cap on EXPENSIVE walks only (F2): counts the repos that actually require the + // full-history `allowed_blob_set_for_caller_bounded` walk (a path-scoped blob), + // checked immediately before the spawn_blocking below. Cheap probe-only visits + // are bounded by `repos_visited` — counting them here starved later-ordered + // repos out of a plain 200 on nodes with more readable repos than the cap. let mut repos_walked: usize = 0; + // Ceiling on VISITS (F2): every repo past the visibility gate costs an acquire + // (worst case a full Tigris archive download on a cache miss) plus a cat-file + // probe, so one request can trigger at most `ipfs_max_repo_visits` object-store + // fetches. On exhaustion the scan STOPS — there is no cheaper way to continue. + let mut repos_visited: usize = 0; for repo in &repos { - // Repo-level read gate against THIS row's own rules (KTD2a). + // Repo-level read gate against THIS row's own rules (KTD2a). Deny is a + // VERDICT: this repo would never serve the caller, so skipping it cannot + // hide content from them. let rules: &[crate::db::VisibilityRule] = rules_by_repo .get(&repo.id) .map(Vec::as_slice) @@ -154,23 +185,24 @@ pub async fn get_by_cid( continue; } - // Loop bound (#174 P1-3): once this request has walked its cap of candidate - // repos (each a git subprocess, up to a full-history walk), stop and fall - // through to the opaque 404 rather than serialize an unbounded number under the - // single held admission slot. - if repos_walked >= state.config.ipfs_max_repos_walked { + // Visit ceiling (F2): bound the acquire+probe cost class. Stopping here + // leaves the remaining candidates unproven, so the scan is truncated. + if repos_visited >= state.config.ipfs_max_repo_visits { tracing::warn!( - cap = state.config.ipfs_max_repos_walked, - "/ipfs request hit the per-request repo-walk cap; stopping the scan" + ceiling = state.config.ipfs_max_repo_visits, + "/ipfs request hit the per-request repo-visit ceiling \ + (GITLAWB_IPFS_MAX_REPO_VISITS); stopping the scan without a verdict" ); + taint(&mut truncated_by, "visit-ceiling"); break; } - repos_walked += 1; + repos_visited += 1; // Bound the per-repo acquire under `git_acquire_timeout_secs`: this loop shares // the P1-2 stall vector (a hung Tigris HEAD/GET on one repo would otherwise - // block the whole /ipfs request). On expiry keep the existing fail-closed skip — - // never serve an un-acquired repo; a public copy (if any) still gets its turn. + // block the whole /ipfs request). On expiry keep the fail-closed skip — never + // serve an un-acquired repo; a public copy (if any) still gets its turn — but + // the repo got no verdict, so the skip taints the scan. let acquire_deadline = std::time::Duration::from_secs(state.config.git_acquire_timeout_secs); let repo_path = match tokio::time::timeout( @@ -180,21 +212,28 @@ pub async fn get_by_cid( .await { Ok(Ok(p)) => p, - Ok(Err(_)) => continue, + Ok(Err(e)) => { + tracing::warn!(repo = %repo.name, err = %e, "repo acquire failed during /ipfs scan; skipping repo without a verdict"); + taint(&mut truncated_by, "acquire"); + continue; + } Err(_elapsed) => { - tracing::warn!(repo = %repo.name, "repo acquire timed out during /ipfs walk; skipping repo"); + tracing::warn!(repo = %repo.name, "repo acquire timed out during /ipfs scan; skipping repo without a verdict"); + taint(&mut truncated_by, "acquire"); continue; } }; // Check whether the object exists in this repo before any expensive // reachability walk. This prevents random-CID spray from triggering - // full-history git walks on repos that don't carry the object. + // full-history git walks on repos that don't carry the object. Absent + // (`Ok(None)`) is a VERDICT; a probe that could not run is not. let obj_type = match store::object_type(&repo_path, &sha256_hex) { Ok(Some(t)) => t, Ok(None) => continue, Err(e) => { - tracing::warn!(repo = %repo.name, err = %e, "error checking git object type"); + tracing::warn!(repo = %repo.name, err = %e, "object-type probe failed during /ipfs scan; skipping repo without a verdict"); + taint(&mut truncated_by, "probe"); continue; } }; @@ -205,6 +244,21 @@ pub async fn get_by_cid( let path_scoped = has_path_scoped_rule(rules); if path_scoped && obj_type == "blob" { if !allowed_memo.contains_key(&repo.id) { + // Walk cap (F2), checked at the one site that actually spends a walk: + // on exhaustion skip THIS repo without a verdict and KEEP scanning — + // later candidates may still reach a cheap probe-only verdict (a plain + // public copy serves its 200 with no walk at all). + if repos_walked >= state.config.ipfs_max_repos_walked { + tracing::warn!( + cap = state.config.ipfs_max_repos_walked, + repo = %repo.name, + "/ipfs request hit the per-request walk cap \ + (GITLAWB_IPFS_MAX_REPOS_WALKED); skipping repo without a verdict" + ); + taint(&mut truncated_by, "walk-cap"); + continue; + } + repos_walked += 1; let rp = repo_path.clone(); let r = rules.to_vec(); let is_public = repo.is_public; @@ -229,20 +283,25 @@ pub async fn get_by_cid( .await; // Fail closed on EITHER a task panic (JoinError) or a walk error: // we cannot prove the caller may read here, so skip this repo and - // let a public copy (if any) serve. Never serve on an unproven gate. + // let a public copy (if any) serve. Never serve on an unproven gate + // — and never report absent on one either (no verdict, taint). let set = match walk { Ok(Ok(set)) => set, Ok(Err(e)) => { - tracing::warn!(repo = %repo.name, err = %e, "allowed-blob walk failed; skipping repo"); + tracing::warn!(repo = %repo.name, err = %e, "allowed-blob walk failed during /ipfs scan; skipping repo without a verdict"); + taint(&mut truncated_by, "walk-failure"); continue; } Err(e) => { - tracing::warn!(repo = %repo.name, err = %e, "allowed-blob walk task panicked; skipping repo"); + tracing::warn!(repo = %repo.name, err = %e, "allowed-blob walk task panicked during /ipfs scan; skipping repo without a verdict"); + taint(&mut truncated_by, "walk-failure"); continue; } }; allowed_memo.insert(repo.id.clone(), set); } + // Not in the caller's reachable allowed-set: a VERDICT (deny), the walk + // proved this repo would never serve the blob to this caller. let in_allowed = allowed_memo .get(&repo.id) .is_some_and(|set| set.contains(&sha256_hex)); @@ -251,11 +310,14 @@ pub async fn get_by_cid( } } - // Now that we've passed the gate, read the content. + // Now that we've passed the gate, read the content. A failed read after a + // passed gate is not an absence verdict — the probe just said the object + // exists here — so the skip taints the scan. let content = match store::read_object_content(&repo_path, &sha256_hex, &obj_type) { Ok(c) => c, Err(e) => { - tracing::warn!(repo = %repo.name, err = %e, "error reading git object content"); + tracing::warn!(repo = %repo.name, err = %e, "object content read failed during /ipfs scan; skipping repo without a verdict"); + taint(&mut truncated_by, "read"); continue; } }; @@ -279,7 +341,20 @@ pub async fn get_by_cid( return Ok((StatusCode::OK, headers, content).into_response()); } - // Not found in any repo + // Truncated scan (F2): at least one candidate repo yielded no verdict, so the + // object is not proven absent. A 404 here would misreport existing content, so + // shed retryable instead — Overloaded is the single 503 + Retry-After site in + // error.rs, and the message names the truncation sources so the operator can + // map the shed to the right knob or backend. + if !truncated_by.is_empty() { + return Err(AppError::Overloaded(format!( + "ipfs scan incomplete ({}) for CID {cid_str}; retry shortly", + truncated_by.join("+") + ))); + } + + // Complete scan: every candidate reached a verdict and none served, so the + // object is definitively absent (or denied) for this caller. Err(AppError::RepoNotFound(format!( "no git object found for CID {cid_str}" ))) @@ -353,6 +428,554 @@ mod tests { req } + /// Run real git, asserting success. Shared by the F2 scan-verdict tests. + fn run_git(args: &[&str], cwd: &std::path::Path) { + let out = std::process::Command::new("git") + .args(args) + .current_dir(cwd) + .output() + .expect("git runs"); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + } + + /// Seed a repo row plus a REAL sha256 bare repo at its acquired path holding one + /// committed blob (`src/secret.txt` = `content`). Returns `(repo_id, blob_oid)`. + /// Same recipe as `get_by_cid_walk_permit_held_through_blocking_walk`: the CID + /// digest IS the sha256 object id under `--object-format=sha256`, so the real + /// `cat-file` probe finds the blob. + async fn seed_repo_with_blob( + state: &crate::state::AppState, + tmp: &std::path::Path, + owner: &str, + name: &str, + content: &[u8], + ) -> (String, String) { + state + .db + .upsert_mirror_repo(owner, name, &format!("/unused-{name}"), None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + let bare = state + .repo_store + .acquire(&rec.owner_did, &rec.name) + .await + .unwrap(); + let _ = std::fs::remove_dir_all(&bare); + std::fs::create_dir_all(&bare).unwrap(); + let work = tmp.join(format!("work-{owner}-{name}")); + std::fs::create_dir_all(work.join("src")).unwrap(); + std::fs::write(work.join("src/secret.txt"), content).unwrap(); + run_git( + &["init", "-q", "--object-format=sha256", "-b", "main"], + &work, + ); + run_git(&["config", "user.email", "t@t"], &work); + run_git(&["config", "user.name", "t"], &work); + run_git(&["add", "src/secret.txt"], &work); + run_git(&["commit", "-q", "-m", "seed"], &work); + run_git( + &[ + "clone", + "--bare", + "-q", + work.to_str().unwrap(), + bare.to_str().unwrap(), + ], + tmp, + ); + let out = std::process::Command::new("git") + .args(["rev-parse", "HEAD:src/secret.txt"]) + .current_dir(&work) + .output() + .expect("git rev-parse runs"); + assert!(out.status.success(), "rev-parse failed"); + let oid = String::from_utf8_lossy(&out.stdout).trim().to_string(); + (rec.id, oid) + } + + /// CIDv1(raw, sha2-256) for a sha256 object id, as the handler resolves it. + fn cid_for_oid(oid: &str) -> String { + let oid_bytes = gitlawb_core::cid::sha256_hex_to_bytes(oid).unwrap(); + gitlawb_core::cid::Cid::from_sha256_bytes(&oid_bytes) + .as_str() + .to_string() + } + + /// Fake git for the WALK only (`state.git_bin`): empty refs, `rev-parse` + /// resolves, and each `rev-list` appends one line to `log` and prints nothing — + /// every walked repo yields an EMPTY allowed-set (path-gate deny verdict) and + /// the log's line count == the number of expensive walks run. The probe and the + /// content read shell to the real `git`, so seeded objects must genuinely exist. + #[cfg(unix)] + fn walk_logging_fake_git(dir: &std::path::Path, log: &std::path::Path) -> String { + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + for-each-ref) : ;;\n\ + rev-parse) echo deadbeef ;;\n\ + rev-list) echo walk >> \"{}\" ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + log.display() + ); + let git_path = dir.join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + git_path.to_str().unwrap().to_string() + } + + /// F2 buried-row repro: with more readable repos than `ipfs_max_repos_walked`, + /// existing PUBLIC content past the cap must still serve. The cap counts + /// EXPENSIVE walks only — this request has no path-scoped rules anywhere, so it + /// runs ZERO walks (the fake-git walk log stays empty) and the cap can never cut + /// the scan: the blob buried in the OLDER-updated repo (iterated last under + /// `list_all_repos`' updated_at DESC) serves its 200. Before F2 the cap counted + /// visibility-passing VISITS and broke the loop into the opaque 404 — existing + /// content misreported absent because of unrelated repos. MUTATION (RED): count + /// visits against the cap again (re-add the check+increment at the visibility + /// gate) and the buried row 503s instead of serving. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_buried_public_row_past_walk_cap_still_serves(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + let walk_log = tmp.path().join("walks.log"); + state.git_bin = walk_logging_fake_git(tmp.path(), &walk_log); + // Tighter than the repo count: the old visit-counting cap cut the scan here. + let mut cfg = (*state.config).clone(); + cfg.ipfs_max_repos_walked = 1; + state.config = Arc::new(cfg); + + // Seed the blob-carrying repo FIRST so its updated_at is OLDER: the empty + // repo is iterated first and the blob row sits past the old visit budget. + let (_, oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6f2buried", + "buried", + b"buried row proof\n", + ) + .await; + seed_repo_with_blob( + &state, + tmp.path(), + "z6f2buried", + "fresh", + b"unrelated content\n", + ) + .await; + + let peer: SocketAddr = "203.0.113.60:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&cid_for_oid(&oid), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::OK, + "a public blob in a repo past the walk cap must still serve — the cap \ + counts expensive walks and this scan needs none" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + assert_eq!(&body[..], b"buried row proof\n"); + let walks = std::fs::read_to_string(&walk_log) + .map(|s| s.lines().count()) + .unwrap_or(0); + assert_eq!( + walks, 0, + "a request with no path-scoped rules anywhere must run zero expensive walks" + ); + } + + /// F2 walk-cap skip-and-continue: exhausting `ipfs_max_repos_walked` skips the + /// walk-NEEDING repo without a verdict but keeps the scan alive. Three public + /// repos carry the same blob, newest first: the first (path-scoped) consumes the + /// cap-of-1 walk and denies (empty allowed-set — a verdict); the second + /// (path-scoped) needs a walk the cap forbids and is skipped WITHOUT one (taint); + /// the third is plain public and serves the 200 from a cheap probe — found beats + /// taint, and exactly one expensive walk ran. Before F2 the cap broke the loop at + /// the second repo and the request 404'd despite the public copy. MUTATION (RED): + /// turn the walk-cap skip back into a `break` and the public copy never serves. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_walk_cap_skip_continues_to_later_public_copy(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + let walk_log = tmp.path().join("walks.log"); + state.git_bin = walk_logging_fake_git(tmp.path(), &walk_log); + let mut cfg = (*state.config).clone(); + cfg.ipfs_max_repos_walked = 1; + state.config = Arc::new(cfg); + + // Insert order = oldest first, so iteration (updated_at DESC) is reversed: + // gatedwalk, then gatedskip, then pubcopy. Identical content -> one CID. + let content = b"skip and continue proof\n"; + let (_, oid) = + seed_repo_with_blob(&state, tmp.path(), "z6f2skip", "pubcopy", content).await; + let (skip_id, _) = + seed_repo_with_blob(&state, tmp.path(), "z6f2skip", "gatedskip", content).await; + let (walk_id, _) = + seed_repo_with_blob(&state, tmp.path(), "z6f2skip", "gatedwalk", content).await; + for id in [&walk_id, &skip_id] { + state + .db + .set_visibility_rule( + id, + "src/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkU3IpfsReaderCCCCCCCCCCCCCCCCCCCCCCCC".to_string()], + "z6f2skip", + ) + .await + .unwrap(); + } + + let peer: SocketAddr = "203.0.113.61:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&cid_for_oid(&oid), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::OK, + "the walk-cap skip must continue the scan so the plain public copy serves" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + assert_eq!(&body[..], content.as_slice()); + let walks = std::fs::read_to_string(&walk_log) + .map(|s| s.lines().count()) + .unwrap_or(0); + assert_eq!( + walks, 1, + "cap honored exactly: the first path-scoped repo walks, the second is cut" + ); + } + + /// F2 visit ceiling: `ipfs_max_repo_visits` bounds the acquire+probe cost class + /// (each visit can be a full Tigris archive fetch on a cache miss). Unlike the + /// walk cap there is no cheap way to keep scanning, so exhaustion STOPS the scan + /// — and the stop is a truncation, not an absence: with ceiling 1 the newer + /// empty repo consumes the only visit and the blob-carrying older repo is never + /// probed, so the request sheds a retryable 503 + Retry-After, never a false + /// 404. MUTATION (RED): drop the ceiling check and the blob serves (200); drop + /// only the taint on the break and the 503 decays to a 404. + #[sqlx::test] + async fn get_by_cid_visit_ceiling_stops_scan_with_503(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + let mut cfg = (*state.config).clone(); + cfg.ipfs_max_repo_visits = 1; + state.config = Arc::new(cfg); + + // Blob repo first (older, iterated second); empty repo second (newer, + // consumes the single visit). + let (_, oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6f2visit", + "buried", + b"visit ceiling proof\n", + ) + .await; + seed_repo_with_blob(&state, tmp.path(), "z6f2visit", "fresh", b"unrelated\n").await; + + let peer: SocketAddr = "203.0.113.62:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&cid_for_oid(&oid), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a visit-ceiling truncation must shed a retryable 503, not report absent" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the truncation 503 must carry Retry-After" + ); + } + + /// F2 negative arm: a COMPLETE scan that finds nothing keeps its definitive 404 + /// — the truncation 503 must never fire when every candidate reached a verdict. + /// Two public repos both probe clean (the requested CID is nowhere), no rules, + /// no cap or ceiling hit: 404 with no Retry-After. MUTATION (RED): taint the + /// scan unconditionally and this decays into a 503. + #[sqlx::test] + async fn get_by_cid_complete_scan_keeps_definitive_404(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + seed_repo_with_blob(&state, tmp.path(), "z6f2clean", "one", b"content one\n").await; + seed_repo_with_blob(&state, tmp.path(), "z6f2clean", "two", b"content two\n").await; + + // valid_cid() is the "hello" blob — present in neither repo. + let peer: SocketAddr = "203.0.113.63:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&valid_cid(), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::NOT_FOUND, + "a complete clean scan is a definitive absence — 404, never the 503 shed" + ); + assert!( + resp.headers().get("retry-after").is_none(), + "a definitive 404 must not advertise a retry" + ); + } + + /// F2 acquire taint: a repo row with NO local copy over a Tigris backend that + /// stalls (non-routable endpoint — the connect just hangs) hits the 1s acquire + /// timeout at the read-acquire site. The skip carries no verdict, so the scan is + /// truncated: retryable 503 + Retry-After, never the old silent-skip 404. + /// MUTATION (RED): drop the taint on the acquire-timeout arm and this decays to + /// a 404. + #[sqlx::test] + async fn get_by_cid_acquire_timeout_taints_scan_to_503(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + // Endpoint-pinned test client (no AWS_* env reads — env is racy under a + // parallel test run); 10.255.255.1 is non-routable so the HEAD hangs. + let tigris = crate::git::tigris::TigrisClient::for_testing_with_endpoint( + "test-bucket", + "http://10.255.255.1:9000", + ) + .await; + state.repo_store = crate::git::repo_store::RepoStore::new(repos_dir, Some(tigris), pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + let mut cfg = (*state.config).clone(); + cfg.git_acquire_timeout_secs = 1; + state.config = Arc::new(cfg); + + // Row exists in the DB but has no local copy, so the read acquire must + // consult Tigris (local-miss path) and stall until the timeout. + state + .db + .upsert_mirror_repo("z6f2acq", "ghost", "/unused-ghost", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.64:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&valid_cid(), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an acquire timeout leaves the repo unproven — the scan must shed 503, not 404" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the truncation 503 must carry Retry-After" + ); + } + + /// F2 probe taint: a repo row whose local dir does not exist (no Tigris) — + /// `RepoStore::acquire` returns the path anyway (local passthrough), and the + /// `cat-file -t` probe cannot even spawn (missing working dir), so + /// `object_type` is Err. That is not an absence verdict, so the scan is + /// truncated: 503, never 404. A second, real repo probes clean (absent verdict) + /// — the one bad row is what taints. NOTE: the probe shells to the real `git` + /// (not `state.git_bin`) and maps a NONZERO cat-file exit to `Ok(None)` (an + /// absent verdict), so a fake git exiting nonzero cannot reach this arm; only a + /// spawn failure is Err, hence the missing-dir recipe. MUTATION (RED): drop the + /// taint on the probe-error arm and this decays to a 404. + #[sqlx::test] + async fn get_by_cid_probe_error_taints_scan_to_503(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + // Older row: a real repo that probes clean. Newer row: no dir on disk. + seed_repo_with_blob(&state, tmp.path(), "z6f2probe", "real", b"probe clean\n").await; + state + .db + .upsert_mirror_repo("z6f2probe", "ghost", "/unused-ghost", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.65:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&valid_cid(), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a failed probe leaves the repo unproven — the scan must shed 503, not 404" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the truncation 503 must carry Retry-After" + ); + } + + /// F2 read taint: the gate passes (the probe reads the truncated loose object's + /// intact "blob 64" header) but the content read fails (`cat-file blob` dies on + /// the deflate stream cut mid-content) — the probe just said the object EXISTS + /// here, so the failed read is no absence verdict: 503, never 404. The loose + /// object is hand-rolled: zlib header + one stored deflate block declaring 72 + /// bytes ("blob 64\0" + 64), truncated after the header NUL + 4 content bytes, + /// no adler trailer. MUTATION (RED): drop the taint on the read-error arm and + /// this decays to a 404. + #[sqlx::test] + async fn get_by_cid_read_error_taints_scan_to_503(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + state + .db + .upsert_mirror_repo("z6f2read", "corrupt", "/unused-corrupt", None, false) + .await + .unwrap(); + let rec = state + .db + .get_repo("z6f2read", "corrupt") + .await + .unwrap() + .unwrap(); + let bare = state + .repo_store + .acquire(&rec.owner_did, &rec.name) + .await + .unwrap(); + std::fs::create_dir_all(&bare).unwrap(); + run_git(&["init", "-q", "--bare", "--object-format=sha256"], &bare); + // Hand-rolled truncated loose object (dangling is fine: no path-scoped rules, + // so the "/" gate is the whole story and the read follows the probe). + let oid = "6bf5122f344554c53bde2ebb8cd2b7e3d1600ad631c385a5d7cce23c7785459c"; + let mut corrupt: Vec = vec![0x78, 0x01, 0x01, 0x48, 0x00, 0xb7, 0xff]; + corrupt.extend_from_slice(b"blob 64\0AAAA"); + let obj_dir = bare.join("objects").join(&oid[..2]); + std::fs::create_dir_all(&obj_dir).unwrap(); + std::fs::write(obj_dir.join(&oid[2..]), &corrupt).unwrap(); + // Preconditions: the probe classifies it as a blob, the full read fails — + // otherwise the test would pass vacuously via some other arm. + assert_eq!( + crate::git::store::object_type(&bare, oid) + .unwrap() + .as_deref(), + Some("blob"), + "the truncated loose object's header must still probe as a blob" + ); + assert!( + crate::git::store::read_object_content(&bare, oid, "blob").is_err(), + "the truncated loose object's content read must fail" + ); + + let peer: SocketAddr = "203.0.113.66:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&cid_for_oid(oid), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a failed read after a passed gate leaves the repo unproven — 503, not 404" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the truncation 503 must carry Retry-After" + ); + } + + /// F2 denied-is-a-verdict: repos that DENY the caller at the visibility gate + /// are settled, not skipped — an all-denied scan is COMPLETE: 404, zero visits. + /// The private rows deliberately have no local dirs: if the deny didn't + /// short-circuit before the visit, the missing-dir probe would taint the scan + /// into a 503, which the 404 assertion rules out — so the 404 also proves zero + /// acquires, probes, or walks ran for denied rows. + #[sqlx::test] + async fn get_by_cid_all_denied_is_complete_scan_404(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + for name in ["priv-a", "priv-b"] { + let now = chrono::Utc::now(); + state + .db + .create_repo(&crate::db::RepoRecord { + id: uuid::Uuid::new_v4().to_string(), + name: name.to_string(), + owner_did: "did:key:z6MkF2DenyOwnerAAAAAAAAAAAAAAAAAAAAAAAAA".to_string(), + description: None, + is_public: false, + default_branch: "main".to_string(), + created_at: now, + updated_at: now, + disk_path: format!("/nonexistent/{name}"), + forked_from: None, + machine_id: None, + }) + .await + .unwrap(); + } + + let peer: SocketAddr = "203.0.113.67:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&valid_cid(), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::NOT_FOUND, + "an anonymous caller denied by every repo gets a complete-scan 404 — a deny \ + is a verdict and must not visit, taint, or 503" + ); + } + /// Shed at capacity: an exhausted `git_ipfs_walk_semaphore` sheds a `/ipfs/{cid}` /// request with 503 BEFORE any DB/git walk (the acquire is the first thing after CID /// validation), so a lazy DB-free state suffices — exactly like the served-git shed @@ -736,13 +1359,16 @@ mod tests { ); } - /// Loop bound (cap N): one `/ipfs/{cid}` request against a CID present in many repos - /// must not serialize an unbounded number of full-history walks. With - /// `ipfs_max_repos_walked = 1` and TWO public, path-scoped repos both carrying the - /// blob at the requested CID, the handler walks only the FIRST candidate then stops - /// (the second is cut by the cap), so the fake git's `rev-list` (one per walk) runs - /// exactly once. MUTATION (RED): remove the `repos_walked >= cap` break and both - /// repos are walked (count 2). + /// Loop bound (cap N) + F2 truncation verdict: one `/ipfs/{cid}` request against a + /// CID present in many path-scoped repos must not serialize an unbounded number of + /// full-history walks — and cutting a candidate WITHOUT a verdict must not report + /// the object absent. With `ipfs_max_repos_walked = 1` and TWO public, path-scoped + /// repos both carrying the blob, the first candidate is walked (empty allowed-set → + /// a deny VERDICT) and the second is cut by the cap (no verdict), so the fake git's + /// `rev-list` runs exactly once and the request sheds a retryable 503 + Retry-After + /// — never the old false 404 (the blob genuinely sits in the second repo). + /// MUTATION (RED): remove the `repos_walked >= cap` skip and both repos are walked + /// (count 2); drop the truncation taint on the skip and the 503 decays to a 404. #[cfg(unix)] #[sqlx::test] async fn get_by_cid_caps_repos_walked_per_request(pool: sqlx::PgPool) { @@ -871,9 +1497,21 @@ mod tests { .unwrap(); req.extensions_mut().insert(ConnectInfo(peer)); let resp = ipfs_router(state).oneshot(req).await.unwrap(); - // The empty allowed-set path-gates both repos to a `continue`, so a 404; the - // point is HOW MANY walks ran to get there. - assert_eq!(resp.status(), StatusCode::NOT_FOUND); + // The first repo's walk yields the empty allowed-set (deny verdict); the second + // repo NEEDS a walk the cap forbids, so the scan is truncated without a verdict + // on it: retryable 503, never a false 404 for the blob it genuinely carries. + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a walk-cap truncation must shed a retryable 503, not report the object absent" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the truncation 503 must carry Retry-After" + ); let walks = std::fs::read_to_string(&walk_log) .map(|s| s.lines().count()) diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index b3667024..3e0a8e50 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -372,13 +372,17 @@ pub struct Config { )] pub ipfs_walk_per_source: usize, - /// Upper bound on the number of candidate repos a single `/ipfs/{cid}` request - /// will walk before giving up (returning the opaque 404). The handler already - /// short-circuits the moment it serves the object, but a CID that is present in - /// (or path-gated out of) many repos could otherwise serialize one full-history - /// walk per repo inside a single held admission slot. Capping the count bounds - /// the worst-case work one request can pin its slot with. Must be between 1 and - /// 1_048_576. Default: 64. + /// Upper bound on the number of EXPENSIVE visibility walks + /// (`allowed_blob_set_for_caller_bounded`, a full-history git walk in a + /// blocking thread) a single `/ipfs/{cid}` request may run. Only a blob in a + /// path-scoped repo costs a walk, so the cap counts exactly those candidates + /// — cheap probe-only visits are bounded by `ipfs_max_repo_visits` instead + /// (counting them here would starve a plain public copy past the cap out of + /// its 200). On exhaustion the walk-needing repo is skipped WITHOUT a verdict + /// and the scan continues; if the request then finds the object nowhere it + /// sheds a retryable 503 + Retry-After rather than misreport existing content + /// absent with a 404. The handler still short-circuits the moment it serves. + /// Must be between 1 and 1_048_576. Default: 64. #[arg( long, env = "GITLAWB_IPFS_MAX_REPOS_WALKED", @@ -387,6 +391,24 @@ pub struct Config { )] pub ipfs_max_repos_walked: usize, + /// Ceiling on the number of repos a single `/ipfs/{cid}` request may VISIT — + /// pass the repo-level visibility gate into the acquire + `cat-file` probe. + /// Each visit costs a `RepoStore::acquire` (on a Tigris cache miss that is a + /// full repo-archive download from object storage, so the worst-case + /// object-store fetch count for one request equals this ceiling) plus a git + /// probe subprocess. On exhaustion the scan STOPS — unlike + /// `ipfs_max_repos_walked`, which skips just the walk-needing repo, there is + /// no cheaper way to keep scanning — and the request sheds a retryable 503 + + /// Retry-After rather than a false 404. Must be between 1 and 1_048_576. + /// Default: 1024. + #[arg( + long, + env = "GITLAWB_IPFS_MAX_REPO_VISITS", + default_value_t = 1024, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub ipfs_max_repo_visits: usize, + /// Per-client-IP rate limit for `GET /ipfs/{cid}`, in requests per hour. The /// route is publicly reachable (`optional_signature`) and each request can drive /// a full-history git walk, so it carries a per-IP flood brake in addition to the @@ -548,6 +570,24 @@ mod tests { ); } + #[test] + fn ipfs_max_repo_visits_defaults_and_rejects_out_of_range() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).ipfs_max_repo_visits, + 1024 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--ipfs-max-repo-visits", "8"]) + .ipfs_max_repo_visits, + 8 + ); + // 0 would visit no repos (serve nothing); clap must reject it. + assert!(Config::try_parse_from(["gitlawb-node", "--ipfs-max-repo-visits", "0"]).is_err()); + assert!( + Config::try_parse_from(["gitlawb-node", "--ipfs-max-repo-visits", "1048577"]).is_err() + ); + } + #[test] fn max_concurrent_reads_per_caller_defaults_and_rejects_out_of_range() { assert_eq!( diff --git a/crates/gitlawb-node/src/git/tigris.rs b/crates/gitlawb-node/src/git/tigris.rs index ad26ddc5..cf7abfd5 100644 --- a/crates/gitlawb-node/src/git/tigris.rs +++ b/crates/gitlawb-node/src/git/tigris.rs @@ -31,6 +31,25 @@ impl TigrisClient { }) } + /// Test-only constructor with an explicit S3 endpoint, region, and static + /// credentials — no env-var reads, so parallel tests cannot race each other's + /// `AWS_*` environment the way the env-based `new` would. Lets a test point + /// the client at a non-routable endpoint to exercise acquire-stall paths. + #[cfg(test)] + pub(crate) async fn for_testing_with_endpoint(bucket: &str, endpoint_url: &str) -> Self { + let creds = aws_sdk_s3::config::Credentials::new("test", "test", None, None, "test"); + let config = aws_config::defaults(aws_config::BehaviorVersion::latest()) + .endpoint_url(endpoint_url) + .region(aws_config::Region::new("auto")) + .credentials_provider(creds) + .load() + .await; + Self { + s3: S3Client::new(&config), + bucket: bucket.to_string(), + } + } + /// S3 key for a given repo: `repos/v1/{owner_slug}/{repo_name}.tar.zst` fn repo_key(owner_slug: &str, repo_name: &str) -> String { format!("repos/v1/{owner_slug}/{repo_name}.tar.zst") diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 92f41a8d..df095b44 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -2409,7 +2409,10 @@ mod tests { /// the handler skips the whole repo rather than serving. Asserts no leak of the /// withheld blob AND that even the *public* blob in that repo is withheld — the /// latter distinguishes fail-closed-skip from normal per-blob withholding and - /// would serve 200 if the error arm wrongly proceeded. + /// would serve 200 if the error arm wrongly proceeded. The skip carries no + /// VERDICT (F2), so the response is the retryable truncation 503, not a 404 + /// claiming the object is absent — never-serve-unproven and never-404-unproven + /// hold together. #[sqlx::test] async fn ipfs_cid_walk_error_fails_closed(pool: PgPool) { use crate::db::VisibilityMode; @@ -2454,7 +2457,8 @@ mod tests { .await .expect("deny rule"); - // Withheld secret CID under a walk error → 404, no leak. + // Withheld secret CID under a walk error → the repo is skipped without a + // verdict, so the scan is truncated (503), and nothing leaks. let (st, body) = cid_parts( cid_router(&state) .oneshot(cid_anon(&secret_cid)) @@ -2464,17 +2468,17 @@ mod tests { .await; assert_eq!( st, - StatusCode::NOT_FOUND, - "walk error must not serve the withheld blob" + StatusCode::SERVICE_UNAVAILABLE, + "walk error must not serve the withheld blob — the unproven skip sheds 503" ); assert!( !body.contains("TOP SECRET"), - "walk-error 404 must not leak the secret" + "walk-error 503 must not leak the secret" ); - // The PUBLIC blob in the same repo is also 404: the walk error fails closed - // by skipping the whole repo, not by serving. Without the fail-closed arm - // this would serve 200, so this assertion is the load-bearing discriminator. + // The PUBLIC blob in the same repo is also not served: the walk error fails + // closed by skipping the whole repo. Without the fail-closed arm this would + // serve 200, so this assertion is the load-bearing discriminator. let (st, _) = cid_parts( cid_router(&state) .oneshot(cid_anon(&public_cid)) @@ -2484,8 +2488,9 @@ mod tests { .await; assert_eq!( st, - StatusCode::NOT_FOUND, - "walk error fails closed: repo skipped, even the public blob is not served" + StatusCode::SERVICE_UNAVAILABLE, + "walk error fails closed: repo skipped without a verdict, even the public \ + blob is not served and the scan sheds 503" ); } From 39d30dc3c11d08b1c816b38c108eb56f8111d11d Mon Sep 17 00:00:00 2001 From: t Date: Fri, 17 Jul 2026 18:03:49 -0500 Subject: [PATCH 40/58] fix(node): bound an admitted /ipfs request's total lifetime with a shared budget (#174) GITLAWB_IPFS_REQUEST_BUDGET_SECS (default 600) is captured as one deadline at handler entry: every acquire and expensive walk is clamped to the remaining budget, every stage checks remaining before starting, and exhaustion taints the scan into the retryable 503. A walk in flight is never aborted at the tokio layer (that would free the walk permit while the blocking thread still runs); its clamped git deadline plus group teardown ends it. Docs state the residual overshoot: the in-flight stage's kill/reap slack plus the unclamped probe subprocesses. --- .env.example | 7 + README.md | 1 + crates/gitlawb-node/src/api/ipfs.rs | 500 +++++++++++++++++++++++++++- crates/gitlawb-node/src/config.rs | 44 +++ 4 files changed, 546 insertions(+), 6 deletions(-) diff --git a/.env.example b/.env.example index 144108ba..94685069 100644 --- a/.env.example +++ b/.env.example @@ -179,6 +179,13 @@ GITLAWB_IPFS_MAX_REPOS_WALKED=64 # cat-file probe. On exhaustion the scan stops and sheds a retryable 503. # Default 1024. GITLAWB_IPFS_MAX_REPO_VISITS=1024 +# Absolute wall-clock budget for one admitted /ipfs request's acquire+walk +# lifetime (all stages of the whole scan). Bounds how LONG one request may hold +# its walk slot: per-stage timeouts are clamped to the remaining budget and no +# stage starts once it is exhausted (the scan then sheds a retryable 503). +# Residual overshoot: the in-flight clamped stage's kill/reap slack plus one +# unclamped cat-file probe subprocess. Default 600. +GITLAWB_IPFS_REQUEST_BUDGET_SECS=600 # Max /ipfs/{cid} requests per client IP per hour (route flood brake, distinct # from the concurrency caps above). 0 disables. Default 600. GITLAWB_IPFS_RATE_LIMIT=600 diff --git a/README.md b/README.md index f3fa361b..bc338339 100644 --- a/README.md +++ b/README.md @@ -349,6 +349,7 @@ Important node settings: | `GITLAWB_IPFS_WALK_PER_SOURCE` | Max concurrent `/ipfs` walks a single source IP may hold. Default 4. | | `GITLAWB_IPFS_MAX_REPOS_WALKED` | Max expensive path-scope visibility walks per `/ipfs/{cid}` request; over-cap repos are skipped and the scan continues, shedding a retryable 503 (not a false 404) if the object is then found nowhere. Default 64. | | `GITLAWB_IPFS_MAX_REPO_VISITS` | Ceiling on repos one `/ipfs/{cid}` request may visit (acquire + probe) past the visibility gate — also the worst-case per-request Tigris fetch count. On exhaustion the scan stops with a retryable 503. Default 1024. | +| `GITLAWB_IPFS_REQUEST_BUDGET_SECS` | Absolute wall-clock budget for one admitted `/ipfs/{cid}` request's acquire+walk lifetime. Per-stage timeouts are clamped to the remaining budget and no stage starts once it is exhausted; the scan then stops with a retryable 503. Overshoot is bounded by the in-flight clamped stage's kill/reap slack plus one unclamped `cat-file` probe. Default 600. | | `GITLAWB_IPFS_RATE_LIMIT` | Max `/ipfs/{cid}` requests per client IP per hour (route flood brake). 0 disables. Default 600. | | `GITLAWB_TIGRIS_BUCKET` | Optional S3/Tigris shared repo storage bucket. | | `GITLAWB_PINATA_JWT` | Optional Pinata/IPFS warm-storage pinning. | diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index d4a6c7a8..28b9cc67 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -55,11 +55,19 @@ use crate::visibility::{visibility_check, Decision}; /// repo reached a VERDICT — visibility deny, probe-says-absent, walk-gate deny, /// or served. A candidate skipped WITHOUT a verdict (acquire failure/timeout, /// probe error, walk failure/panic, content-read error, or truncation by -/// `ipfs_max_repos_walked` / `ipfs_max_repo_visits`) taints the scan, and a +/// `ipfs_max_repos_walked` / `ipfs_max_repo_visits` / +/// `ipfs_request_budget_secs`) taints the scan, and a /// tainted scan that found nothing sheds a retryable 503 + Retry-After naming /// the truncation sources — existing content is never misreported absent /// because of unrelated repos or transient faults. /// +/// Request budget (F3): one absolute clock (`ipfs_request_budget_secs`) spans +/// the whole admitted request. No stage (acquire, probe, walk, content read) +/// starts once it is exhausted, and the acquire wait and walk deadline are +/// clamped to the remainder, so an admitted request cannot hold its scarce walk +/// slot past the budget plus the in-flight stage's kill/reap slack and one +/// unclamped probe subprocess. +/// /// Scope: this closes the direct unauthenticated scan, including the dangling /// case. A stale-public mirror row still serves withheld content (tracked /// separately, #124). @@ -90,6 +98,19 @@ pub async fn get_by_cid( let caller = auth.as_ref().map(|e| e.0 .0.as_str()); let caller_owned = caller.map(|c| c.to_string()); + // One absolute budget bounds this request's whole acquire+walk lifetime (F3), + // captured before admission so the clock covers everything the walk permit + // holds. Each stage below (acquire, probe, walk, read) starts only while + // budget remains, and the acquire wait + walk deadline run clamped to the + // remainder, so an admitted request cannot hold its scarce walk slot for + // hours by drawing a fresh per-stage timeout every iteration. The budget + // NEVER aborts a running spawn_blocking walk: the clamped git deadline + // inside the walk is what ends it (a tokio timeout around the walk future + // would free the walk permit while the blocking thread still runs, the + // exact hole the held permit closes). + let request_deadline = std::time::Instant::now() + + std::time::Duration::from_secs(state.config.ipfs_request_budget_secs); + // Bounded walk admission (#174 P1-3), taken before any DB/git work so a flood sheds // cheaply. The per-repo `spawn_blocking` walk below is a full-history git walk with // no served-git admission of its own; a permissionless caller could otherwise fan @@ -161,6 +182,16 @@ pub async fn get_by_cid( } } + // Remaining request budget (F3), or None once exhausted. A stage is never + // started with zero remaining; the probe and read subprocesses carry no + // internal duration clamp, so the check before them is their entire bound + // (documented overshoot: at most one unclamped cat-file plus the in-flight + // clamped stage's kill/reap slack). + fn budget_remaining(deadline: std::time::Instant) -> Option { + let left = deadline.saturating_duration_since(std::time::Instant::now()); + (!left.is_zero()).then_some(left) + } + // Cap on EXPENSIVE walks only (F2): counts the repos that actually require the // full-history `allowed_blob_set_for_caller_bounded` walk (a path-scoped blob), // checked immediately before the spawn_blocking below. Cheap probe-only visits @@ -185,6 +216,21 @@ pub async fn get_by_cid( continue; } + // Budget gate for the acquire stage (F3): once the request budget is + // exhausted the scan STOPS, leaving this and every later candidate + // unproven (taint, never a false 404). Checked ahead of the visit + // bookkeeping so an unstarted acquire is not counted as a visit. + let Some(budget_left) = budget_remaining(request_deadline) else { + tracing::warn!( + repo = %repo.name, + budget_secs = state.config.ipfs_request_budget_secs, + "/ipfs request budget exhausted before the repo acquire \ + (GITLAWB_IPFS_REQUEST_BUDGET_SECS); stopping the scan without a verdict" + ); + taint(&mut truncated_by, "budget"); + break; + }; + // Visit ceiling (F2): bound the acquire+probe cost class. Stopping here // leaves the remaining candidates unproven, so the scan is truncated. if repos_visited >= state.config.ipfs_max_repo_visits { @@ -202,9 +248,13 @@ pub async fn get_by_cid( // the P1-2 stall vector (a hung Tigris HEAD/GET on one repo would otherwise // block the whole /ipfs request). On expiry keep the fail-closed skip — never // serve an un-acquired repo; a public copy (if any) still gets its turn — but - // the repo got no verdict, so the skip taints the scan. - let acquire_deadline = - std::time::Duration::from_secs(state.config.git_acquire_timeout_secs); + // the repo got no verdict, so the skip taints the scan. Clamped to the + // remaining request budget (F3) so per-repo acquires cannot each draw a + // fresh full timeout past it. + let acquire_deadline = std::cmp::min( + std::time::Duration::from_secs(state.config.git_acquire_timeout_secs), + budget_left, + ); let repo_path = match tokio::time::timeout( acquire_deadline, state.repo_store.acquire(&repo.owner_did, &repo.name), @@ -224,6 +274,18 @@ pub async fn get_by_cid( } }; + // Budget gate for the probe stage (F3). The probe subprocess has no + // internal duration clamp, so this pre-start check is its entire bound. + if budget_remaining(request_deadline).is_none() { + tracing::warn!( + repo = %repo.name, + "/ipfs request budget exhausted before the object-type probe; \ + stopping the scan without a verdict" + ); + taint(&mut truncated_by, "budget"); + break; + } + // Check whether the object exists in this repo before any expensive // reachability walk. This prevents random-CID spray from triggering // full-history git walks on repos that don't carry the object. Absent @@ -244,6 +306,21 @@ pub async fn get_by_cid( let path_scoped = has_path_scoped_rule(rules); if path_scoped && obj_type == "blob" { if !allowed_memo.contains_key(&repo.id) { + // Budget gate for the walk stage (F3): a walk is never STARTED + // with zero remaining (probed-present is not a serve), and a + // started walk runs its git children under a deadline clamped + // to the remainder (the min below), so a walk can never + // complete past the budget. + let Some(budget_left) = budget_remaining(request_deadline) else { + tracing::warn!( + repo = %repo.name, + budget_secs = state.config.ipfs_request_budget_secs, + "/ipfs request budget exhausted before the visibility walk \ + (GITLAWB_IPFS_REQUEST_BUDGET_SECS); stopping the scan without a verdict" + ); + taint(&mut truncated_by, "budget"); + break; + }; // Walk cap (F2), checked at the one site that actually spends a walk: // on exhaustion skip THIS repo without a verdict and KEEP scanning — // later candidates may still reach a cheap probe-only verdict (a plain @@ -265,8 +342,10 @@ pub async fn get_by_cid( let owner = repo.owner_did.clone(); let caller_for_walk = caller_owned.clone(); let git_bin = state.git_bin.clone(); - let walk_timeout = - std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let walk_timeout = std::cmp::min( + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + budget_left, + ); // Full-history walk shells out to git — keep it off the async runtime, // bounded and reaped like the served-git ops (#174). let walk = tokio::task::spawn_blocking(move || { @@ -310,6 +389,21 @@ pub async fn get_by_cid( } } + // Budget gate for the content-read stage (F3): the read subprocess is + // unclamped, so it never starts past the budget. Tainting instead of + // serving keeps the terminal arm honest and the stop unconditional; the + // retryable 503 tells the caller to come back rather than letting an + // over-budget request keep spending. + if budget_remaining(request_deadline).is_none() { + tracing::warn!( + repo = %repo.name, + "/ipfs request budget exhausted before the content read; \ + stopping the scan without a verdict" + ); + taint(&mut truncated_by, "budget"); + break; + } + // Now that we've passed the gate, read the content. A failed read after a // passed gate is not an absence verdict — the probe just said the object // exists here — so the skip taints the scan. @@ -976,6 +1070,400 @@ mod tests { ); } + /// F3 budget expiry mid-loop: one absolute request budget + /// (`ipfs_request_budget_secs`) bounds the whole admitted scan; per-repo + /// stages may not each draw a fresh timeout past it. Budget 1s, per-iteration + /// acquire timeout 2s; the NEWER row is a Tigris-backed ghost (no local copy, + /// non-routable endpoint) whose acquire stalls, the OLDER row is a plain + /// public repo carrying the blob. The ghost's acquire runs clamped to the ~1s + /// remainder and times out; at the next repo the budget gate sees zero + /// remaining, taints "budget", and STOPS the scan, so the blob repo is never + /// visited (a visit would probe the healthy public copy and serve 200, which + /// the 503 assertion rules out) and the shed names the budget. Without the + /// budget the acquire would time out at its own 2s, the scan would continue, + /// and the buried blob would serve 200 (the recorded RED). MUTATION (RED): + /// remove the `request_deadline` capture (or make the remaining budget + /// infinite) and this serves 200 again. + #[sqlx::test] + async fn get_by_cid_request_budget_expiry_stops_scan_with_503(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // Seed the blob repo through a LOCAL-ONLY store first, so seeding never + // consults the (deliberately unreachable) Tigris endpoint. + state.repo_store = + crate::git::repo_store::RepoStore::for_testing(repos_dir.clone(), pool.clone()); + let (_, oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6f3budget", + "buried", + b"budget expiry proof\n", + ) + .await; + // Swap in a Tigris-backed store over the SAME repos_dir (the seeded bare + // repo stays a fast local hit) and add a NEWER ghost row with no local + // copy: its acquire consults the non-routable endpoint and stalls + // (endpoint-pinned test client, no AWS_* env reads; 10.255.255.1 hangs). + let tigris = crate::git::tigris::TigrisClient::for_testing_with_endpoint( + "test-bucket", + "http://10.255.255.1:9000", + ) + .await; + state.repo_store = crate::git::repo_store::RepoStore::new(repos_dir, Some(tigris), pool); + state + .db + .upsert_mirror_repo("z6f3budget", "ghost", "/unused-ghost", None, false) + .await + .unwrap(); + let mut cfg = (*state.config).clone(); + cfg.ipfs_request_budget_secs = 1; + cfg.git_acquire_timeout_secs = 2; + state.config = Arc::new(cfg); + + let peer: SocketAddr = "203.0.113.70:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&cid_for_oid(&oid), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an exhausted request budget must stop the scan with a retryable 503; \ + scanning on into the later public blob repo would have served 200" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the budget-truncation 503 must carry Retry-After" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + let body = String::from_utf8_lossy(&body); + assert!( + body.contains("budget"), + "the truncation body must name the budget taint so the operator can \ + map the shed to GITLAWB_IPFS_REQUEST_BUDGET_SECS; got: {body}" + ); + } + + /// F3 clamped walk at expiry: a walk that starts with little budget left runs + /// its git children under `min(git_service_timeout_secs, remaining)`, so the + /// clamp (not any tokio-level abort) is what ends it and a walk can never + /// complete past the budget. Budget 2s, service timeout at its 600s default, + /// fake walk git that sleeps 8s: the walk STARTS (pid file), the walk permit + /// stays held while the blocking walk runs (`available_permits == 0`), the + /// clamped deadline SIGTERM/SIGKILLs the child group at ~2s remaining (the + /// response lands after the ~1s watchdog grace, far before the 8s sleep, and + /// the recorded pid is already dead: a tokio abort would have left it + /// running), the log shows the walk started but never completed, and the + /// request sheds the terminal budget-truncated 503 without ever reaching the + /// OLDER public copy of the same blob (which would have served 200). After + /// the response the permit is free: the spawn_blocking closure genuinely + /// returned. MUTATION (RED): drop the `min` clamp on `walk_timeout` and the + /// walk runs its full 8s sleep (elapsed and log-completion assertions fail). + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_budget_clamps_walk_deadline_and_holds_permit(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let walk_log = tmp.path().join("walks.log"); + let revlist_pid = tmp.path().join("revlist.pid"); + // Fake git for the WALK only: `rev-list` records its pid and a start + // marker, sleeps far past the budget, then records a done marker. Under + // the clamped walk deadline the whole process group is torn down mid + // sleep, so "done" never appears. The 8s sleep also bounds a RED run. + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + for-each-ref) : ;;\n\ + rev-parse) echo deadbeef ;;\n\ + rev-list) echo $$ > \"{pid}\"; echo start >> \"{log}\"; sleep 8; echo done >> \"{log}\" ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + pid = revlist_pid.display(), + log = walk_log.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.git_bin = git_path.to_str().unwrap().to_string(); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // Global walk pool of 1 so the held permit is observable; per-source cap + // permissive so only the global pool matters. + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(1)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + let mut cfg = (*state.config).clone(); + // The budget is the ONLY thing that can end this walk early: the service + // timeout stays at its generous 600s default. + cfg.ipfs_request_budget_secs = 2; + state.config = Arc::new(cfg); + + // Older row: a plain public copy of the same blob, which must never be + // reached. Newer row: path-scoped, so its blob costs the clamped walk. + let content = b"budget walk clamp proof\n"; + let (_, oid) = + seed_repo_with_blob(&state, tmp.path(), "z6f3clamp", "pubcopy", content).await; + let (walk_id, _) = + seed_repo_with_blob(&state, tmp.path(), "z6f3clamp", "gated", content).await; + state + .db + .set_visibility_rule( + &walk_id, + "src/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkU3IpfsReaderDDDDDDDDDDDDDDDDDDDDDDDD".to_string()], + "z6f3clamp", + ) + .await + .unwrap(); + + let sem = state.git_ipfs_walk_semaphore.clone(); + let router = ipfs_router(state); + let started = std::time::Instant::now(); + let peer: SocketAddr = "203.0.113.71:5000".parse().unwrap(); + let mut fut = Box::pin(router.oneshot(get_cid(&cid_for_oid(&oid), Some(peer)))); + + // Drive until the fake git's rev-list records its pid: the walk is now in + // the blocking pool and the request future is `.await`ing its join. Stop + // polling the instant the future completes (re-polling would panic). + let mut walk_pid: Option = None; + let mut early = None; + for _ in 0..500 { + let done = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + if let Some(p) = std::fs::read_to_string(&revlist_pid) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + walk_pid = Some(p); + break; + } + if let Ok(resp) = done { + early = Some(resp.map(|r| r.status())); + break; + } + } + let pid = walk_pid.unwrap_or_else(|| { + panic!( + "the budget-clamped walk must have STARTED (nonzero remaining); early: {early:?}" + ) + }); + // Reap the sleeping child on drop so a RED run leaks no orphan. + struct ReapOnDrop(i32); + impl Drop for ReapOnDrop { + fn drop(&mut self) { + unsafe { + libc::kill(self.0, libc::SIGKILL); + } + } + } + let _cleanup = ReapOnDrop(pid); + + // While the blocking walk runs the permit is HELD: the budget never frees + // a slot whose blocking thread is still burning. + assert_eq!( + sem.available_permits(), + 0, + "the walk permit must stay held while the budget-clamped walk runs" + ); + + let resp = tokio::time::timeout(std::time::Duration::from_secs(20), &mut fut) + .await + .expect("the clamped walk deadline must end the request; it never hung") + .unwrap(); + let elapsed = started.elapsed(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a budget-clamped walk that could not finish leaves no verdict: 503, not 404/200" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the truncation 503 must carry Retry-After" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + let body = String::from_utf8_lossy(&body); + assert!( + body.contains("budget"), + "the terminal shed must name the budget taint; got: {body}" + ); + // Deadline-killed at ~remaining, not run to completion: the response + // lands at ~budget + the watchdog's kill/reap slack, well before the 8s + // sleep could have finished. + assert!( + elapsed < std::time::Duration::from_secs(7), + "the clamped git deadline must end the walk at ~remaining; got {elapsed:?}" + ); + // The child group is already dead AT response time: the clamp killed it. + // (A tokio-level abort of the walk future would have answered while the + // blocking thread and its child still ran.) + assert_eq!( + unsafe { libc::kill(pid, 0) }, + -1, + "the walk's git child must be reaped by the clamped deadline before the response" + ); + let log = std::fs::read_to_string(&walk_log).unwrap_or_default(); + assert!( + log.contains("start"), + "the walk must have started (the budget gate passed with remaining > 0)" + ); + assert!( + !log.contains("done"), + "the walk must never complete past the budget; the clamp kills it mid-run" + ); + // The spawn_blocking closure returned and the handler finished: the + // permit is free again (held through the blocking run, no longer). + assert_eq!( + sem.available_permits(), + 1, + "the walk permit must free once the blocking walk genuinely returns" + ); + } + + /// F3 expiry between probe and walk: an object PROBED PRESENT in a + /// path-scoped repo is still not served once the budget is gone; the + /// walk-stage gate taints and stops before any walk starts. The probe is + /// stalled past the 1s budget while still SUCCEEDING via a FIFO at + /// `objects/info/alternates`: real `git cat-file -t` blocks opening it at + /// odb setup until a plain OS thread feeds it an empty alternates list at + /// ~2s, after which the probe finds the genuine loose object and reports + /// "blob". The walk-needing repo thus reaches the walk gate with zero + /// remaining: 503 naming the budget, never a 404 (the walk-gate deny of the + /// fake git's empty allowed-set would have been a verdict) and never a + /// serve, and the fake-git walk log stays EMPTY (no walk ever started). + /// MUTATION (RED): drop the walk-stage budget gate and this decays to the + /// walked 404 with one logged walk. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_budget_expiry_between_probe_and_walk_sheds_503(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + let walk_log = tmp.path().join("walks.log"); + state.git_bin = walk_logging_fake_git(tmp.path(), &walk_log); + let mut cfg = (*state.config).clone(); + cfg.ipfs_request_budget_secs = 1; + state.config = Arc::new(cfg); + + let (repo_id, oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6f3probe", + "gated", + b"probe-then-expire proof\n", + ) + .await; + state + .db + .set_visibility_rule( + &repo_id, + "src/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkU3IpfsReaderEEEEEEEEEEEEEEEEEEEEEEEE".to_string()], + "z6f3probe", + ) + .await + .unwrap(); + + // Stall the REAL-git probe past the budget while still letting it + // succeed: `objects/info/alternates` as a FIFO blocks cat-file's odb + // setup until a writer appears. The feeder thread supplies an EMPTY + // alternates list (open + close, no bytes) after 2s, so the probe then + // resolves the genuine loose object. Non-blocking open with retries so + // a handler that never probes cannot wedge the feeder forever. + let rec = state + .db + .get_repo("z6f3probe", "gated") + .await + .unwrap() + .unwrap(); + let bare = state + .repo_store + .acquire(&rec.owner_did, &rec.name) + .await + .unwrap(); + let fifo = bare.join("objects").join("info").join("alternates"); + let c_path = std::ffi::CString::new(fifo.to_str().unwrap()).unwrap(); + assert_eq!( + unsafe { libc::mkfifo(c_path.as_ptr(), 0o644) }, + 0, + "mkfifo(objects/info/alternates) must succeed" + ); + let feeder = std::thread::spawn(move || { + use std::os::unix::fs::OpenOptionsExt; + std::thread::sleep(std::time::Duration::from_secs(2)); + for _ in 0..300 { + if std::fs::OpenOptions::new() + .write(true) + .custom_flags(libc::O_NONBLOCK) + .open(&fifo) + .is_ok() + { + return; + } + std::thread::sleep(std::time::Duration::from_millis(50)); + } + }); + + let peer: SocketAddr = "203.0.113.72:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&cid_for_oid(&oid), Some(peer))) + .await + .unwrap(); + feeder.join().unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "probed-present with the budget gone must shed the truncation 503: \ + never the walked 404, never a serve" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the truncation 503 must carry Retry-After" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + let body = String::from_utf8_lossy(&body); + assert!( + body.contains("budget"), + "the shed must name the budget taint; got: {body}" + ); + let walks = std::fs::read_to_string(&walk_log) + .map(|s| s.lines().count()) + .unwrap_or(0); + assert_eq!( + walks, 0, + "no walk may START once the budget is exhausted, even for a probed-present object" + ); + } + /// Shed at capacity: an exhausted `git_ipfs_walk_semaphore` sheds a `/ipfs/{cid}` /// request with 503 BEFORE any DB/git walk (the acquire is the first thing after CID /// validation), so a lazy DB-free state suffices — exactly like the served-git shed diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index 3e0a8e50..d564a554 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -409,6 +409,32 @@ pub struct Config { )] pub ipfs_max_repo_visits: usize, + /// Absolute wall-clock budget for one admitted `GET /ipfs/{cid}` request's + /// acquire+walk lifetime, in seconds. `max_concurrent_ipfs_walks` bounds how + /// MANY requests hold walk slots; this bounds how LONG one admitted request + /// may keep its slot. Without it, each repo iteration draws a fresh + /// `git_acquire_timeout_secs` and each expensive walk a fresh + /// `git_service_timeout_secs`, so one request scanning many repos could hold + /// a scarce walk slot for hours. Every stage (acquire, `cat-file` probe, + /// visibility walk, content read) starts only while budget remains, and the + /// acquire wait and walk deadline are clamped to `min(their own timeout, + /// remaining budget)`; a stage is never started with zero remaining. On + /// exhaustion the scan stops without a verdict and the request sheds a + /// retryable 503 + Retry-After rather than a false 404. Residual overshoot + /// past the budget is bounded by the kill/reap slack of the one in-flight + /// clamped stage (the walk watchdog's SIGTERM grace + SIGKILL settle) plus + /// the `object_type` / `read_object_content` probe subprocesses, which are + /// budget-checked before they start but carry no internal duration clamp. + /// Must be positive. Default: 600s (10 min), matching + /// `git_service_timeout_secs` so a single full-length walk still fits. + #[arg( + long, + env = "GITLAWB_IPFS_REQUEST_BUDGET_SECS", + default_value_t = 600, + value_parser = clap::value_parser!(u64).range(1..) + )] + pub ipfs_request_budget_secs: u64, + /// Per-client-IP rate limit for `GET /ipfs/{cid}`, in requests per hour. The /// route is publicly reachable (`optional_signature`) and each request can drive /// a full-history git walk, so it carries a per-IP flood brake in addition to the @@ -588,6 +614,24 @@ mod tests { ); } + #[test] + fn ipfs_request_budget_secs_defaults_to_600_and_rejects_zero() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).ipfs_request_budget_secs, + 600 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--ipfs-request-budget-secs", "30"]) + .ipfs_request_budget_secs, + 30 + ); + // 0 would expire every /ipfs request at its first stage (unconditional + // 503); clap must reject it. + assert!( + Config::try_parse_from(["gitlawb-node", "--ipfs-request-budget-secs", "0"]).is_err() + ); + } + #[test] fn max_concurrent_reads_per_caller_defaults_and_rejects_out_of_range() { assert_eq!( From daf9195f29192b7dba610238529091e4402e7edd Mon Sep 17 00:00:00 2001 From: t Date: Fri, 17 Jul 2026 18:37:40 -0500 Subject: [PATCH 41/58] fix(node): admit the post-receive git scans to the encrypt pool, defer-not-shed (#174) replication_withheld_set's walk arm, resolve_candidates_for_push's git stages, and fail_closed_full_scan_objects each take a git_encrypt_semaphore permit before their spawn_blocking and carry it inside the closure, so a push burst holds at most pool-many concurrent scans and a started scan finishes with its permit. Deletion-only pushes and repos without path-scoped rules never park. Contention defers the landed push's tail (logged with queue_wait_ms), never errors it; the acquire sites document the accepted disconnect-during-park residual. --- crates/gitlawb-node/src/api/repos.rs | 523 ++++++++++++++++++++++ crates/gitlawb-node/src/git/push_delta.rs | 156 +++++++ crates/gitlawb-node/src/state.rs | 22 +- crates/gitlawb-node/tests/inv22_gates.rs | 33 ++ 4 files changed, 726 insertions(+), 8 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index bb9e743b..f25293b8 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -39,7 +39,15 @@ const ZERO_SHA: &str = "0000000000000000000000000000000000000000"; /// `withheld` is `None`, so an unvetted push neither replicates blobs nor /// announces. Returning both keeps the gate's announce decision a single /// source rather than recomputing it at each call site. +/// +/// The walk arm runs under a `git_encrypt_semaphore` admission permit (#174 F4): +/// by the time the receive-pack tail calls this, the handler's write permit has +/// already been released (receive_pack's AdmissionGuard drops when the git group +/// is reaped), so without the gate a burst of completed pushes accumulates +/// unbounded concurrent full-history walks. `encrypt_sem` is threaded in so the +/// no-walk fast paths (not announceable; no path-scoped rule) never touch it. async fn replication_withheld_set( + encrypt_sem: std::sync::Arc, rules: Option>, owner_did: &str, is_public: bool, @@ -66,7 +74,31 @@ async fn replication_withheld_set( // that off the async worker thread. Some(rules) => { let owner_did = owner_did.to_string(); + // Scan admission (#174 F4): DEFER (await), never shed — dropping the + // walk would skip the vetting and fail the push's replication closed + // for no reason. Accepted residuals, stated honestly: (1) the park + // wait is queue-depth multiplied — post-receive tails are no longer + // admission-bounded once the write permit is released, so N landed + // pushes can queue N walks and the last waits N walk-durations; + // (2) a client-timeout disconnect while parked HERE drops the + // handler future and silently loses this push's replication work — + // the encrypt_inflight coalescing requeue does NOT cover it, because + // this park precedes the `try_begin` spawn gate. + let parked = std::time::Instant::now(); + let permit = encrypt_sem + .acquire_owned() + .await + .expect("git_encrypt_semaphore is never closed"); + tracing::debug!( + repo = %disk_path.display(), + queue_wait_ms = parked.elapsed().as_millis() as u64, + "post-push withheld walk admitted to the scan pool" + ); tokio::task::spawn_blocking(move || { + // The permit lives inside the blocking closure: a started walk + // always completes holding it (a disconnect cannot cancel + // spawn_blocking or leak the permit mid-walk). + let _permit = permit; crate::git::visibility_pack::withheld_blob_oids_bounded( &disk_path, &git_bin, timeout, &rules, is_public, &owner_did, None, ) @@ -102,7 +134,15 @@ async fn replication_withheld_set( /// non-blobs plus allowed blobs. Any error in either walk (or a task panic) /// pins nothing this push, mirroring the degraded-path shape of /// `replication_withheld_set`. +/// +/// Always walks (there is no no-git arm), so the whole blocking scan runs under +/// one `git_encrypt_semaphore` admission permit (#174 F4) — see +/// `replication_withheld_set`'s acquire for the defer rationale and the honest +/// residuals (queue-depth-multiplied park wait; a disconnect while parked loses +/// this push's replication work, uncovered by the coalescing requeue). +#[allow(clippy::too_many_arguments)] async fn fail_closed_full_scan_objects( + encrypt_sem: std::sync::Arc, disk_path: std::path::PathBuf, rules: Vec, is_public: bool, @@ -111,7 +151,20 @@ async fn fail_closed_full_scan_objects( git_bin: String, timeout: std::time::Duration, ) -> Vec { + // Scan admission (#174 F4): DEFER, never shed; the permit moves into the + // closure so a started scan always completes holding it. + let parked = std::time::Instant::now(); + let permit = encrypt_sem + .acquire_owned() + .await + .expect("git_encrypt_semaphore is never closed"); + tracing::debug!( + repo = %disk_path.display(), + queue_wait_ms = parked.elapsed().as_millis() as u64, + "post-push fail-closed full scan admitted to the scan pool" + ); tokio::task::spawn_blocking(move || -> anyhow::Result> { + let _permit = permit; let allowed = crate::git::visibility_pack::replicable_blob_set_bounded( &disk_path, &git_bin, timeout, &rules, is_public, &owner_did, )?; @@ -1385,6 +1438,7 @@ pub async fn git_receive_pack( // never leaks. let rules_opt = state.db.list_visibility_rules(&record.id).await.ok(); let (announce, withheld) = replication_withheld_set( + state.git_encrypt_semaphore.clone(), rules_opt.clone(), &record.owner_did, record.is_public, @@ -1413,6 +1467,7 @@ pub async fn git_receive_pack( .filter(|s| s != ZERO_SHA) .collect(); let pin_set = crate::git::push_delta::resolve_candidates_for_push( + state.git_encrypt_semaphore.clone(), disk_path.clone(), new_tips, old_tips, @@ -1422,6 +1477,7 @@ pub async fn git_receive_pack( .await; if pin_set.full_scan { fail_closed_full_scan_objects( + state.git_encrypt_semaphore.clone(), disk_path.clone(), rules_opt.clone().unwrap_or_default(), record.is_public, @@ -2286,6 +2342,7 @@ mod tests { // Private: no rules at all. let (announce, _) = replication_withheld_set( + std::sync::Arc::new(tokio::sync::Semaphore::new(64)), None, OWNER_DID, false, @@ -2298,6 +2355,7 @@ mod tests { // Private: empty rule set, is_public=false → still not listable at root. let (announce, _) = replication_withheld_set( + std::sync::Arc::new(tokio::sync::Semaphore::new(64)), Some(vec![]), OWNER_DID, false, @@ -2310,6 +2368,7 @@ mod tests { // Public: empty rule set, is_public=true → listable at root, announces. let (announce, _) = replication_withheld_set( + std::sync::Arc::new(tokio::sync::Semaphore::new(64)), Some(vec![]), OWNER_DID, true, @@ -2441,6 +2500,7 @@ mod tests { let result = tokio::time::timeout( Duration::from_secs(10), replication_withheld_set( + std::sync::Arc::new(tokio::sync::Semaphore::new(64)), rules, OWNER_DID, true, @@ -4635,6 +4695,469 @@ mod tests { ); } + /// F4 defer proof 1: `replication_withheld_set`'s WALK arm acquires a + /// `git_encrypt_semaphore` permit before its spawn_blocking git walk, deferring + /// (never shedding) when the pool is exhausted — while its no-walk fast paths + /// (no path-scoped rule; not announceable) complete WITHOUT touching the pool. + /// On ungated code the walk runs regardless of a zero-permit pool (RED). + #[cfg(unix)] + #[tokio::test] + async fn replication_walk_defers_when_scan_pool_exhausted() { + use std::sync::Arc; + use std::time::Duration; + use tokio::sync::Semaphore; + + let tmp = tempfile::TempDir::new().unwrap(); + let marker = tmp.path().join("git.ran"); + // Fake git records ANY invocation (the walk's first call is rev-parse), then + // behaves well enough for a successful empty walk: HEAD probe succeeds, + // rev-list lists no commits. + let body = format!( + "#!/bin/sh\necho ran >> \"{}\"\ncase \"$1\" in\n rev-parse) echo deadbeef ;;\n *) : ;;\nesac\nexit 0\n", + marker.display() + ); + let git_bin = write_fake_git(tmp.path(), &body); + let scoped_rules = || Some(vec![vis_rule("/secret/**", &[])]); + + // Zero-permit pool: every gated walk must park forever. + let sem: Arc = Arc::new(Semaphore::new(0)); + + // Fast path A (negative arm): announceable, NO path-scoped rule -> zero git + // work, must complete immediately without acquiring from the empty pool. + let fast = tokio::time::timeout( + Duration::from_millis(500), + replication_withheld_set( + sem.clone(), + Some(vec![]), + OWNER_DID, + true, + tmp.path().to_path_buf(), + git_bin.clone(), + Duration::from_secs(5), + ), + ) + .await + .expect("the no-path-scoped-rule fast path must not park on the scan pool"); + assert_eq!(fast, (true, Some(std::collections::HashSet::new()))); + + // Fast path B (negative arm): not announceable (no rules) -> zero git work, + // must complete immediately without acquiring. + let fast = tokio::time::timeout( + Duration::from_millis(500), + replication_withheld_set( + sem.clone(), + None, + OWNER_DID, + false, + tmp.path().to_path_buf(), + git_bin.clone(), + Duration::from_secs(5), + ), + ) + .await + .expect("the not-announceable fast path must not park on the scan pool"); + assert_eq!(fast, (false, None)); + assert!(!marker.exists(), "the fast paths must spawn no git at all"); + + // Walk arm with the pool exhausted: must DEFER (park), spawning no git. + let blocked = tokio::time::timeout( + Duration::from_millis(500), + replication_withheld_set( + sem.clone(), + scoped_rules(), + OWNER_DID, + true, + tmp.path().to_path_buf(), + git_bin.clone(), + Duration::from_secs(5), + ), + ) + .await; + assert!( + blocked.is_err(), + "the withheld walk must defer (park on admission) when the pool is exhausted" + ); + assert!( + !marker.exists(), + "the withheld walk's git must not spawn while its admission permit is unavailable (F4)" + ); + + // Release admission: the SAME walk now runs (defer, not shed) and succeeds. + sem.add_permits(1); + let ran = replication_withheld_set( + sem, + scoped_rules(), + OWNER_DID, + true, + tmp.path().to_path_buf(), + git_bin, + Duration::from_secs(5), + ) + .await; + assert!( + marker.exists(), + "once admission is available the deferred withheld walk runs its git" + ); + assert_eq!( + ran, + (true, Some(std::collections::HashSet::new())), + "the released walk completes and vets the (empty) withheld set" + ); + } + + /// F4 defer proof 3: `fail_closed_full_scan_objects` ALWAYS walks, so its + /// spawn_blocking is always admission-gated: with the pool exhausted it defers + /// and spawns no git; with a permit the same call runs. Ungated it runs + /// regardless (RED). + #[cfg(unix)] + #[tokio::test] + async fn full_scan_pin_walk_defers_when_scan_pool_exhausted() { + use std::sync::Arc; + use std::time::Duration; + use tokio::sync::Semaphore; + + let tmp = tempfile::TempDir::new().unwrap(); + let marker = tmp.path().join("git.ran"); + let body = format!( + "#!/bin/sh\necho ran >> \"{}\"\ncase \"$1\" in\n rev-parse) echo deadbeef ;;\n *) : ;;\nesac\nexit 0\n", + marker.display() + ); + let git_bin = write_fake_git(tmp.path(), &body); + let candidates = vec!["3333333333333333333333333333333333333333".to_string()]; + + let sem: Arc = Arc::new(Semaphore::new(0)); + let blocked = tokio::time::timeout( + Duration::from_millis(500), + fail_closed_full_scan_objects( + sem.clone(), + tmp.path().to_path_buf(), + vec![vis_rule("/secret/**", &[])], + true, + OWNER_DID.to_string(), + candidates.clone(), + git_bin.clone(), + Duration::from_secs(5), + ), + ) + .await; + assert!( + blocked.is_err(), + "the fail-closed full scan must defer (park on admission) when the pool is exhausted" + ); + assert!( + !marker.exists(), + "the full scan's git must not spawn while its admission permit is unavailable (F4)" + ); + + // Release admission: the SAME scan now runs (defer, not shed). + sem.add_permits(1); + let _objs = fail_closed_full_scan_objects( + sem, + tmp.path().to_path_buf(), + vec![vis_rule("/secret/**", &[])], + true, + OWNER_DID.to_string(), + candidates, + git_bin, + Duration::from_secs(5), + ) + .await; + assert!( + marker.exists(), + "once admission is available the deferred full scan runs its git" + ); + } + + /// Shared fixture for the F4 handler-layer tests: a state whose repo_store and + /// git_bin point at the given tempdir/fake-git, plus a seeded on-disk repo, + /// optionally with a path-scoped rule (so the post-receive walks actually run). + #[cfg(unix)] + async fn f4_state_with_repo( + pool: sqlx::PgPool, + tmp: &std::path::Path, + git_bin: &str, + owner: &str, + name: &str, + path_scoped: bool, + ) -> AppState { + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.git_bin = git_bin.to_string(); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo(owner, name, &format!("/unused-{owner}-{name}"), None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + state + .repo_store + .init(&rec.owner_did, &rec.name) + .await + .unwrap(); + if path_scoped { + state + .db + .set_visibility_rule( + &rec.id, + "/secret/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkF4ReaderAAAAAAAAAAAAAAAAAAAAAAAAAAAA".to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + } + state + } + + /// A pkt-line receive-pack body carrying one branch-create ref update, so the + /// handler's post-receive tail resolves a non-empty new-tip set (the delta + /// scan's git stages run). + fn ref_update_body(new_sha: &str) -> axum::body::Bytes { + let line = format!("{ZERO_SHA} {new_sha} refs/heads/main"); + axum::body::Bytes::from(format!("{:04x}{}0000", line.len() + 4, line)) + } + + /// F4 scenario 2 — push-burst bound at the handler layer: with a scan pool of + /// ONE, two concurrent pushes to two path-scoped repos never have more than one + /// scan's git alive at a time (an atomic mkdir lock in the fake git detects any + /// overlap), and BOTH pushes still succeed 200 — defer, not shed. Two distinct + /// repos on purpose: the per-repo advisory write lock must not be what + /// serializes the scans. + #[cfg(unix)] + #[sqlx::test] + async fn receive_pack_burst_scans_serialized_and_both_pushes_succeed(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + + let tmp = tempfile::TempDir::new().unwrap(); + let lockdir = tmp.path().join("scan.lock"); + let ranfile = tmp.path().join("scan.ran"); + let overlap = tmp.path().join("scan.overlap"); + // receive-pack succeeds instantly; every candidate-scan git op (cat-file / + // rev-list / ls-tree) holds an atomic mkdir lock for 150ms — a second scan + // process alive at the same instant records an overlap. + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + receive-pack) cat > /dev/null 2>/dev/null ;;\n\ + rev-parse) echo deadbeef ;;\n\ + cat-file|rev-list|ls-tree)\n\ + if mkdir \"{lock}\" 2>/dev/null; then\n\ + echo 1 >> \"{ran}\"\n\ + sleep 0.15\n\ + rmdir \"{lock}\"\n\ + else\n\ + echo 1 >> \"{over}\"\n\ + fi\n\ + if [ \"$1\" = cat-file ]; then echo commit; fi ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + lock = lockdir.display(), + ran = ranfile.display(), + over = overlap.display(), + ); + let git_bin = write_fake_git(tmp.path(), &body); + + let mut state = + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6f4burst1", "b1", true).await; + // Second path-scoped repo on the same state/store. + state + .db + .upsert_mirror_repo("z6f4burst2", "b2", "/unused-z6f4burst2-b2", None, false) + .await + .unwrap(); + let rec2 = state + .db + .get_repo("z6f4burst2", "b2") + .await + .unwrap() + .unwrap(); + state + .repo_store + .init(&rec2.owner_did, &rec2.name) + .await + .unwrap(); + state + .db + .set_visibility_rule( + &rec2.id, + "/secret/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkF4ReaderAAAAAAAAAAAAAAAAAAAAAAAAAAAA".to_string()], + &rec2.owner_did, + ) + .await + .unwrap(); + // Scan pool of ONE: at most one post-receive walk may run at a time. + state.git_encrypt_semaphore = Arc::new(Semaphore::new(1)); + + let did = "did:key:z6MkF4BurstPusherAAAAAAAAAAAAAAAAAAAAAAAA"; + let new_sha = "1111111111111111111111111111111111111111"; + let push = |owner: &'static str, name: &'static str, peer: &'static str| { + let state = state.clone(); + tokio::spawn(async move { + git_receive_pack( + State(state), + Path((owner.to_string(), name.to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(peer.parse::().unwrap())), + axum::http::HeaderMap::new(), + ref_update_body(new_sha), + ) + .await + }) + }; + + let (a, b) = ( + push("z6f4burst1", "b1", "203.0.113.71:5000"), + push("z6f4burst2", "b2", "203.0.113.72:5000"), + ); + let a = tokio::time::timeout(std::time::Duration::from_secs(60), a) + .await + .expect("push A must complete — a scan gate must defer, never wedge") + .expect("push A task must not panic"); + let b = tokio::time::timeout(std::time::Duration::from_secs(60), b) + .await + .expect("push B must complete — a scan gate must defer, never wedge") + .expect("push B task must not panic"); + let a = a.expect("push A must succeed"); + let b = b.expect("push B must succeed"); + assert_eq!(a.status(), 200, "push A lands 200 despite scan contention"); + assert_eq!(b.status(), 200, "push B lands 200 despite scan contention"); + + // Let the detached recipients walks drain through the same pool before + // reading the detector files. + tokio::time::sleep(std::time::Duration::from_millis(800)).await; + assert!( + !overlap.exists(), + "with a scan pool of 1, no two scans' git may ever be alive at once \ + (found overlap records: {:?})", + std::fs::read_to_string(&overlap).unwrap_or_default() + ); + let ran = std::fs::read_to_string(&ranfile).unwrap_or_default(); + assert!( + ran.lines().count() >= 6, + "both pushes' scans must actually have run (withheld walk + delta probe + \ + delta rev-list each); got {} runs", + ran.lines().count() + ); + } + + /// F4 scenario 3 — fast-path non-acquisition at the handler layer: a push to a + /// public repo with NO path-scoped rules does zero post-receive git scanning + /// (the withheld short-circuit; a deletion-free flush-only body resolves no new + /// tips), so it must complete 200 even with the scan pool at ZERO permits. + /// A gate that wrongly captured a no-walk path would park this push forever. + /// Note: `resolve_candidates_for_push` spawns git for ANY non-empty new-tip set + /// (the per-tip cat-file probe), so the genuinely git-free negative arm is the + /// no-ref-update body, not a branch-create push. + #[cfg(unix)] + #[sqlx::test] + async fn receive_pack_no_scan_fast_path_completes_with_zero_scan_permits(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + + let tmp = tempfile::TempDir::new().unwrap(); + let marker = tmp.path().join("scan.ran"); + let body = format!( + "#!/bin/sh\ncase \"$1\" in\n receive-pack) cat > /dev/null 2>/dev/null ;;\n cat-file|rev-list|ls-tree) echo 1 >> \"{}\" ;;\n *) : ;;\nesac\nexit 0\n", + marker.display() + ); + let git_bin = write_fake_git(tmp.path(), &body); + let mut state = + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6f4fast", "f1", false).await; + state.git_encrypt_semaphore = Arc::new(Semaphore::new(0)); + + let peer: SocketAddr = "203.0.113.73:5000".parse().unwrap(); + let resp = tokio::time::timeout( + std::time::Duration::from_secs(30), + git_receive_pack( + State(state), + Path(("z6f4fast".to_string(), "f1".to_string())), + Extension(crate::auth::AuthenticatedDid( + "did:key:z6MkF4FastPusherAAAAAAAAAAAAAAAAAAAAAAAAA".to_string(), + )), + crate::rate_limit::PeerAddr(Some(peer)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ), + ) + .await + .expect("a no-scan push must not park on the (empty) scan pool") + .expect("the push must succeed"); + assert_eq!(resp.status(), 200); + assert!( + !marker.exists(), + "the no-walk fast paths must spawn no scan git at all" + ); + } + + /// F4 scenario 4 — landed-push-never-fails: a push whose post-receive walk must + /// park (pool held elsewhere) DEFERS and then returns the receive-pack success + /// once admission frees; contention never converts the landed push into a 5xx. + #[cfg(unix)] + #[sqlx::test] + async fn receive_pack_landed_push_defers_on_scan_contention_never_errors(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + + let tmp = tempfile::TempDir::new().unwrap(); + let body = "#!/bin/sh\ncase \"$1\" in\n receive-pack) cat > /dev/null 2>/dev/null ;;\n rev-parse) echo deadbeef ;;\n cat-file) echo commit ;;\n *) : ;;\nesac\nexit 0\n"; + let git_bin = write_fake_git(tmp.path(), body); + let mut state = + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6f4park", "p1", true).await; + let sem = Arc::new(Semaphore::new(1)); + state.git_encrypt_semaphore = sem.clone(); + // The test holds the pool's only permit: the push's withheld walk must park. + let held = sem.clone().acquire_owned().await.unwrap(); + + let peer: SocketAddr = "203.0.113.74:5000".parse().unwrap(); + let mut handle = tokio::spawn(git_receive_pack( + State(state), + Path(("z6f4park".to_string(), "p1".to_string())), + Extension(crate::auth::AuthenticatedDid( + "did:key:z6MkF4ParkPusherAAAAAAAAAAAAAAAAAAAAAAAAA".to_string(), + )), + crate::rate_limit::PeerAddr(Some(peer)), + axum::http::HeaderMap::new(), + ref_update_body("2222222222222222222222222222222222222222"), + )); + + // Parked, not errored: the handler must NOT complete while the pool is held. + let parked = tokio::time::timeout(std::time::Duration::from_millis(700), &mut handle).await; + assert!( + parked.is_err(), + "the post-receive walk must defer while the scan pool is held; got {parked:?}" + ); + + // Release admission: the SAME push now finishes with the receive-pack 200. + drop(held); + let resp = tokio::time::timeout(std::time::Duration::from_secs(30), handle) + .await + .expect("the deferred push must complete once admission frees") + .expect("the push task must not panic") + .expect("contention must never convert a landed push into an error"); + assert_eq!( + resp.status(), + 200, + "the response is the receive-pack success, not a contention 5xx" + ); + } + // ---- #174 U4 (P2-2): post-push encryption task set bounded by per-repo coalescing ---- // // The residual jatmn found is not the WALK (bounded by `git_encrypt_semaphore`, diff --git a/crates/gitlawb-node/src/git/push_delta.rs b/crates/gitlawb-node/src/git/push_delta.rs index 962c9c65..18f2c4da 100644 --- a/crates/gitlawb-node/src/git/push_delta.rs +++ b/crates/gitlawb-node/src/git/push_delta.rs @@ -282,14 +282,47 @@ pub struct PinCandidateSet { /// it can never leak because the withheld/fail-closed filter still runs on /// whatever set is returned. `full_scan` rides on the returned set so the caller /// knows when the dangling-inclusive filter is required. +/// +/// `scan_sem` is the post-receive scan admission pool (`git_encrypt_semaphore`, +/// #174 F4): both git-spawning stages — the per-tip `cat-file` probe + delta +/// rev-list, and the full-scan fallback — run under ONE permit held for the +/// whole blocking scan. The deletion-only fast path (no new tips) spawns no git +/// and never parks. pub async fn resolve_candidates_for_push( + scan_sem: std::sync::Arc, repo_path: PathBuf, new_tips: Vec, old_tips: Vec, git_bin: String, timeout: Duration, ) -> PinCandidateSet { + // No-git fast path: a deletion-only push resolves to an empty delta without + // spawning any git child, so it must not park on the scan pool. The + // kill-switch disqualifies it — forcing the full scan spawns git. + if new_tips.is_empty() && !force_full_scan() { + tracing::info!(delta = 0usize, repo = %repo_path.display(), "pin candidate set from push delta"); + return PinCandidateSet { + candidates: Vec::new(), + full_scan: false, + }; + } + // Scan admission (#174 F4): DEFER (await), never shed — a dropped scan would + // silently under-pin this push. The permit moves into the blocking closure so + // a started scan always completes holding it. Residuals as documented at + // `replication_withheld_set`'s acquire: park wait is queue-depth multiplied, + // and a client disconnect while parked loses this push's replication work. + let parked = Instant::now(); + let permit = scan_sem + .acquire_owned() + .await + .expect("git_encrypt_semaphore is never closed"); + tracing::debug!( + repo = %repo_path.display(), + queue_wait_ms = parked.elapsed().as_millis() as u64, + "pin-candidate scan admitted to the scan pool" + ); tokio::task::spawn_blocking(move || { + let _permit = permit; // ONE shared deadline for the whole scan, per jatmn ("the same deadline"). let deadline = Instant::now() + timeout; let new_refs: Vec<&str> = new_tips.iter().map(String::as_str).collect(); @@ -547,6 +580,7 @@ mod tests { let c1 = repo.commit_file("a.txt", "one\n"); let c2 = repo.commit_file("b.txt", "two\n"); let set = resolve_candidates_for_push( + std::sync::Arc::new(tokio::sync::Semaphore::new(64)), repo.path.clone(), vec![c2.clone()], vec![c1.clone()], @@ -576,6 +610,7 @@ mod tests { .into_iter() .collect(); let set = resolve_candidates_for_push( + std::sync::Arc::new(tokio::sync::Semaphore::new(64)), repo.path.clone(), vec![blob], vec![], @@ -588,6 +623,127 @@ mod tests { assert_eq!(got, all, "non-commit tip falls back to full repo scan"); } + /// F4 defer proof 2: `resolve_candidates_for_push`'s git stages (the per-tip + /// cat-file type probe + delta rev-list, and the full-scan fallback) run under a + /// scan-admission permit: with a zero-permit pool the call parks and spawns no + /// git; once a permit is available the SAME call runs (defer, not shed). On + /// ungated code the git runs regardless of the pool (RED). + #[cfg(unix)] + #[tokio::test] + async fn resolve_candidates_defers_when_scan_pool_exhausted() { + use std::os::unix::fs::PermissionsExt; + use std::sync::Arc; + use std::time::Duration; + use tokio::sync::Semaphore; + + let dir = tempfile::TempDir::new().unwrap(); + let marker = dir.path().join("git.ran"); + // Fake git records ANY invocation; cat-file reports a commit tip so the + // delta stage proceeds, rev-list yields an empty delta. + let fake = dir.path().join("fakegit"); + std::fs::write( + &fake, + format!( + "#!/bin/sh\necho ran >> \"{}\"\ncase \"$1\" in\n cat-file) echo commit ;;\n *) : ;;\nesac\nexit 0\n", + marker.display() + ), + ) + .unwrap(); + let mut perm = std::fs::metadata(&fake).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&fake, perm).unwrap(); + let git_bin = fake.to_str().unwrap().to_string(); + let tip = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef".to_string(); + + let sem: Arc = Arc::new(Semaphore::new(0)); + let blocked = tokio::time::timeout( + Duration::from_millis(500), + resolve_candidates_for_push( + sem.clone(), + dir.path().to_path_buf(), + vec![tip.clone()], + vec![], + git_bin.clone(), + Duration::from_secs(5), + ), + ) + .await; + assert!( + blocked.is_err(), + "the pin-candidate scan must defer (park on admission) when the pool is exhausted" + ); + assert!( + !marker.exists(), + "the scan's git must not spawn while its admission permit is unavailable (F4)" + ); + + // Release admission: the SAME scan now runs (defer, not shed). + sem.add_permits(1); + let set = resolve_candidates_for_push( + sem, + dir.path().to_path_buf(), + vec![tip], + vec![], + git_bin, + Duration::from_secs(5), + ) + .await; + assert!( + marker.exists(), + "once admission is available the deferred scan runs its git" + ); + assert!(!set.full_scan, "commit tip + empty rev-list is a delta"); + assert!(set.candidates.is_empty()); + } + + /// F4 fast-path negative arm: a deletion-only push (no new tips, kill-switch + /// off) computes its empty delta without spawning ANY git child, so it must + /// complete without acquiring from the scan pool — even at zero permits. The + /// per-tip cat-file probe means every push with a non-empty new tip DOES spawn + /// git, so this is the only genuinely git-free stage. + #[cfg(unix)] + #[tokio::test] + async fn resolve_candidates_no_git_fast_path_skips_admission() { + use std::os::unix::fs::PermissionsExt; + use std::sync::Arc; + use std::time::Duration; + use tokio::sync::Semaphore; + + let dir = tempfile::TempDir::new().unwrap(); + let marker = dir.path().join("git.ran"); + let fake = dir.path().join("fakegit"); + std::fs::write( + &fake, + format!("#!/bin/sh\necho ran >> \"{}\"\nexit 0\n", marker.display()), + ) + .unwrap(); + let mut perm = std::fs::metadata(&fake).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&fake, perm).unwrap(); + + let set = tokio::time::timeout( + Duration::from_millis(500), + resolve_candidates_for_push( + Arc::new(Semaphore::new(0)), + dir.path().to_path_buf(), + vec![], + vec!["deadbeefdeadbeefdeadbeefdeadbeefdeadbeef".to_string()], + fake.to_str().unwrap().to_string(), + Duration::from_secs(5), + ), + ) + .await + .expect("the no-new-tips fast path must not park on the scan pool"); + assert_eq!( + set, + PinCandidateSet { + candidates: Vec::new(), + full_scan: false + } + ); + assert!(!marker.exists(), "the fast path must spawn no git at all"); + } + #[test] fn missing_oid_tip_forces_full_scan() { let repo = Repo::new(); diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index deccf4c7..0badf802 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -110,14 +110,20 @@ pub struct AppState { /// advert pool (each source also capped by `git_push_advert_per_caller` and the /// per-IP push rate limiter), and the reserved POST pool is untouched (#174). pub git_push_advert_semaphore: Arc, - /// Bounds concurrent post-push encrypt-then-pin history walks. Each successful - /// path-scoped push releases its handler write permit and then runs a DETACHED - /// full-history walk (`withheld_blob_recipients_bounded`) to seal withheld blobs; - /// without a cap, N fast pushes spawn N concurrent full-history git walks past - /// `max_concurrent_git_pushes` (which only bounds the in-handler phase). The walk - /// acquires a permit here and DEFERS (blocks) when the pool is full rather than - /// shedding — the work is background and dropping it would lose the recovery copy - /// (#174 P1-e). A pool of its own, not `git_write_semaphore`: a long background + /// Bounds concurrent post-receive git scans. Each successful push releases its + /// handler write permit the moment receive-pack's git group is reaped, then runs + /// up to four scans over the repo: the anonymous withheld walk + /// (`replication_withheld_set`), the pin-candidate scan + /// (`resolve_candidates_for_push`), the fail-closed full scan + /// (`fail_closed_full_scan_objects`), and the DETACHED encrypt-then-pin walk + /// (`withheld_blob_recipients_bounded`). Without a cap, N fast pushes spawn N + /// concurrent full-history git walks past `max_concurrent_git_pushes` (which only + /// bounds the in-handler receive-pack phase) — #174 P1-e closed the detached walk, + /// F4 closed the other three. Each scan acquires ONE permit here per walk and + /// DEFERS (blocks) when the pool is full rather than shedding — dropping the work + /// would lose the recovery copy or silently under-pin the push. No-walk fast + /// paths (not announceable, no path-scoped rule, deletion-only push) never touch + /// the pool. A pool of its own, not `git_write_semaphore`: a long background /// walk must not hold a foreground write slot, and a handler already holding a /// write permit that needed a second would self-deadlock at pool size 1. pub git_encrypt_semaphore: Arc, diff --git a/crates/gitlawb-node/tests/inv22_gates.rs b/crates/gitlawb-node/tests/inv22_gates.rs index 7d3fd1c8..bcd0758e 100644 --- a/crates/gitlawb-node/tests/inv22_gates.rs +++ b/crates/gitlawb-node/tests/inv22_gates.rs @@ -87,4 +87,37 @@ fn inv22_concurrency_gates_present_and_not_bypassed() { "U4/P2-2 gate missing: the detached post-push encryption spawn must consult \ encrypt_inflight.try_begin to coalesce per repo (bound the outstanding-task set)" ); + + // F4: every post-receive scan helper acquires a `git_encrypt_semaphore` permit + // BEFORE its spawn_blocking git walk, so a push burst cannot accumulate unbounded + // concurrent scans once the write permit is released. Structural check: within each + // helper the first `acquire_owned` precedes the first `spawn_blocking`. Removing a + // gate pushes the next `acquire_owned` occurrence past the helper's own + // `spawn_blocking` (or off the end of the file), turning the assertion red. + let push_delta = src("git/push_delta.rs"); + for (file_src, file, helper) in [ + (&repos, "api/repos.rs", "fn replication_withheld_set"), + (&repos, "api/repos.rs", "fn fail_closed_full_scan_objects"), + ( + &push_delta, + "git/push_delta.rs", + "fn resolve_candidates_for_push", + ), + ] { + let start = file_src + .find(helper) + .unwrap_or_else(|| panic!("{file}: `{helper}` not found")); + let tail = &file_src[start..]; + let acquire = tail.find("acquire_owned").unwrap_or_else(|| { + panic!("F4 gate missing: {file} `{helper}` no longer acquires a scan permit") + }); + let spawn = tail.find("spawn_blocking").unwrap_or_else(|| { + panic!("{file}: `{helper}` lost its spawn_blocking walk — update this guard") + }); + assert!( + acquire < spawn, + "F4 gate bypassed: {file} `{helper}` must acquire its git_encrypt_semaphore \ + permit BEFORE dispatching the blocking git scan" + ); + } } From 8c95d05661fe5c570ccc60e029297c98fd3b58a9 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 17 Jul 2026 19:17:17 -0500 Subject: [PATCH 42/58] fix(node): requeue coalesced post-push work via an in-lock pending slot and loop-drain (#174) try_begin now merges a losing push's tip pairs into the in-flight key's pending slot in the same critical section as the presence check, bounded at 1024 pairs with a full-scan marker on overflow (force_full_scan threads to the resolver; empty tips never encode the marker). The task loop-drains: finish_or_take_pending either hands back the batch with the key retained, or removes the key and disarms the guard atomically, so a successor task's key survives the guard drop. Drains re-fetch rules fresh and replay the tail pipeline under the U4 helper gates only; the loop holds no task-level pool permit, so a hot repo cannot deadlock the pool at size 1. --- crates/gitlawb-node/src/api/repos.rs | 1035 ++++++++++++++++++--- crates/gitlawb-node/src/git/push_delta.rs | 81 +- crates/gitlawb-node/src/state.rs | 214 ++++- crates/gitlawb-node/tests/inv22_gates.rs | 17 + 4 files changed, 1166 insertions(+), 181 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index f25293b8..2b9cf1d4 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -843,6 +843,244 @@ async fn withheld_recipients_gated( .await } +/// Everything the detached post-push pin/encrypt task needs, cloned once at spawn +/// and shared by the snapshot iteration and every coalesced-drain iteration +/// (#174 F5). +struct EncryptTaskCtx { + ipfs_api: String, + repo_path: std::path::PathBuf, + db: Arc, + repo_id: String, + owner_did: String, + repo_name: String, + irys_url: String, + http_client: Arc, + node_did: String, + node_keypair: Arc, + git_bin: String, + git_timeout: std::time::Duration, + encrypt_sem: Arc, +} + +/// The detached post-push pin/encrypt task (#174 P2-2 + F5): run this push's own +/// pre-resolved snapshot through the pipeline, then loop-drain every push that +/// coalesced against the in-flight key until a finish attempt finds nothing +/// pending (which releases the key in the same critical section). +/// +/// The loop holds NO encrypt-pool permit at the task level: each helper it calls +/// (`replication_withheld_set`, `resolve_candidates_for_push`, +/// `fail_closed_full_scan_objects`, `withheld_recipients_gated`) acquires and +/// releases its own walk permit, so the drain makes progress at pool size 1 — a +/// task-level permit would nest over those same-semaphore acquires and deadlock. +async fn run_encrypt_pin_task( + ctx: EncryptTaskCtx, + guard: crate::state::EncryptInflightGuard, + snapshot_objects: Vec, + snapshot_rules: Option>, + snapshot_is_public: bool, +) { + pin_and_encrypt_objects(&ctx, snapshot_objects, snapshot_rules, snapshot_is_public).await; + let mut guard = guard; + loop { + match guard.finish_or_take_pending() { + crate::state::FinishOutcome::Finished(_) => break, + crate::state::FinishOutcome::Pending(g, pending) => { + guard = g; + if let Some((object_list, rules, is_public)) = + resolve_drain_object_list(&ctx, pending).await + { + pin_and_encrypt_objects(&ctx, object_list, rules, is_public).await; + } + } + } + } +} + +/// Resolve a coalesced-drain iteration's replicable object list. Re-fetches the +/// repo record and visibility rules FRESH — rules tightened between the coalesced +/// push and its drain must be honored, fail closed: a newly-withheld blob is not +/// pinned, and a repo that is no longer announceable (or whose record cannot be +/// re-read) pins nothing at all (`None`). Returns the filtered object list plus +/// the fresh rules/is_public snapshot for the encrypt stage — the same +/// resolution → withheld-filter pipeline the receive-pack tail runs. +async fn resolve_drain_object_list( + ctx: &EncryptTaskCtx, + pending: crate::state::PendingWork, +) -> Option<(Vec, Option>, bool)> { + let record = match ctx.db.get_repo(&ctx.owner_did, &ctx.repo_name).await { + Ok(Some(r)) => r, + Ok(None) => { + tracing::warn!( + repo = %ctx.repo_id, + "coalesced drain: repo record is gone; dropping the pending work" + ); + return None; + } + Err(e) => { + tracing::warn!( + repo = %ctx.repo_id, + err = %e, + "coalesced drain: repo re-fetch failed; pinning nothing (fail closed)" + ); + return None; + } + }; + let rules_opt = ctx.db.list_visibility_rules(&ctx.repo_id).await.ok(); + let (_announce, withheld) = replication_withheld_set( + ctx.encrypt_sem.clone(), + rules_opt.clone(), + &record.owner_did, + record.is_public, + ctx.repo_path.clone(), + ctx.git_bin.clone(), + ctx.git_timeout, + ) + .await; + let withheld_set = match withheld { + Some(w) => w, + None => { + tracing::info!( + repo = %ctx.repo_id, + "coalesced drain: repo is not announceable under current rules; \ + pinning nothing (fail closed)" + ); + return None; + } + }; + let (new_tips, old_tips, force_full_scan) = match pending { + crate::state::PendingWork::Tips(pairs) => { + let new_tips: Vec = pairs + .iter() + .map(|(_, n)| n.clone()) + .filter(|s| s != ZERO_SHA) + .collect(); + let old_tips: Vec = pairs + .into_iter() + .map(|(o, _)| o) + .filter(|s| s != ZERO_SHA) + .collect(); + (new_tips, old_tips, false) + } + // The overflow marker forces the full scan via the explicit flag. It must + // never be encoded as a plain empty-tips call: empty tips resolve to an + // empty delta and would pin nothing (the F5 silent loss again). + crate::state::PendingWork::FullScan => (Vec::new(), Vec::new(), true), + }; + let pin_set = crate::git::push_delta::resolve_candidates_for_push( + ctx.encrypt_sem.clone(), + ctx.repo_path.clone(), + new_tips, + old_tips, + ctx.git_bin.clone(), + ctx.git_timeout, + force_full_scan, + ) + .await; + let object_list = if pin_set.full_scan { + fail_closed_full_scan_objects( + ctx.encrypt_sem.clone(), + ctx.repo_path.clone(), + rules_opt.clone().unwrap_or_default(), + record.is_public, + record.owner_did.clone(), + pin_set.candidates, + ctx.git_bin.clone(), + ctx.git_timeout, + ) + .await + } else { + crate::git::visibility_pack::replicable_objects(pin_set.candidates, &withheld_set) + }; + Some((object_list, rules_opt, record.is_public)) +} + +/// The pin/encrypt pipeline shared by the snapshot iteration and the +/// coalesced-drain iterations: local IPFS pin, then (path-scoped rules only) the +/// admission-gated recipients walk → encrypt-then-pin → Arweave manifest anchor. +async fn pin_and_encrypt_objects( + ctx: &EncryptTaskCtx, + object_list: Vec, + rules: Option>, + is_public: bool, +) { + let pinned = + crate::ipfs_pin::pin_new_objects(&ctx.ipfs_api, &ctx.repo_path, object_list, &ctx.db).await; + if !pinned.is_empty() { + tracing::info!(count = pinned.len(), "pinned git objects to IPFS"); + for (sha, cid) in &pinned { + tracing::info!(sha = %sha, %cid, "pinned"); + } + } + + // Option B1: encrypt-then-pin the withheld blobs so authorized readers can + // recover them when the origin cannot serve them. No path-scoped rule can + // withhold a blob, so withheld_blob_recipients would return an empty map + // after a full per-ref walk; skip it. Mirrors the has_path_scoped_rule gate + // on the other two withheld-walk sites. + if let Some(rules) = rules.filter(|r| visibility_pack::has_path_scoped_rule(r)) { + // Bound the number of concurrent post-push encryption walks (#174 P1-e): + // acquire an admission permit before the full-history walk, deferring + // when the pool is full rather than shedding the recovery pin. + let recip = withheld_recipients_gated( + ctx.encrypt_sem.clone(), + ctx.repo_path.clone(), + ctx.git_bin.clone(), + ctx.git_timeout, + rules, + is_public, + ctx.owner_did.clone(), + ) + .await; + if let Ok(Ok(recipients)) = recip { + let node_seed = ctx.node_keypair.to_seed(); + let delta = crate::encrypted_pin::encrypt_and_pin( + &ctx.ipfs_api, + &ctx.repo_path, + &ctx.db, + &ctx.repo_id, + &node_seed, + &recipients, + ) + .await; + + // Option B3: anchor a per-push manifest of the blobs sealed this + // push to Arweave, so the oid->cid index survives total node loss. + // Best-effort; never fails the push. + if !delta.is_empty() && !ctx.irys_url.is_empty() { + let owner_short = crate::db::normalize_owner_key(&ctx.owner_did); + let repo_slug = format!("{owner_short}/{}", ctx.repo_name); + let ts = chrono::Utc::now().to_rfc3339(); + let manifest = crate::arweave::EncryptedManifest { + repo: &repo_slug, + owner_did: &ctx.owner_did, + node_did: &ctx.node_did, + timestamp: &ts, + blobs: &delta, + }; + match crate::arweave::anchor_encrypted_manifest( + &ctx.http_client, + &ctx.irys_url, + &manifest, + ) + .await + { + Ok(tx) if !tx.is_empty() => tracing::info!( + repo = %repo_slug, + tx_id = %tx, + "anchored encrypted manifest to Arweave" + ), + Ok(_) => {} + Err(e) => tracing::warn!( + repo = %repo_slug, + err = %e, + "encrypted manifest anchor failed" + ), + } + } + } + } +} + /// Map an error from a `smart_http` git service call to the right `AppError`: /// [`smart_http::GitServiceTimeout`] to 504, a malformed client request to 400, /// anything else to a 500 git error. Pure (no logging) so it is unit-testable; @@ -1473,6 +1711,7 @@ pub async fn git_receive_pack( old_tips, state.git_bin.clone(), std::time::Duration::from_secs(state.config.git_service_timeout_secs), + false, ) .await; if pin_set.full_scan { @@ -1497,127 +1736,56 @@ pub async fn git_receive_pack( // Pin new git objects to the local IPFS node (no-op if ipfs_api is empty). // Skipped entirely when the public cannot read the repo (withheld == None). // - // Coalesce per repo (#174 P2-2): this task parks on `git_encrypt_semaphore` - // (which DEFERS when the pool is full rather than dropping the recovery copy). To - // bound the OUTSTANDING parked-task set, only spawn if no encryption task for this - // repo is already in flight; otherwise skip — the pending/next walk over this - // repo's history already covers the newer push's objects, so the recovery copy is - // delayed, never lost. The guard removes the repo key when the task ends (success, - // error, or panic), so a later push is re-admitted (no permanent skip). + // Coalesce-and-requeue per repo (#174 P2-2 + F5): the spawned task's walks park + // on `git_encrypt_semaphore` (which DEFERS when the pool is full rather than + // dropping the recovery copy). To bound the OUTSTANDING task set, at most one + // task per repo is in flight; a push arriving while one is in flight does NOT + // spawn a duplicate — and is NOT dropped either. The in-flight task pins only + // its own pre-spawn object-list snapshot, so this push's (old, new) tip pairs + // are merged into the in-flight key's pending slot in the same critical section + // as the presence check, and the task loop-drains them (fresh rules, fail + // closed) before releasing the key. Without the requeue a coalesced push's pins + // and recovery copies would be silently absent until an unrelated later push + // (the F5 loss). The guard still releases the key on panic (Drop on unwind), so + // a crashed walk never permanently locks the repo out. if withheld.is_some() { - match state.encrypt_inflight.try_begin(&record.id) { - None => { + let tip_pairs: Vec<(String, String)> = ref_updates + .iter() + .map(|u| (u.old_sha.clone(), u.new_sha.clone())) + .collect(); + match state.encrypt_inflight.try_begin(&record.id, tip_pairs) { + crate::state::BeginOutcome::Coalesced => { tracing::debug!( repo = %record.id, - "post-push encryption task already in flight for this repo; coalescing \ - (the pending recovery-copy walk covers this push's objects)" + "post-push encryption task already in flight for this repo; coalesced \ + — this push's tip pairs are queued for that task's drain" ); } - Some(inflight_guard) => { - let object_list_ipfs = object_list.clone(); - let ipfs_api = state.config.ipfs_api.clone(); - let repo_path_clone = disk_path.clone(); - let db_clone = state.db.clone(); - let rules_for_enc = rules_opt.clone(); - let repo_id = record.id.clone(); - let owner_did = record.owner_did.clone(); - let is_public = record.is_public; - let irys_url = state.config.irys_url.clone(); - let http_client = std::sync::Arc::clone(&state.http_client); - let node_did_str = state.node_did.to_string(); - let node_seed = state.node_keypair.to_seed(); - let repo_name = record.name.clone(); - let enc_git_bin = state.git_bin.clone(); - let enc_timeout = - std::time::Duration::from_secs(state.config.git_service_timeout_secs); - let encrypt_sem = state.git_encrypt_semaphore.clone(); - tokio::spawn(async move { - // Held for the whole task; drop (on completion/error/panic) releases the - // repo's coalescing key so the next push for this repo can spawn again. - let _inflight_guard = inflight_guard; - let pinned = crate::ipfs_pin::pin_new_objects( - &ipfs_api, - &repo_path_clone, - object_list_ipfs, - &db_clone, - ) - .await; - if !pinned.is_empty() { - tracing::info!(count = pinned.len(), "pinned git objects to IPFS"); - for (sha, cid) in &pinned { - tracing::info!(sha = %sha, %cid, "pinned"); - } - } - - // Option B1: encrypt-then-pin the withheld blobs so authorized - // readers can recover them when the origin cannot serve them. - // No path-scoped rule can withhold a blob, so withheld_blob_recipients - // would return an empty map after a full per-ref walk; skip it. Mirrors - // the has_path_scoped_rule gate on the other two withheld-walk sites. - if let Some(rules) = - rules_for_enc.filter(|r| visibility_pack::has_path_scoped_rule(r)) - { - // Bound the number of concurrent post-push encryption walks (#174 P1-e): - // acquire an admission permit before the full-history walk, deferring - // when the pool is full rather than shedding the recovery pin. - let recip = withheld_recipients_gated( - encrypt_sem.clone(), - repo_path_clone.clone(), - enc_git_bin.clone(), - enc_timeout, - rules, - is_public, - owner_did.clone(), - ) - .await; - if let Ok(Ok(recipients)) = recip { - let delta = crate::encrypted_pin::encrypt_and_pin( - &ipfs_api, - &repo_path_clone, - &db_clone, - &repo_id, - &node_seed, - &recipients, - ) - .await; - - // Option B3: anchor a per-push manifest of the blobs sealed - // this push to Arweave, so the oid->cid index survives total - // node loss. Best-effort; never fails the push. - if !delta.is_empty() && !irys_url.is_empty() { - let owner_short = crate::db::normalize_owner_key(&owner_did); - let repo_slug = format!("{owner_short}/{repo_name}"); - let ts = chrono::Utc::now().to_rfc3339(); - let manifest = crate::arweave::EncryptedManifest { - repo: &repo_slug, - owner_did: &owner_did, - node_did: &node_did_str, - timestamp: &ts, - blobs: &delta, - }; - match crate::arweave::anchor_encrypted_manifest( - &http_client, - &irys_url, - &manifest, - ) - .await - { - Ok(tx) if !tx.is_empty() => tracing::info!( - repo = %repo_slug, - tx_id = %tx, - "anchored encrypted manifest to Arweave" - ), - Ok(_) => {} - Err(e) => tracing::warn!( - repo = %repo_slug, - err = %e, - "encrypted manifest anchor failed" - ), - } - } - } - } - }); + crate::state::BeginOutcome::Admitted(inflight_guard) => { + let ctx = EncryptTaskCtx { + ipfs_api: state.config.ipfs_api.clone(), + repo_path: disk_path.clone(), + db: state.db.clone(), + repo_id: record.id.clone(), + owner_did: record.owner_did.clone(), + repo_name: record.name.clone(), + irys_url: state.config.irys_url.clone(), + http_client: std::sync::Arc::clone(&state.http_client), + node_did: state.node_did.to_string(), + node_keypair: std::sync::Arc::clone(&state.node_keypair), + git_bin: state.git_bin.clone(), + git_timeout: std::time::Duration::from_secs( + state.config.git_service_timeout_secs, + ), + encrypt_sem: state.git_encrypt_semaphore.clone(), + }; + tokio::spawn(run_encrypt_pin_task( + ctx, + inflight_guard, + object_list.clone(), + rules_opt.clone(), + record.is_public, + )); } } } @@ -5197,9 +5365,9 @@ mod tests { let mut admitted = Vec::new(); let mut coalesced = 0usize; for _ in 0..K { - match inflight.try_begin(repo) { - Some(g) => admitted.push(g), - None => coalesced += 1, + match inflight.try_begin(repo, vec![]) { + crate::state::BeginOutcome::Admitted(g) => admitted.push(g), + crate::state::BeginOutcome::Coalesced => coalesced += 1, } } @@ -5238,12 +5406,15 @@ mod tests { /// one repo in flight cannot starve a second repo's recovery copy. #[test] fn u4_distinct_repos_each_admit_one_encrypt_task() { + use crate::state::BeginOutcome; let inflight = crate::state::EncryptInflight::new(); - let a = inflight.try_begin("owner/repo-a"); - let b = inflight.try_begin("owner/repo-b"); - let c = inflight.try_begin("owner/repo-c"); + let a = inflight.try_begin("owner/repo-a", vec![]); + let b = inflight.try_begin("owner/repo-b", vec![]); + let c = inflight.try_begin("owner/repo-c", vec![]); assert!( - a.is_some() && b.is_some() && c.is_some(), + matches!(&a, BeginOutcome::Admitted(_)) + && matches!(&b, BeginOutcome::Admitted(_)) + && matches!(&c, BeginOutcome::Admitted(_)), "three distinct repos each admit their own encryption task" ); assert_eq!(inflight.len(), 3, "one in-flight entry per distinct repo"); @@ -5258,19 +5429,24 @@ mod tests { /// survives normal completion AND a panic, so no permanent skip / no leaked key. #[test] fn u4_coalesced_repo_is_reprocessed_after_task_ends_not_permanently_skipped() { + use crate::state::{BeginOutcome, FinishOutcome}; let inflight = crate::state::EncryptInflight::new(); let repo = "did:key:z6MkDurableRepoBBBBBBBBBBBBBBBBBBBBBBBBB/repo"; // Push #1 admits and "spawns". A concurrent push #2 (task #1 still in flight) - // coalesces — no duplicate spawn. - let guard1 = inflight.try_begin(repo).expect("first push admits"); + // coalesces — no duplicate spawn; its (empty) tip set is recorded, not lost. + let guard1 = admit(&inflight, repo); assert!( - inflight.try_begin(repo).is_none(), + matches!(inflight.try_begin(repo, vec![]), BeginOutcome::Coalesced), "while task #1 is in flight, push #2 to the same repo coalesces" ); - // Task #1 finishes (encrypt_and_pin ran or errored): guard drops, key released. - drop(guard1); + // Task #1 finishes normally: nothing pending (push #2 carried no tips), so + // the empty-pending check removes the key in its critical section. + assert!( + matches!(guard1.finish_or_take_pending(), FinishOutcome::Finished(_)), + "no pending tips — the task exits and releases the key" + ); assert_eq!( inflight.len(), 0, @@ -5280,18 +5456,21 @@ mod tests { // A LATER push for the SAME repo is admitted again (processed, not skipped // forever). This is what coalesce->shed breaks: shed drops the job and no sweep // re-derives the missing copy, so the recovery copy is permanently lost. - let guard2 = inflight.try_begin(repo).expect( - "a later push for a coalesced repo MUST be re-admitted — durability: the \ - deferred recovery copy is produced eventually, never dropped", - ); + let guard2 = admit(&inflight, repo); + // An errored task (guard dropped without finishing) still releases the key. drop(guard2); assert_eq!(inflight.len(), 0); - // Durability across PANIC: a task that panics mid-walk must still release its key - // (Drop runs on unwind), so one crashed walk never permanently locks a repo out - // of future recovery copies. + // Durability across PANIC: a task that panics mid-walk must still release its + // key (the still-armed guard's Drop runs on unwind), so one crashed walk never + // permanently locks a repo out of future recovery copies. Coalesce real tips + // first: the panic loses them (logged), and the loss must not corrupt the set. let panicked = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - let _g = inflight.try_begin(repo).expect("admit before the panic"); + let _g = admit(&inflight, repo); + assert!(matches!( + inflight.try_begin(repo, vec![("old1".to_string(), "new1".to_string())]), + BeginOutcome::Coalesced + )); assert_eq!(inflight.len(), 1); panic!("simulate the detached encryption task panicking mid-walk"); })); @@ -5302,9 +5481,12 @@ mod tests { "a panicked encryption task still releases its repo key (Drop on unwind) — no \ permanent leak that would block every future recovery copy for the repo" ); + // The next push is re-admitted, and the panicked task's pending tips did NOT + // survive into it (they are lost-and-logged, recovered only by a later push). + let guard3 = admit(&inflight, repo); assert!( - inflight.try_begin(repo).is_some(), - "after a panicked task the repo can still be admitted — durability preserved" + matches!(guard3.finish_or_take_pending(), FinishOutcome::Finished(_)), + "the pre-panic pending tips must not leak into the re-admitted task" ); } @@ -5315,11 +5497,25 @@ mod tests { let inflight = crate::state::EncryptInflight::new(); assert!(inflight.is_empty(), "cold set is empty"); assert!( - inflight.try_begin("owner/first").is_some(), + matches!( + inflight.try_begin("owner/first", vec![]), + crate::state::BeginOutcome::Admitted(_) + ), "the first push on a cold in-flight set must admit (never falsely coalesce)" ); } + /// Unwrap an Admitted outcome (panic on Coalesced) — the u4/u5 suites' shorthand. + fn admit( + inflight: &crate::state::EncryptInflight, + repo: &str, + ) -> crate::state::EncryptInflightGuard { + match inflight.try_begin(repo, vec![]) { + crate::state::BeginOutcome::Admitted(g) => g, + crate::state::BeginOutcome::Coalesced => panic!("expected {repo} to admit"), + } + } + /// Model of the pre-fix / mutated code: no coalescing check, so every push spawns. /// Returns the count of tasks spawned (== the size of the unbounded outstanding set /// the fix prevents), used as the RED comparison in the bound test above. @@ -5327,6 +5523,567 @@ mod tests { (0..pushes).count() } + // ---- #174 U5 (F5): a push that loses try_begin is REQUEUED, never dropped ---- + // + // F5: the in-flight task pins only its own pre-spawn object-list snapshot, so a + // push B arriving while task A is in flight used to be SKIPPED outright (the old + // None arm) — B's pins and recovery copies were silently absent until an + // unrelated later push re-walked the repo. U5 records B's (old, new) tip pairs + // into the in-flight key's pending slot in the SAME critical section as the + // presence check, and A's task loop-drains them before releasing the key. + + /// The F5 lost-update repro. Task A is in flight past its snapshot; push B + /// coalesces carrying its tip pair; when A finishes its snapshot iteration the + /// tracker must hand A exactly B's recorded work with the key retained — and only + /// an empty pending check may remove the key. On pre-U5 code this is RED: the + /// coalesce arm records B's work nowhere and there is no drain surface at all. + #[test] + fn u5_coalesced_push_work_is_drained_by_the_inflight_task() { + use crate::state::{BeginOutcome, FinishOutcome, PendingWork}; + let inflight = crate::state::EncryptInflight::new(); + let repo = "did:key:z6MkF5LostUpdateCCCCCCCCCCCCCCCCCCCCCCCC/repo"; + + // Push A admits; its spawned task is "in flight past its snapshot". + let guard_a = match inflight.try_begin(repo, vec![]) { + BeginOutcome::Admitted(g) => g, + BeginOutcome::Coalesced => panic!("first push must admit"), + }; + + // Push B lands while A is in flight: coalesced, tip pair recorded. + let b_pair = ( + "b0ldb0ldb0ldb0ldb0ldb0ldb0ldb0ldb0ldb0ld".to_string(), + "bnewbnewbnewbnewbnewbnewbnewbnewbnewbnew".to_string(), + ); + match inflight.try_begin(repo, vec![b_pair.clone()]) { + BeginOutcome::Coalesced => {} + BeginOutcome::Admitted(_) => panic!("push B must coalesce while A is in flight"), + } + + // A finishes its snapshot iteration: it must be handed B's work (drained), + // not exit — an exit here is exactly the F5 silent loss. + match guard_a.finish_or_take_pending() { + FinishOutcome::Pending(guard_a, work) => { + assert_eq!( + work, + PendingWork::Tips(vec![b_pair]), + "A drains exactly B's recorded tip pair" + ); + assert_eq!(inflight.len(), 1, "the key is retained while A iterates"); + // Nothing further pending: A now exits and releases the key. + match guard_a.finish_or_take_pending() { + FinishOutcome::Finished(_) => {} + FinishOutcome::Pending(..) => panic!("no second batch was recorded"), + } + assert_eq!( + inflight.len(), + 0, + "an empty pending check at task end releases the key" + ); + } + FinishOutcome::Finished(_) => panic!( + "F5: B's coalesced work vanished — the in-flight task exited without draining it" + ), + } + } + + /// Drain-vs-admit race, both orderings driven deterministically through the lock + /// API (the check+merge and check+remove are each ONE critical section, so a push + /// can only land on one side of A's final pending check — never inside it): + /// before it, the push is merged and A drains it; after it, the key is gone and + /// the push is admitted as a fresh task. Neither ordering loses the work. A + /// check-then-record split (merge moved outside try_begin's critical section) + /// turns ordering 1 RED: the work recorded after A's check is never drained. + #[test] + fn u5_drain_vs_admit_race_loses_no_work_in_either_ordering() { + use crate::state::{BeginOutcome, FinishOutcome, PendingWork}; + let inflight = crate::state::EncryptInflight::new(); + let repo = "did:key:z6MkF5RaceOrderDDDDDDDDDDDDDDDDDDDDDDDDD/repo"; + let pair = ("cold".to_string(), "cnew".to_string()); + + // Ordering 1: push C lands BEFORE A's final pending check → merged in + // try_begin's critical section → A must drain it (key retained). + let guard_a = admit(&inflight, repo); + assert!(matches!( + inflight.try_begin(repo, vec![pair.clone()]), + BeginOutcome::Coalesced + )); + match guard_a.finish_or_take_pending() { + FinishOutcome::Pending(g, work) => { + assert_eq!(work, PendingWork::Tips(vec![pair.clone()])); + assert!(matches!( + g.finish_or_take_pending(), + FinishOutcome::Finished(_) + )); + } + FinishOutcome::Finished(_) => { + panic!("a push merged before the final check must be drained, not lost") + } + } + assert!(inflight.is_empty()); + + // Ordering 2: push C lands AFTER A's final pending check removed the key → + // it must be ADMITTED as a fresh task (its own snapshot covers its work). + let guard_a = admit(&inflight, repo); + assert!(matches!( + guard_a.finish_or_take_pending(), + FinishOutcome::Finished(_) + )); + match inflight.try_begin(repo, vec![pair]) { + BeginOutcome::Admitted(g) => drop(g), + BeginOutcome::Coalesced => panic!( + "a push landing after the key was removed must admit a new task — a \ + coalesce here records work no task will ever drain" + ), + } + } + + /// Exit-vs-successor (the double-remove hazard). A's normal exit removes the key + /// and disarms the guard in ONE critical section, and the disarmed guard is + /// handed back — so its eventual Drop lands in the real remove→drop window. A + /// successor task B admitted inside that window must keep ITS key when A's guard + /// finally drops. With the disarm reverted (Drop removing unconditionally) this + /// is RED: dropping A's guard deletes B's key and the third push falsely admits + /// a second task for the repo. + #[test] + fn u5_disarmed_guard_drop_never_removes_a_successor_key() { + use crate::state::{BeginOutcome, FinishOutcome}; + let inflight = crate::state::EncryptInflight::new(); + let repo = "did:key:z6MkF5DisarmEEEEEEEEEEEEEEEEEEEEEEEEEEEE/repo"; + + // A admits and exits normally; HOLD the disarmed guard to keep the window open. + let guard_a = admit(&inflight, repo); + let disarmed = match guard_a.finish_or_take_pending() { + FinishOutcome::Finished(g) => g, + FinishOutcome::Pending(..) => panic!("nothing was pending"), + }; + assert!(inflight.is_empty(), "A's exit released the key"); + + // Successor B is admitted inside the remove→drop window. + let guard_b = admit(&inflight, repo); + assert_eq!(inflight.len(), 1); + + // A's disarmed guard now drops. B's key must SURVIVE: a third push still + // coalesces against B's in-flight task. + drop(disarmed); + assert_eq!( + inflight.len(), + 1, + "dropping A's disarmed guard must not remove successor B's key" + ); + assert!( + matches!(inflight.try_begin(repo, vec![]), BeginOutcome::Coalesced), + "B's task is still the (only) in-flight task — at-most-one-per-repo holds" + ); + drop(guard_b); + assert!(inflight.is_empty()); + } + + /// Pending overflow: past the 1024-pair bound the slot degrades to the FullScan + /// marker (bounded memory under a hostile push burst); at exactly the bound it + /// stays a Tips batch. The marker is an explicit variant, never an empty tip + /// list — an empty-tips encoding would drain to an empty delta and pin nothing. + #[test] + fn u5_pending_overflow_degrades_to_full_scan_marker() { + use crate::state::{BeginOutcome, FinishOutcome, PendingWork}; + let inflight = crate::state::EncryptInflight::new(); + let pair = |i: usize| (format!("old{i}"), format!("new{i}")); + + // At the bound: exactly 1024 pairs stay a Tips batch. + let repo_at = "owner/at-bound"; + let g = admit(&inflight, repo_at); + assert!(matches!( + inflight.try_begin(repo_at, (0..1024).map(pair).collect()), + BeginOutcome::Coalesced + )); + match g.finish_or_take_pending() { + FinishOutcome::Pending(g, PendingWork::Tips(v)) => { + assert_eq!(v.len(), 1024, "at the bound the pairs are kept verbatim"); + assert!(matches!( + g.finish_or_take_pending(), + FinishOutcome::Finished(_) + )); + } + other => panic!( + "expected a Tips batch at the bound, got {:?}", + match other { + FinishOutcome::Pending(_, w) => Some(w), + FinishOutcome::Finished(_) => None, + } + ), + } + + // Past the bound: the accumulated slot degrades to FullScan and later + // merges are absorbed (still one bounded marker, not a growing list). + let repo_over = "owner/over-bound"; + let g = admit(&inflight, repo_over); + assert!(matches!( + inflight.try_begin(repo_over, (0..1024).map(pair).collect()), + BeginOutcome::Coalesced + )); + assert!(matches!( + inflight.try_begin(repo_over, vec![pair(9999)]), + BeginOutcome::Coalesced + )); + assert!(matches!( + inflight.try_begin(repo_over, vec![pair(10000)]), + BeginOutcome::Coalesced + )); + match g.finish_or_take_pending() { + FinishOutcome::Pending(g, work) => { + assert_eq!( + work, + PendingWork::FullScan, + "overflow degrades to the explicit FullScan marker" + ); + assert!(matches!( + g.finish_or_take_pending(), + FinishOutcome::Finished(_) + )); + } + FinishOutcome::Finished(_) => panic!("the overflowed pending work vanished"), + } + } + + // ---- u5 drain-pipeline fixtures: a real git repo + a DB repo row ---- + + fn u5_git(dir: &std::path::Path, args: &[&str]) -> String { + let out = std::process::Command::new("git") + .args(args) + .current_dir(dir) + .output() + .unwrap(); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + } + + fn u5_init_repo(dir: &std::path::Path) { + u5_git(dir, &["init", "-q", "-b", "main"]); + u5_git(dir, &["config", "user.email", "t@t"]); + u5_git(dir, &["config", "user.name", "t"]); + } + + /// Commit `name` (parent dirs created) with `body`; returns the commit sha. + fn u5_commit_file(dir: &std::path::Path, name: &str, body: &str) -> String { + let path = dir.join(name); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(&path, body).unwrap(); + u5_git(dir, &["add", name]); + u5_git(dir, &["commit", "-qm", &format!("add {name}")]); + u5_git(dir, &["rev-parse", "HEAD"]) + } + + /// A drain-task context over the test state and an on-disk repo. Empty + /// `ipfs_api`/`irys_url` keep the pin/anchor stages inert (no network). + fn u5_ctx( + state: &AppState, + rec: &crate::db::RepoRecord, + repo_path: std::path::PathBuf, + git_bin: &str, + sem: std::sync::Arc, + ) -> EncryptTaskCtx { + EncryptTaskCtx { + ipfs_api: String::new(), + repo_path, + db: state.db.clone(), + repo_id: rec.id.clone(), + owner_did: rec.owner_did.clone(), + repo_name: rec.name.clone(), + irys_url: String::new(), + http_client: std::sync::Arc::clone(&state.http_client), + node_did: state.node_did.to_string(), + node_keypair: std::sync::Arc::clone(&state.node_keypair), + git_bin: git_bin.to_string(), + git_timeout: std::time::Duration::from_secs(600), + encrypt_sem: sem, + } + } + + /// The drain resolves a coalesced push's tip pair to exactly that push's + /// introduced objects (delta semantics — the F5 observable: push B's pins are + /// recorded by the drain, and pre-existing objects are not re-listed). + #[sqlx::test] + async fn u5_drain_resolves_coalesced_tips_to_their_objects(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + u5_init_repo(tmp.path()); + let c1 = u5_commit_file(tmp.path(), "a.txt", "one\n"); + let c2 = u5_commit_file(tmp.path(), "b.txt", "two\n"); + state + .db + .upsert_mirror_repo("z6u5delta", "d", "/unused", None, false) + .await + .unwrap(); + let rec = state.db.get_repo("z6u5delta", "d").await.unwrap().unwrap(); + let ctx = u5_ctx( + &state, + &rec, + tmp.path().to_path_buf(), + "git", + std::sync::Arc::new(tokio::sync::Semaphore::new(4)), + ); + + // Push B advanced main c1 -> c2 and lost try_begin; its pair was coalesced. + let (list, _rules, is_public) = resolve_drain_object_list( + &ctx, + crate::state::PendingWork::Tips(vec![(c1.clone(), c2.clone())]), + ) + .await + .expect("a public repo drains to a pin list"); + assert!(is_public, "mirror rows are public"); + let got: std::collections::HashSet = list.into_iter().collect(); + let new_blob = u5_git(tmp.path(), &["rev-parse", "HEAD:b.txt"]); + let old_blob = u5_git(tmp.path(), &["rev-parse", &format!("{c1}:a.txt")]); + assert!( + got.contains(&c2) && got.contains(&new_blob), + "B's commit and blob are in the drained pin list (the F5 fix)" + ); + assert!( + !got.contains(&c1) && !got.contains(&old_blob), + "pre-existing objects are not re-listed (delta, not full scan)" + ); + } + + /// The FullScan marker drains through the FLAGGED full-scan path to a NON-EMPTY + /// candidate set. RED arm of the encoding: were the marker a plain empty-tips + /// call, the deletion-only fast path would return an empty delta and the drain + /// would pin nothing (the F5 silent loss resurfacing). + #[sqlx::test] + async fn u5_drain_full_scan_marker_yields_nonempty_candidates(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + u5_init_repo(tmp.path()); + let c1 = u5_commit_file(tmp.path(), "a.txt", "one\n"); + state + .db + .upsert_mirror_repo("z6u5full", "f", "/unused", None, false) + .await + .unwrap(); + let rec = state.db.get_repo("z6u5full", "f").await.unwrap().unwrap(); + let ctx = u5_ctx( + &state, + &rec, + tmp.path().to_path_buf(), + "git", + std::sync::Arc::new(tokio::sync::Semaphore::new(4)), + ); + + let (list, _rules, _pub) = + resolve_drain_object_list(&ctx, crate::state::PendingWork::FullScan) + .await + .expect("a public repo drains to a pin list"); + let got: std::collections::HashSet = list.into_iter().collect(); + assert!( + !got.is_empty(), + "the FullScan drain must enumerate the repo — an empty list means the \ + marker collapsed into the empty-tips fast path" + ); + let blob = u5_git(tmp.path(), &["rev-parse", "HEAD:a.txt"]); + assert!( + got.contains(&c1) && got.contains(&blob), + "the full-scan drain covers the repo's commit and blob" + ); + } + + /// Rules tightened between the coalesced push and its drain are honored, fail + /// closed: the drain re-fetches rules/is_public fresh, so (1) a newly-withheld + /// blob is NOT pinned, and (2) a repo whose root became unreadable to the + /// anonymous public drains to nothing at all. + #[sqlx::test] + async fn u5_drain_honors_rules_tightened_after_the_push(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + u5_init_repo(tmp.path()); + u5_commit_file(tmp.path(), "pub.txt", "public\n"); + let c2 = u5_commit_file(tmp.path(), "secret/hidden.txt", "sealed\n"); + state + .db + .upsert_mirror_repo("z6u5tight", "t", "/unused", None, false) + .await + .unwrap(); + let rec = state.db.get_repo("z6u5tight", "t").await.unwrap().unwrap(); + let ctx = u5_ctx( + &state, + &rec, + tmp.path().to_path_buf(), + "git", + std::sync::Arc::new(tokio::sync::Semaphore::new(4)), + ); + let pending = || crate::state::PendingWork::Tips(vec![(ZERO_SHA.to_string(), c2.clone())]); + + // At push time the repo had no rules. TIGHTEN before the drain: /secret/** + // becomes reader-gated. The drain must re-fetch and withhold the new blob. + state + .db + .set_visibility_rule( + &rec.id, + "/secret/**", + crate::db::VisibilityMode::B, + &[READER_DID.to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + let (list, _rules, _pub) = resolve_drain_object_list(&ctx, pending()) + .await + .expect("still announceable at root"); + let got: std::collections::HashSet = list.into_iter().collect(); + let pub_blob = u5_git(tmp.path(), &["rev-parse", "HEAD:pub.txt"]); + let secret_blob = u5_git(tmp.path(), &["rev-parse", "HEAD:secret/hidden.txt"]); + assert!( + got.contains(&pub_blob), + "the still-public blob is pinned by the drain" + ); + assert!( + !got.contains(&secret_blob), + "a blob withheld by a rule added AFTER the push must NOT be pinned by \ + the drain (fresh rules, fail closed)" + ); + + // Tighten further: root becomes reader-gated → not announceable to the + // anonymous public → the drain pins nothing at all. + state + .db + .set_visibility_rule( + &rec.id, + "/", + crate::db::VisibilityMode::A, + &[READER_DID.to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + assert!( + resolve_drain_object_list(&ctx, pending()).await.is_none(), + "a repo no longer announceable under current rules drains to nothing \ + (fail closed)" + ); + } + + /// Hot-repo drain at encrypt-pool size 1: the task loop holds NO task-level + /// permit, so per-iteration helper acquires (withheld walk, candidate scan, + /// recipients walk) each get the pool's single permit in turn and BOTH the + /// snapshot iteration and the coalesced-drain iteration complete. RED if the + /// loop takes a task-level permit: the first helper acquire nests over the + /// same exhausted semaphore and the task parks forever. + #[sqlx::test] + async fn u5_hot_repo_drain_completes_at_pool_size_one(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + u5_init_repo(tmp.path()); + u5_commit_file(tmp.path(), "pub.txt", "public\n"); + let c2 = u5_commit_file(tmp.path(), "secret/hidden.txt", "sealed\n"); + state + .db + .upsert_mirror_repo("z6u5hot", "h", "/unused", None, false) + .await + .unwrap(); + let rec = state.db.get_repo("z6u5hot", "h").await.unwrap().unwrap(); + // A path-scoped rule so every gated walk actually runs (withheld walk on + // the drain, recipients walk on both iterations). READER_DID carries no + // resolvable key, so the encrypt stage plans no seal and stays offline. + state + .db + .set_visibility_rule( + &rec.id, + "/secret/**", + crate::db::VisibilityMode::B, + &[READER_DID.to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + let rules = state.db.list_visibility_rules(&rec.id).await.unwrap(); + + let sem = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + let ctx = u5_ctx(&state, &rec, tmp.path().to_path_buf(), "git", sem); + + let inflight = crate::state::EncryptInflight::new(); + let guard = admit(&inflight, &rec.id); + assert!(matches!( + inflight.try_begin(&rec.id, vec![(ZERO_SHA.to_string(), c2)]), + crate::state::BeginOutcome::Coalesced + )); + + tokio::time::timeout( + std::time::Duration::from_secs(60), + run_encrypt_pin_task(ctx, guard, Vec::new(), Some(rules), true), + ) + .await + .expect( + "the drain must complete at pool size 1 — a task-level permit would \ + deadlock the helper-internal acquires", + ); + assert!( + inflight.is_empty(), + "the drained task released its repo key on exit" + ); + } + + /// The task LOOP is load-bearing: work coalesced during the snapshot iteration + /// is drained (its candidate scan runs git) before the task exits. RED under + /// the drain-loop revert (task drops its guard after the snapshot without + /// checking pending): the fake git never runs and the marker is absent. + #[cfg(unix)] + #[sqlx::test] + async fn u5_task_drains_coalesced_work_before_exiting(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + let marker = tmp.path().join("git.ran"); + // The drain's candidate scan probes the tip type then walks: report a + // commit tip and an empty rev-list, recording every invocation. + let body = format!( + "#!/bin/sh\necho ran >> \"{}\"\ncase \"$1\" in\n cat-file) echo commit ;;\n *) : ;;\nesac\nexit 0\n", + marker.display() + ); + let git_bin = write_fake_git(tmp.path(), &body); + state + .db + .upsert_mirror_repo("z6u5loop", "l", "/unused", None, false) + .await + .unwrap(); + let rec = state.db.get_repo("z6u5loop", "l").await.unwrap().unwrap(); + let ctx = u5_ctx( + &state, + &rec, + tmp.path().to_path_buf(), + &git_bin, + std::sync::Arc::new(tokio::sync::Semaphore::new(4)), + ); + + let inflight = crate::state::EncryptInflight::new(); + let guard = admit(&inflight, &rec.id); + // Push B coalesces mid-flight with a real (created-ref) tip pair. + assert!(matches!( + inflight.try_begin( + &rec.id, + vec![( + ZERO_SHA.to_string(), + "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef".to_string(), + )] + ), + crate::state::BeginOutcome::Coalesced + )); + + // Snapshot: empty object list, no rules — the snapshot iteration itself + // spawns no git, so any git invocation below belongs to the DRAIN. + run_encrypt_pin_task(ctx, guard, Vec::new(), None, true).await; + assert!( + marker.exists(), + "the task must drain B's coalesced tips (candidate scan runs git) \ + before exiting — an absent marker is the F5 skip" + ); + assert!( + inflight.is_empty(), + "the empty pending check at task end released the key" + ); + } + /// #174 SC2 (per-source key, U1): the per-caller read sub-cap keys on the /// resolved source IP, NOT the signed DID, so a disposable-DID farm cannot /// multiply its budget. Fill the source IP's single read slot, then drive two diff --git a/crates/gitlawb-node/src/git/push_delta.rs b/crates/gitlawb-node/src/git/push_delta.rs index 18f2c4da..27064357 100644 --- a/crates/gitlawb-node/src/git/push_delta.rs +++ b/crates/gitlawb-node/src/git/push_delta.rs @@ -288,6 +288,13 @@ pub struct PinCandidateSet { /// rev-list, and the full-scan fallback — run under ONE permit held for the /// whole blocking scan. The deletion-only fast path (no new tips) spawns no git /// and never parks. +/// +/// `force_full_scan` forces the full-scan fallback regardless of the tips — the +/// coalesced-drain `PendingWork::FullScan` marker (#174 F5). It composes with +/// the env kill-switch (either forces), and it disqualifies the empty-tips fast +/// path: a forced-scan call with no tips must still enumerate the repo, never +/// silently resolve to an empty delta (which would pin nothing and lose the +/// drained pushes' work — the F5 bug shape). pub async fn resolve_candidates_for_push( scan_sem: std::sync::Arc, repo_path: PathBuf, @@ -295,11 +302,13 @@ pub async fn resolve_candidates_for_push( old_tips: Vec, git_bin: String, timeout: Duration, + force_full_scan: bool, ) -> PinCandidateSet { + let force = force_full_scan || self::force_full_scan(); // No-git fast path: a deletion-only push resolves to an empty delta without - // spawning any git child, so it must not park on the scan pool. The - // kill-switch disqualifies it — forcing the full scan spawns git. - if new_tips.is_empty() && !force_full_scan() { + // spawning any git child, so it must not park on the scan pool. A forced + // full scan (caller flag or kill-switch) disqualifies it — the scan spawns git. + if new_tips.is_empty() && !force { tracing::info!(delta = 0usize, repo = %repo_path.display(), "pin candidate set from push delta"); return PinCandidateSet { candidates: Vec::new(), @@ -327,7 +336,16 @@ pub async fn resolve_candidates_for_push( let deadline = Instant::now() + timeout; let new_refs: Vec<&str> = new_tips.iter().map(String::as_str).collect(); let old_refs: Vec<&str> = old_tips.iter().map(String::as_str).collect(); - match resolve_push_delta(&repo_path, &new_refs, &old_refs, &git_bin, deadline) { + // `force` already folds in the env kill-switch, so the forced arm skips the + // delta machinery outright; the unforced arm goes through the normal + // resolver (whose own env read is false here by construction). + let resolved = if force { + tracing::debug!("full scan forced (coalesced-drain marker or kill-switch)"); + PinCandidates::FullScanRequired + } else { + resolve_push_delta(&repo_path, &new_refs, &old_refs, &git_bin, deadline) + }; + match resolved { PinCandidates::Delta(objs) => { tracing::info!(delta = objs.len(), repo = %repo_path.display(), "pin candidate set from push delta"); PinCandidateSet { candidates: objs, full_scan: false } @@ -586,6 +604,7 @@ mod tests { vec![c1.clone()], "git".to_string(), std::time::Duration::from_secs(600), + false, ) .await; assert!(!set.full_scan, "happy-path delta is not a full scan"); @@ -616,6 +635,7 @@ mod tests { vec![], "git".to_string(), std::time::Duration::from_secs(600), + false, ) .await; assert!(set.full_scan, "non-commit tip is signalled as a full scan"); @@ -665,6 +685,7 @@ mod tests { vec![], git_bin.clone(), Duration::from_secs(5), + false, ), ) .await; @@ -686,6 +707,7 @@ mod tests { vec![], git_bin, Duration::from_secs(5), + false, ) .await; assert!( @@ -730,6 +752,7 @@ mod tests { vec!["deadbeefdeadbeefdeadbeefdeadbeefdeadbeef".to_string()], fake.to_str().unwrap().to_string(), Duration::from_secs(5), + false, ), ) .await @@ -744,6 +767,56 @@ mod tests { assert!(!marker.exists(), "the fast path must spawn no git at all"); } + /// #174 F5: the coalesced-drain FullScan marker is signalled via the explicit + /// `force_full_scan` flag, and a flagged call with NO tips must still enumerate + /// the whole repo (full_scan=true, non-empty candidates). The RED arm of the + /// encoding question: if the marker were encoded as a plain empty-tips call + /// (flag off — the fast path), the drain would resolve to an empty delta and + /// pin nothing, silently losing the coalesced pushes' work. + #[tokio::test] + async fn forced_full_scan_with_no_tips_enumerates_the_repo() { + let repo = Repo::new(); + repo.commit_file("a.txt", "one\n"); + let all: HashSet = list_all_objects(&repo.path, "git", td()) + .unwrap() + .into_iter() + .collect(); + assert!(!all.is_empty(), "fixture repo has objects"); + + let set = resolve_candidates_for_push( + std::sync::Arc::new(tokio::sync::Semaphore::new(64)), + repo.path.clone(), + vec![], + vec![], + "git".to_string(), + std::time::Duration::from_secs(600), + true, + ) + .await; + assert!(set.full_scan, "the forced call is signalled as a full scan"); + let got: HashSet = set.candidates.into_iter().collect(); + assert_eq!( + got, all, + "a forced full scan with no tips enumerates the repo — never an empty delta" + ); + + // The discriminator: the SAME empty-tips call without the flag is the + // deletion-only fast path (empty delta, pin nothing). The two must differ, + // or the marker encoding has collapsed into the silent-loss shape. + let unforced = resolve_candidates_for_push( + std::sync::Arc::new(tokio::sync::Semaphore::new(64)), + repo.path.clone(), + vec![], + vec![], + "git".to_string(), + std::time::Duration::from_secs(600), + false, + ) + .await; + assert!(!unforced.full_scan); + assert!(unforced.candidates.is_empty()); + } + #[test] fn missing_oid_tip_forces_full_scan() { let repo = Repo::new(); diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 0badf802..4f8ffb0e 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -132,8 +132,9 @@ pub struct AppState { /// detached tasks *spawn and park* on that semaphore's `acquire_owned().await`. /// Before spawning a per-push encryption task, the receive-pack handler consults /// this set: if the repo already has a task in flight it coalesces (skips the - /// duplicate spawn) rather than parking a new unbounded waiter. Coalescing only - /// delays a duplicate walk — it never drops the withheld-blob recovery copy, which + /// duplicate spawn) rather than parking a new unbounded waiter, and its tip pairs + /// are recorded for that task's drain loop (#174 F5). Coalescing only delays the + /// coalesced push's walk — it never drops the withheld-blob recovery copy, which /// `2a54c15` deliberately kept fail-closed (there is no reconciliation sweep to /// re-derive a dropped copy). See [`EncryptInflight`]. pub encrypt_inflight: EncryptInflight, @@ -230,23 +231,80 @@ impl AppState { /// unbounded outstanding set. /// /// This tracks the repo keys with an in-flight encryption task. Before spawning, the -/// handler calls [`try_begin`](Self::try_begin): if a task for the repo is already -/// in-flight it returns `None` and the handler SKIPS spawning a duplicate (coalesce — -/// the newer push's objects are covered by the pending/next walk over the same repo's -/// history). This bounds the outstanding set to <=1 pending task per repo WITHOUT -/// dropping work: coalescing only delays a duplicate walk, it never sheds the recovery -/// copy (there is no reconciliation sweep, so a *dropped* job would be lost forever). +/// handler calls [`try_begin`](Self::try_begin) with the push's (old, new) tip pairs: +/// if no task is in-flight the push is [`Admitted`](BeginOutcome::Admitted) and spawns +/// one; if a task IS in-flight the push [`Coalesces`](BeginOutcome::Coalesced) — no +/// duplicate spawn — and its tip pairs are merged into the in-flight key's pending +/// slot in the SAME critical section as the presence check. The in-flight task pins +/// only its own pre-spawn object-list snapshot, so the merge is what keeps coalescing +/// lossless (#174 F5): the task loop-drains the pending slot via +/// [`EncryptInflightGuard::finish_or_take_pending`] before releasing the key, so a +/// coalesced push's pins and recovery copies are delayed, never dropped (there is no +/// reconciliation sweep, so a *dropped* job would be lost forever). Check-then-record +/// as two lock acquisitions would race the task's final pending check — hence one +/// critical section for both. /// -/// The returned [`EncryptInflightGuard`] is moved into the detached task and removes -/// the repo key on drop — on normal completion, error, OR panic (Drop runs on unwind) -/// — so one crashed walk can never permanently lock a repo out of future recovery -/// copies. +/// The returned [`EncryptInflightGuard`] is moved into the detached task. On normal +/// exit the key is removed (and the guard disarmed) inside `finish_or_take_pending`'s +/// empty-pending critical section; the guard's Drop is the PANIC backstop (Drop runs +/// on unwind), so one crashed walk can never permanently lock a repo out of future +/// recovery copies. #[derive(Clone, Default)] pub struct EncryptInflight { - // std::sync::Mutex: only ever held for O(1) HashSet insert/remove in a sync - // context (right before `tokio::spawn`, and in the guard's Drop) — never across - // an await, so a std Mutex is correct and cheaper than a tokio one. - repos: Arc>>, + // std::sync::Mutex: only ever held for O(1)-ish map ops (insert/remove/merge — + // the merge is an O(pairs) Vec extend bounded by MAX_PENDING_TIP_PAIRS) in a + // sync context, never across an await, so a std Mutex is correct and cheaper + // than a tokio one. Key present == task in flight; the value is the work + // recorded by pushes that coalesced against it. + repos: Arc>>, +} + +/// Cap on the accumulated coalesced tip pairs per repo. Past it the pending slot +/// degrades to [`PendingWork::FullScan`], so a hostile pusher cannot grow the slot +/// without bound while a walk is in flight; the drain then costs one full-repo +/// enumeration instead (the same already-tested fallback the push path uses). +const MAX_PENDING_TIP_PAIRS: usize = 1024; + +/// Work recorded by pushes that coalesced against an in-flight encryption task, +/// drained by that task one batch per loop iteration. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PendingWork { + /// The coalesced pushes' raw (old_sha, new_sha) ref-update pairs, zeros + /// included — the drain strips the create/delete sentinels exactly like the + /// handler tail does. An EMPTY vec is "nothing pending", never a work item. + Tips(Vec<(String, String)>), + /// The pair bound overflowed: drain with a FORCED full-repo scan. This must be + /// signalled explicitly (the `force_full_scan` flag on + /// `resolve_candidates_for_push`), never encoded as an empty tip list — empty + /// tips resolve to an empty delta and would pin nothing (the F5 loss again). + FullScan, +} + +/// Outcome of [`EncryptInflight::try_begin`]. +pub enum BeginOutcome { + /// No task was in flight: the caller spawns one, moving the guard into it. The + /// push's own tip pairs are NOT recorded — the caller's pre-spawn snapshot + /// covers them; the pending slot starts empty. + Admitted(EncryptInflightGuard), + /// A task is in flight; this push's tip pairs were merged into its pending + /// slot (same critical section as the presence check). The in-flight task's + /// drain loop will process them. + Coalesced, +} + +/// Outcome of [`EncryptInflightGuard::finish_or_take_pending`]. +pub enum FinishOutcome { + /// Coalesced work was pending: it is handed back with the still-armed guard + /// (the repo key is retained) and the task must run another drain iteration. + Pending(EncryptInflightGuard, PendingWork), + /// Nothing was pending: the repo key was removed AND the guard disarmed in one + /// critical section, so dropping the returned guard is inert. The task exits. + /// Remove-then-drop as two steps would double-remove: a successor task admitted + /// between them would have ITS key deleted by the late Drop. The disarmed guard + /// is handed back rather than dropped internally so that remove→drop window is + /// real and the disarm is testable; production just lets it fall out of scope + /// (hence the allow). + Finished(#[allow(dead_code)] EncryptInflightGuard), } impl EncryptInflight { @@ -254,19 +312,25 @@ impl EncryptInflight { Self::default() } - /// Try to begin an encryption task for `repo_id`. Returns `Some(guard)` if no task - /// for the repo was in-flight (the caller should spawn), or `None` if one already - /// is (the caller should COALESCE — skip spawning a duplicate). The guard releases - /// the repo key on drop. - pub fn try_begin(&self, repo_id: &str) -> Option { - let mut set = self.repos.lock().expect("encrypt_inflight mutex poisoned"); - if set.insert(repo_id.to_string()) { - Some(EncryptInflightGuard { - repos: Arc::clone(&self.repos), - repo_id: repo_id.to_string(), - }) - } else { - None + /// Begin-or-coalesce an encryption task for `repo_id`, in one critical section. + /// `tip_pairs` is this push's raw (old_sha, new_sha) ref-update list; it is + /// merged into the pending slot only on the [`Coalesced`](BeginOutcome::Coalesced) + /// arm (an admitted caller's own snapshot already covers its pairs). + pub fn try_begin(&self, repo_id: &str, tip_pairs: Vec<(String, String)>) -> BeginOutcome { + let mut map = self.repos.lock().expect("encrypt_inflight mutex poisoned"); + match map.entry(repo_id.to_string()) { + std::collections::hash_map::Entry::Vacant(slot) => { + slot.insert(PendingWork::Tips(Vec::new())); + BeginOutcome::Admitted(EncryptInflightGuard { + repos: Arc::clone(&self.repos), + repo_id: repo_id.to_string(), + armed: true, + }) + } + std::collections::hash_map::Entry::Occupied(mut slot) => { + merge_pending(slot.get_mut(), tip_pairs); + BeginOutcome::Coalesced + } } } @@ -287,21 +351,95 @@ impl EncryptInflight { } } -/// RAII guard removing a repo key from [`EncryptInflight`] when the detached -/// encryption task finishes (drop on completion, error, or panic-unwind). Move-only — -/// there is no reason to clone a guard, and cloning would double-remove. +/// Merge a coalesced push's tip pairs into a repo's pending slot. FullScan absorbs +/// everything; a Tips slot that would exceed [`MAX_PENDING_TIP_PAIRS`] degrades to +/// FullScan rather than growing without bound. +fn merge_pending(slot: &mut PendingWork, pairs: Vec<(String, String)>) { + match slot { + PendingWork::FullScan => {} + PendingWork::Tips(acc) => { + if acc.len().saturating_add(pairs.len()) > MAX_PENDING_TIP_PAIRS { + *slot = PendingWork::FullScan; + } else { + acc.extend(pairs); + } + } + } +} + +/// Guard owned by the detached encryption task for its repo key. Move-only — there +/// is no reason to clone a guard, and cloning would double-remove. Normal exit goes +/// through [`finish_or_take_pending`](Self::finish_or_take_pending); Drop is the +/// panic-path backstop only. pub struct EncryptInflightGuard { - repos: Arc>>, + repos: Arc>>, repo_id: String, + /// True until the normal-exit path removes the key. A disarmed guard's Drop is + /// a no-op: the key slot may already belong to a successor task admitted after + /// our removal, and removing THAT key would break at-most-one-task-per-repo. + armed: bool, +} + +impl EncryptInflightGuard { + /// The task's end-of-iteration step, one critical section: if coalesced work is + /// pending, take it and hand the still-armed guard back (key retained — iterate); + /// if nothing is pending, remove the key and disarm the guard (task exits; the + /// returned guard's Drop is inert). The atomicity is load-bearing both ways: a + /// push landing before this call is merged and therefore drained here; a push + /// landing after it finds the key gone and is admitted as a fresh task. No + /// interleaving can lose the work or admit two tasks for one repo. + pub fn finish_or_take_pending(mut self) -> FinishOutcome { + let mut map = self.repos.lock().expect("encrypt_inflight mutex poisoned"); + match map.get_mut(&self.repo_id) { + Some(PendingWork::Tips(acc)) if acc.is_empty() => { + map.remove(&self.repo_id); + self.armed = false; + drop(map); + FinishOutcome::Finished(self) + } + Some(slot) => { + let work = std::mem::replace(slot, PendingWork::Tips(Vec::new())); + drop(map); + FinishOutcome::Pending(self, work) + } + None => { + // Unreachable while armed (only this method removes a live key), + // but never panic in the release path: treat as finished. + self.armed = false; + drop(map); + FinishOutcome::Finished(self) + } + } + } } impl Drop for EncryptInflightGuard { fn drop(&mut self) { - // A poisoned lock (a prior panic while holding it) still lets us take the inner - // set via into_inner-on-guard; but poisoning here is not expected because the - // only critical sections are the O(1) ops above. Remove best-effort. - if let Ok(mut set) = self.repos.lock() { - set.remove(&self.repo_id); + // Normal exit disarmed us inside finish_or_take_pending's critical section; + // an armed drop means the task ended abnormally (panic-unwind, or a future + // code path that returns without finishing). Release the key so the repo is + // not permanently locked out, and log any pending work this loses — there + // is no sweep, so it stays lost until a later push re-walks the repo. + if !self.armed { + return; + } + // A poisoned lock is not expected (the critical sections above are small + // and panic-free); remove best-effort. + if let Ok(mut map) = self.repos.lock() { + match map.remove(&self.repo_id) { + Some(PendingWork::Tips(acc)) if !acc.is_empty() => tracing::warn!( + repo = %self.repo_id, + lost_tip_pairs = acc.len(), + "encryption task ended abnormally with coalesced pushes pending; \ + their pins/recovery copies are lost until a later push" + ), + Some(PendingWork::FullScan) => tracing::warn!( + repo = %self.repo_id, + "encryption task ended abnormally with a pending full-scan drain; \ + it is lost until a later push" + ), + _ => {} + } } } } diff --git a/crates/gitlawb-node/tests/inv22_gates.rs b/crates/gitlawb-node/tests/inv22_gates.rs index bcd0758e..75285f21 100644 --- a/crates/gitlawb-node/tests/inv22_gates.rs +++ b/crates/gitlawb-node/tests/inv22_gates.rs @@ -88,6 +88,23 @@ fn inv22_concurrency_gates_present_and_not_bypassed() { encrypt_inflight.try_begin to coalesce per repo (bound the outstanding-task set)" ); + // F5: coalescing must REQUEUE, not shed. The in-flight task pins only its own + // pre-spawn snapshot, so a coalesced push's tip pairs are recorded on its key and + // the task must loop-drain them (`finish_or_take_pending`) before releasing it. + // Removing the drain reverts to the silent loss: a coalesced push's pins and + // recovery copies are absent until an unrelated later push. Scan only the + // production half of the file — the u5 tests in its `mod tests` also name the + // drain call, and matching them would make this check vacuous. + let repos_production = repos + .split("#[cfg(test)]") + .next() + .expect("split always yields a first chunk"); + assert!( + repos_production.contains("finish_or_take_pending"), + "F5 gate missing: the post-push encryption task must loop-drain coalesced \ + pushes via finish_or_take_pending before releasing its repo key" + ); + // F4: every post-receive scan helper acquires a `git_encrypt_semaphore` permit // BEFORE its spawn_blocking git walk, so a push burst cannot accumulate unbounded // concurrent scans once the write permit is released. Structural check: within each From 48ed0a741f16904e15124033c0618480be3a9932 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 17 Jul 2026 20:16:04 -0500 Subject: [PATCH 43/58] fix(review): close the round's review findings on the #174 fix series object_type now distinguishes repo-level git failures (not-a-repository, error:-prefixed corruption) from genuine object absence, so a corrupt repo taints the /ipfs scan into the retryable 503 instead of vouching for a definitive 404; the drain reads visibility rules by the freshly fetched record id, closing a latent fail-open on id divergence. The four budget gates collapse into one stage-labelled helper, the three scan sites share state::acquire_scan_permit (inv22 tripwire repointed and sever-verified), and the Tigris stall tests use a local accept-and-park listener instead of a non-routable address. --- crates/gitlawb-node/src/api/ipfs.rs | 229 ++++++++++++++++------ crates/gitlawb-node/src/api/repos.rs | 50 ++--- crates/gitlawb-node/src/git/push_delta.rs | 21 +- crates/gitlawb-node/src/git/store.rs | 16 +- crates/gitlawb-node/src/state.rs | 31 +++ crates/gitlawb-node/tests/inv22_gates.rs | 33 +++- 6 files changed, 259 insertions(+), 121 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index 28b9cc67..014411e7 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -182,14 +182,35 @@ pub async fn get_by_cid( } } - // Remaining request budget (F3), or None once exhausted. A stage is never - // started with zero remaining; the probe and read subprocesses carry no - // internal duration clamp, so the check before them is their entire bound - // (documented overshoot: at most one unclamped cat-file plus the in-flight - // clamped stage's kill/reap slack). - fn budget_remaining(deadline: std::time::Instant) -> Option { + // Budget gate shared by the four per-stage checks (F3): the remaining + // request budget, or — once exhausted — None, after logging the stage and + // knob and tainting "budget"; the call site only breaks (the scan STOPS, + // leaving this and every later candidate unproven, never a false 404). A + // stage is never started with zero remaining; the probe and read + // subprocesses carry no internal duration clamp, so this pre-start check is + // their entire bound (documented overshoot: at most one unclamped cat-file + // plus the in-flight clamped stage's kill/reap slack). The acquire and walk + // stages clamp their deadlines to the returned remainder. + fn budget_gate( + truncated_by: &mut Vec<&'static str>, + deadline: std::time::Instant, + budget_secs: u64, + repo_name: &str, + stage: &'static str, + ) -> Option { let left = deadline.saturating_duration_since(std::time::Instant::now()); - (!left.is_zero()).then_some(left) + if left.is_zero() { + tracing::warn!( + repo = %repo_name, + stage, + budget_secs, + "/ipfs request budget exhausted before the stage \ + (GITLAWB_IPFS_REQUEST_BUDGET_SECS); stopping the scan without a verdict" + ); + taint(truncated_by, "budget"); + return None; + } + Some(left) } // Cap on EXPENSIVE walks only (F2): counts the repos that actually require the @@ -216,18 +237,15 @@ pub async fn get_by_cid( continue; } - // Budget gate for the acquire stage (F3): once the request budget is - // exhausted the scan STOPS, leaving this and every later candidate - // unproven (taint, never a false 404). Checked ahead of the visit + // Budget gate for the acquire stage (F3), checked ahead of the visit // bookkeeping so an unstarted acquire is not counted as a visit. - let Some(budget_left) = budget_remaining(request_deadline) else { - tracing::warn!( - repo = %repo.name, - budget_secs = state.config.ipfs_request_budget_secs, - "/ipfs request budget exhausted before the repo acquire \ - (GITLAWB_IPFS_REQUEST_BUDGET_SECS); stopping the scan without a verdict" - ); - taint(&mut truncated_by, "budget"); + let Some(budget_left) = budget_gate( + &mut truncated_by, + request_deadline, + state.config.ipfs_request_budget_secs, + &repo.name, + "repo acquire", + ) else { break; }; @@ -276,13 +294,15 @@ pub async fn get_by_cid( // Budget gate for the probe stage (F3). The probe subprocess has no // internal duration clamp, so this pre-start check is its entire bound. - if budget_remaining(request_deadline).is_none() { - tracing::warn!( - repo = %repo.name, - "/ipfs request budget exhausted before the object-type probe; \ - stopping the scan without a verdict" - ); - taint(&mut truncated_by, "budget"); + if budget_gate( + &mut truncated_by, + request_deadline, + state.config.ipfs_request_budget_secs, + &repo.name, + "object-type probe", + ) + .is_none() + { break; } @@ -311,14 +331,13 @@ pub async fn get_by_cid( // started walk runs its git children under a deadline clamped // to the remainder (the min below), so a walk can never // complete past the budget. - let Some(budget_left) = budget_remaining(request_deadline) else { - tracing::warn!( - repo = %repo.name, - budget_secs = state.config.ipfs_request_budget_secs, - "/ipfs request budget exhausted before the visibility walk \ - (GITLAWB_IPFS_REQUEST_BUDGET_SECS); stopping the scan without a verdict" - ); - taint(&mut truncated_by, "budget"); + let Some(budget_left) = budget_gate( + &mut truncated_by, + request_deadline, + state.config.ipfs_request_budget_secs, + &repo.name, + "visibility walk", + ) else { break; }; // Walk cap (F2), checked at the one site that actually spends a walk: @@ -394,13 +413,15 @@ pub async fn get_by_cid( // serving keeps the terminal arm honest and the stop unconditional; the // retryable 503 tells the caller to come back rather than letting an // over-budget request keep spending. - if budget_remaining(request_deadline).is_none() { - tracing::warn!( - repo = %repo.name, - "/ipfs request budget exhausted before the content read; \ - stopping the scan without a verdict" - ); - taint(&mut truncated_by, "budget"); + if budget_gate( + &mut truncated_by, + request_deadline, + state.config.ipfs_request_budget_secs, + &repo.name, + "content read", + ) + .is_none() + { break; } @@ -600,6 +621,24 @@ mod tests { .to_string() } + /// A local endpoint whose TCP accept succeeds instantly but that never writes + /// an HTTP response, so a Tigris HEAD against it stalls deterministically + /// until the caller's timeout. (A non-routable address hangs only if the + /// network blackholes the SYN — a fast RST would end the stall early.) The + /// accepted sockets are parked in the spawned task, which dies with the + /// test's runtime, so the peer never sees a close mid-test. + async fn silent_tigris_endpoint() -> String { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + tokio::spawn(async move { + let mut held = Vec::new(); + while let Ok((sock, _)) = listener.accept().await { + held.push(sock); + } + }); + endpoint + } + /// Fake git for the WALK only (`state.git_bin`): empty refs, `rev-parse` /// resolves, and each `rev-list` appends one line to `log` and prints nothing — /// every walked repo yields an EMPTY allowed-set (path-gate deny verdict) and @@ -854,9 +893,9 @@ mod tests { } /// F2 acquire taint: a repo row with NO local copy over a Tigris backend that - /// stalls (non-routable endpoint — the connect just hangs) hits the 1s acquire - /// timeout at the read-acquire site. The skip carries no verdict, so the scan is - /// truncated: retryable 503 + Retry-After, never the old silent-skip 404. + /// stalls (a silent local endpoint — accepted, never answered) hits the 1s + /// acquire timeout at the read-acquire site. The skip carries no verdict, so the + /// scan is truncated: retryable 503 + Retry-After, never the old silent-skip 404. /// MUTATION (RED): drop the taint on the acquire-timeout arm and this decays to /// a 404. #[sqlx::test] @@ -866,12 +905,12 @@ mod tests { std::fs::create_dir_all(&repos_dir).unwrap(); let mut state = crate::test_support::test_state(pool.clone()).await; // Endpoint-pinned test client (no AWS_* env reads — env is racy under a - // parallel test run); 10.255.255.1 is non-routable so the HEAD hangs. - let tigris = crate::git::tigris::TigrisClient::for_testing_with_endpoint( - "test-bucket", - "http://10.255.255.1:9000", - ) - .await; + // parallel test run); the silent local endpoint stalls the HEAD + // deterministically. + let endpoint = silent_tigris_endpoint().await; + let tigris = + crate::git::tigris::TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint) + .await; state.repo_store = crate::git::repo_store::RepoStore::new(repos_dir, Some(tigris), pool); state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; let mut cfg = (*state.config).clone(); @@ -911,9 +950,10 @@ mod tests { /// `object_type` is Err. That is not an absence verdict, so the scan is /// truncated: 503, never 404. A second, real repo probes clean (absent verdict) /// — the one bad row is what taints. NOTE: the probe shells to the real `git` - /// (not `state.git_bin`) and maps a NONZERO cat-file exit to `Ok(None)` (an - /// absent verdict), so a fake git exiting nonzero cannot reach this arm; only a - /// spawn failure is Err, hence the missing-dir recipe. MUTATION (RED): drop the + /// (not `state.git_bin`), and a clean missing/invalid-object nonzero exit is + /// still `Ok(None)` (an absent verdict) — this arm needs a probe that could + /// not RUN, hence the missing-dir spawn failure here; the corrupt-repo test + /// below drives the stderr-discriminated Err. MUTATION (RED): drop the /// taint on the probe-error arm and this decays to a 404. #[sqlx::test] async fn get_by_cid_probe_error_taints_scan_to_503(pool: sqlx::PgPool) { @@ -951,6 +991,76 @@ mod tests { ); } + /// F2 probe taint, corrupt-repo arm: a repo whose git dir EXISTS but is broken + /// (objects/ removed, HEAD garbage) makes the real `cat-file -t` die with the + /// repo-level `fatal: not a git repository` — a probe that could not examine + /// the object store, not an absence verdict, so `object_type` must map it to + /// Err and the scan must shed the probe-tainted 503, never the silent-absence + /// 404. A second, real repo probes clean (absent verdict) — the corrupt row is + /// what taints. MUTATION (RED): map every nonzero cat-file exit back to + /// `Ok(None)` in `object_type` (drop the stderr discrimination) and this + /// decays to a 404. + #[sqlx::test] + async fn get_by_cid_corrupt_repo_dir_probe_error_taints_scan_to_503(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + // Older row: a real repo that probes clean. Newer row: a bare repo whose + // git dir exists on disk but is corrupt at the repo level. + seed_repo_with_blob(&state, tmp.path(), "z6f2corrupt", "real", b"probe clean\n").await; + state + .db + .upsert_mirror_repo("z6f2corrupt", "broken", "/unused-broken", None, false) + .await + .unwrap(); + let rec = state + .db + .get_repo("z6f2corrupt", "broken") + .await + .unwrap() + .unwrap(); + let bare = state + .repo_store + .acquire(&rec.owner_did, &rec.name) + .await + .unwrap(); + std::fs::create_dir_all(&bare).unwrap(); + run_git(&["init", "-q", "--bare", "--object-format=sha256"], &bare); + std::fs::remove_dir_all(bare.join("objects")).unwrap(); + std::fs::write(bare.join("HEAD"), b"junk\n").unwrap(); + + let peer: SocketAddr = "203.0.113.68:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&valid_cid(), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a repo-level cat-file fatal leaves the repo unproven — the scan must \ + shed 503, not report the object absent" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the truncation 503 must carry Retry-After" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + let body = String::from_utf8_lossy(&body); + assert!( + body.contains("probe"), + "the shed must name the probe taint; got: {body}" + ); + } + /// F2 read taint: the gate passes (the probe reads the truncated loose object's /// intact "blob 64" header) but the content read fails (`cat-file blob` dies on /// the deflate stream cut mid-content) — the probe just said the object EXISTS @@ -1074,7 +1184,7 @@ mod tests { /// (`ipfs_request_budget_secs`) bounds the whole admitted scan; per-repo /// stages may not each draw a fresh timeout past it. Budget 1s, per-iteration /// acquire timeout 2s; the NEWER row is a Tigris-backed ghost (no local copy, - /// non-routable endpoint) whose acquire stalls, the OLDER row is a plain + /// silent local endpoint) whose acquire stalls, the OLDER row is a plain /// public repo carrying the blob. The ghost's acquire runs clamped to the ~1s /// remainder and times out; at the next repo the budget gate sees zero /// remaining, taints "budget", and STOPS the scan, so the blob repo is never @@ -1105,13 +1215,12 @@ mod tests { .await; // Swap in a Tigris-backed store over the SAME repos_dir (the seeded bare // repo stays a fast local hit) and add a NEWER ghost row with no local - // copy: its acquire consults the non-routable endpoint and stalls - // (endpoint-pinned test client, no AWS_* env reads; 10.255.255.1 hangs). - let tigris = crate::git::tigris::TigrisClient::for_testing_with_endpoint( - "test-bucket", - "http://10.255.255.1:9000", - ) - .await; + // copy: its acquire consults the silent local endpoint and stalls past + // the budget (endpoint-pinned test client, no AWS_* env reads). + let endpoint = silent_tigris_endpoint().await; + let tigris = + crate::git::tigris::TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint) + .await; state.repo_store = crate::git::repo_store::RepoStore::new(repos_dir, Some(tigris), pool); state .db diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 2b9cf1d4..2770ffb9 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -74,30 +74,14 @@ async fn replication_withheld_set( // that off the async worker thread. Some(rules) => { let owner_did = owner_did.to_string(); - // Scan admission (#174 F4): DEFER (await), never shed — dropping the - // walk would skip the vetting and fail the push's replication closed - // for no reason. Accepted residuals, stated honestly: (1) the park - // wait is queue-depth multiplied — post-receive tails are no longer - // admission-bounded once the write permit is released, so N landed - // pushes can queue N walks and the last waits N walk-durations; - // (2) a client-timeout disconnect while parked HERE drops the - // handler future and silently loses this push's replication work — - // the encrypt_inflight coalescing requeue does NOT cover it, because - // this park precedes the `try_begin` spawn gate. - let parked = std::time::Instant::now(); - let permit = encrypt_sem - .acquire_owned() - .await - .expect("git_encrypt_semaphore is never closed"); - tracing::debug!( - repo = %disk_path.display(), - queue_wait_ms = parked.elapsed().as_millis() as u64, - "post-push withheld walk admitted to the scan pool" - ); + // Scan admission (#174 F4): DEFER, never shed — dropping the walk + // would skip the vetting and fail the push's replication closed for + // no reason. Residuals at `acquire_scan_permit`. + let permit = + crate::state::acquire_scan_permit(encrypt_sem, &disk_path, "withheld walk").await; tokio::task::spawn_blocking(move || { // The permit lives inside the blocking closure: a started walk - // always completes holding it (a disconnect cannot cancel - // spawn_blocking or leak the permit mid-walk). + // always completes holding it. let _permit = permit; crate::git::visibility_pack::withheld_blob_oids_bounded( &disk_path, &git_bin, timeout, &rules, is_public, &owner_did, None, @@ -137,9 +121,7 @@ async fn replication_withheld_set( /// /// Always walks (there is no no-git arm), so the whole blocking scan runs under /// one `git_encrypt_semaphore` admission permit (#174 F4) — see -/// `replication_withheld_set`'s acquire for the defer rationale and the honest -/// residuals (queue-depth-multiplied park wait; a disconnect while parked loses -/// this push's replication work, uncovered by the coalescing requeue). +/// `acquire_scan_permit` for the defer rationale and the honest residuals. #[allow(clippy::too_many_arguments)] async fn fail_closed_full_scan_objects( encrypt_sem: std::sync::Arc, @@ -153,16 +135,8 @@ async fn fail_closed_full_scan_objects( ) -> Vec { // Scan admission (#174 F4): DEFER, never shed; the permit moves into the // closure so a started scan always completes holding it. - let parked = std::time::Instant::now(); - let permit = encrypt_sem - .acquire_owned() - .await - .expect("git_encrypt_semaphore is never closed"); - tracing::debug!( - repo = %disk_path.display(), - queue_wait_ms = parked.elapsed().as_millis() as u64, - "post-push fail-closed full scan admitted to the scan pool" - ); + let permit = + crate::state::acquire_scan_permit(encrypt_sem, &disk_path, "fail-closed full scan").await; tokio::task::spawn_blocking(move || -> anyhow::Result> { let _permit = permit; let allowed = crate::git::visibility_pack::replicable_blob_set_bounded( @@ -925,7 +899,11 @@ async fn resolve_drain_object_list( return None; } }; - let rules_opt = ctx.db.list_visibility_rules(&ctx.repo_id).await.ok(); + // record.id, never the spawn-time ctx.repo_id: the record above is re-fetched + // fresh by owner/name, and a delete+re-create between spawn and drain gives + // the row a NEW id — rules read against the stale id come back empty and + // would fail open for the new row. + let rules_opt = ctx.db.list_visibility_rules(&record.id).await.ok(); let (_announce, withheld) = replication_withheld_set( ctx.encrypt_sem.clone(), rules_opt.clone(), diff --git a/crates/gitlawb-node/src/git/push_delta.rs b/crates/gitlawb-node/src/git/push_delta.rs index 27064357..0b569693 100644 --- a/crates/gitlawb-node/src/git/push_delta.rs +++ b/crates/gitlawb-node/src/git/push_delta.rs @@ -315,21 +315,12 @@ pub async fn resolve_candidates_for_push( full_scan: false, }; } - // Scan admission (#174 F4): DEFER (await), never shed — a dropped scan would - // silently under-pin this push. The permit moves into the blocking closure so - // a started scan always completes holding it. Residuals as documented at - // `replication_withheld_set`'s acquire: park wait is queue-depth multiplied, - // and a client disconnect while parked loses this push's replication work. - let parked = Instant::now(); - let permit = scan_sem - .acquire_owned() - .await - .expect("git_encrypt_semaphore is never closed"); - tracing::debug!( - repo = %repo_path.display(), - queue_wait_ms = parked.elapsed().as_millis() as u64, - "pin-candidate scan admitted to the scan pool" - ); + // Scan admission (#174 F4): DEFER, never shed — a dropped scan would + // silently under-pin this push. The permit moves into the blocking closure + // so a started scan always completes holding it. Residuals at + // `acquire_scan_permit`. + let permit = + crate::state::acquire_scan_permit(scan_sem, &repo_path, "pin-candidate scan").await; tokio::task::spawn_blocking(move || { let _permit = permit; // ONE shared deadline for the whole scan, per jatmn ("the same deadline"). diff --git a/crates/gitlawb-node/src/git/store.rs b/crates/gitlawb-node/src/git/store.rs index 229ee695..5697c229 100644 --- a/crates/gitlawb-node/src/git/store.rs +++ b/crates/gitlawb-node/src/git/store.rs @@ -271,7 +271,8 @@ pub struct TreeEntry { /// `/ipfs/` is computed from these same content bytes via /// `gitlawb_core::cid::Cid::from_git_object_bytes`. /// -/// Get just the object type. Returns `None` if the object doesn't exist. +/// Get just the object type. Returns `None` if the object doesn't exist; a +/// probe that could not examine the object store is `Err`, never `None`. pub fn object_type(repo_path: &Path, sha256_hex: &str) -> Result> { let type_output = Command::new("git") .args(["cat-file", "-t", sha256_hex]) @@ -280,6 +281,19 @@ pub fn object_type(repo_path: &Path, sha256_hex: &str) -> Result> .context("failed to run git cat-file -t")?; if !type_output.status.success() { + // A nonzero exit is an ABSENCE verdict only when git could examine the + // object store: missing-object and invalid-oid probes die with a single + // clean `fatal:` line. A broken repo dir (`fatal: not a git repository`) + // or a corrupt object (`error: inflate` / `error: unable to unpack` + // lines before the fatal) proves nothing about absence, so it must + // surface as Err — the /ipfs scan taints on Err rather than treating + // the repo as probed-clean. + let stderr = String::from_utf8_lossy(&type_output.stderr); + if stderr.contains("not a git repository") + || stderr.lines().any(|l| l.starts_with("error:")) + { + bail!("git cat-file -t failed: {}", stderr.trim()); + } return Ok(None); } diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 4f8ffb0e..c4d2b079 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -443,3 +443,34 @@ impl Drop for EncryptInflightGuard { } } } + +/// Admit a post-receive git scan to the shared `git_encrypt_semaphore` pool +/// (#174 F4): DEFER (await), never shed — a dropped scan would lose the push's +/// recovery copy or silently under-pin it. The returned permit must move into +/// the blocking closure so a started scan always completes holding it (a +/// disconnect cannot cancel `spawn_blocking` or leak the permit mid-walk). +/// Accepted residuals, stated once for every caller: (1) the park wait is +/// queue-depth multiplied — post-receive tails are no longer admission-bounded +/// once the write permit is released, so N landed pushes can queue N scans and +/// the last waits N scan-durations; (2) a client-timeout disconnect while +/// parked HERE drops the handler future and silently loses this push's +/// replication work — the `encrypt_inflight` coalescing requeue does NOT cover +/// it, because this park precedes the `try_begin` spawn gate. +pub async fn acquire_scan_permit( + scan_sem: Arc, + repo: &std::path::Path, + stage: &'static str, +) -> tokio::sync::OwnedSemaphorePermit { + let parked = std::time::Instant::now(); + let permit = scan_sem + .acquire_owned() + .await + .expect("git_encrypt_semaphore is never closed"); + tracing::debug!( + repo = %repo.display(), + stage, + queue_wait_ms = parked.elapsed().as_millis() as u64, + "post-receive scan admitted to the scan pool" + ); + permit +} diff --git a/crates/gitlawb-node/tests/inv22_gates.rs b/crates/gitlawb-node/tests/inv22_gates.rs index 75285f21..cbfa0d4c 100644 --- a/crates/gitlawb-node/tests/inv22_gates.rs +++ b/crates/gitlawb-node/tests/inv22_gates.rs @@ -105,12 +105,25 @@ fn inv22_concurrency_gates_present_and_not_bypassed() { pushes via finish_or_take_pending before releasing its repo key" ); - // F4: every post-receive scan helper acquires a `git_encrypt_semaphore` permit - // BEFORE its spawn_blocking git walk, so a push burst cannot accumulate unbounded - // concurrent scans once the write permit is released. Structural check: within each - // helper the first `acquire_owned` precedes the first `spawn_blocking`. Removing a - // gate pushes the next `acquire_owned` occurrence past the helper's own - // `spawn_blocking` (or off the end of the file), turning the assertion red. + // F4: every post-receive scan helper admits itself to the shared scan pool via + // `crate::state::acquire_scan_permit` BEFORE its spawn_blocking git walk, so a + // push burst cannot accumulate unbounded concurrent scans once the write permit + // is released. Two halves, both load-bearing: the helper body must actually + // acquire the pool (state.rs sits the helper at the end of the file, so the + // definition tail contains no other `acquire_owned` to match vacuously), and + // within each scan helper the first qualified call precedes the first + // `spawn_blocking`. Severing a call site pushes the next occurrence past the + // helper's own `spawn_blocking` (or off the end of the file), turning the + // assertion red; comments name the helper without the `crate::state::` prefix, + // so this targets the real call sites. + let state_rs = src("state.rs"); + let helper_def = state_rs + .find("fn acquire_scan_permit") + .expect("F4 gate missing: state.rs no longer defines acquire_scan_permit"); + assert!( + state_rs[helper_def..].contains("acquire_owned"), + "F4 gate gutted: acquire_scan_permit must acquire the scan pool via acquire_owned" + ); let push_delta = src("git/push_delta.rs"); for (file_src, file, helper) in [ (&repos, "api/repos.rs", "fn replication_withheld_set"), @@ -125,9 +138,11 @@ fn inv22_concurrency_gates_present_and_not_bypassed() { .find(helper) .unwrap_or_else(|| panic!("{file}: `{helper}` not found")); let tail = &file_src[start..]; - let acquire = tail.find("acquire_owned").unwrap_or_else(|| { - panic!("F4 gate missing: {file} `{helper}` no longer acquires a scan permit") - }); + let acquire = tail + .find("crate::state::acquire_scan_permit(") + .unwrap_or_else(|| { + panic!("F4 gate missing: {file} `{helper}` no longer acquires a scan permit") + }); let spawn = tail.find("spawn_blocking").unwrap_or_else(|| { panic!("{file}: `{helper}` lost its spawn_blocking walk — update this guard") }); From b6d835d61228c01e7055ba6b9c6188e711d3a6e6 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 17 Jul 2026 20:42:48 -0500 Subject: [PATCH 44/58] fix(review): state the probe/read stages' unbounded-hang residual honestly; net the acquire-taint found path The budget-knob docs claimed the overshoot was bounded by one unclamped probe; an unclamped subprocess is not a bound, so the README, .env.example, config help, and module comments now say plainly that a hung git probe holds the walk slot for the hang's duration. New regression: an acquire-tainting ghost row ahead of a healthy public copy must not stop the scan; the later found short-circuit still serves 200. --- .env.example | 12 ++-- README.md | 2 +- crates/gitlawb-node/src/api/ipfs.rs | 102 ++++++++++++++++++++++++++-- crates/gitlawb-node/src/config.rs | 12 ++-- 4 files changed, 111 insertions(+), 17 deletions(-) diff --git a/.env.example b/.env.example index 94685069..8f9bfa2e 100644 --- a/.env.example +++ b/.env.example @@ -180,11 +180,13 @@ GITLAWB_IPFS_MAX_REPOS_WALKED=64 # Default 1024. GITLAWB_IPFS_MAX_REPO_VISITS=1024 # Absolute wall-clock budget for one admitted /ipfs request's acquire+walk -# lifetime (all stages of the whole scan). Bounds how LONG one request may hold -# its walk slot: per-stage timeouts are clamped to the remaining budget and no -# stage starts once it is exhausted (the scan then sheds a retryable 503). -# Residual overshoot: the in-flight clamped stage's kill/reap slack plus one -# unclamped cat-file probe subprocess. Default 600. +# lifetime (all stages of the whole scan). Per-stage clamps bound the acquire +# and walk stages to the remaining budget, and no stage starts once it is +# exhausted (the scan then sheds a retryable 503). The object-type probe and +# content-read cat-file subprocesses are budget-checked before starting but +# have NO duration bound of their own: a hung git probe (corrupt pack, stuck +# filesystem) holds the request's walk slot for the full duration of the hang. +# Default 600. GITLAWB_IPFS_REQUEST_BUDGET_SECS=600 # Max /ipfs/{cid} requests per client IP per hour (route flood brake, distinct # from the concurrency caps above). 0 disables. Default 600. diff --git a/README.md b/README.md index bc338339..cc32bcf7 100644 --- a/README.md +++ b/README.md @@ -349,7 +349,7 @@ Important node settings: | `GITLAWB_IPFS_WALK_PER_SOURCE` | Max concurrent `/ipfs` walks a single source IP may hold. Default 4. | | `GITLAWB_IPFS_MAX_REPOS_WALKED` | Max expensive path-scope visibility walks per `/ipfs/{cid}` request; over-cap repos are skipped and the scan continues, shedding a retryable 503 (not a false 404) if the object is then found nowhere. Default 64. | | `GITLAWB_IPFS_MAX_REPO_VISITS` | Ceiling on repos one `/ipfs/{cid}` request may visit (acquire + probe) past the visibility gate — also the worst-case per-request Tigris fetch count. On exhaustion the scan stops with a retryable 503. Default 1024. | -| `GITLAWB_IPFS_REQUEST_BUDGET_SECS` | Absolute wall-clock budget for one admitted `/ipfs/{cid}` request's acquire+walk lifetime. Per-stage timeouts are clamped to the remaining budget and no stage starts once it is exhausted; the scan then stops with a retryable 503. Overshoot is bounded by the in-flight clamped stage's kill/reap slack plus one unclamped `cat-file` probe. Default 600. | +| `GITLAWB_IPFS_REQUEST_BUDGET_SECS` | Absolute wall-clock budget for one admitted `/ipfs/{cid}` request's acquire+walk lifetime. Per-stage clamps bound the acquire and walk stages to the remaining budget, and no stage starts once it is exhausted; the scan then stops with a retryable 503. The object-type probe and content-read `cat-file` subprocesses are budget-checked before starting but have no duration bound of their own, so a hung git probe (corrupt pack, stuck filesystem) holds the request's walk slot for the full duration of the hang. Default 600. | | `GITLAWB_IPFS_RATE_LIMIT` | Max `/ipfs/{cid}` requests per client IP per hour (route flood brake). 0 disables. Default 600. | | `GITLAWB_TIGRIS_BUCKET` | Optional S3/Tigris shared repo storage bucket. | | `GITLAWB_PINATA_JWT` | Optional Pinata/IPFS warm-storage pinning. | diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index 014411e7..41080833 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -64,9 +64,9 @@ use crate::visibility::{visibility_check, Decision}; /// Request budget (F3): one absolute clock (`ipfs_request_budget_secs`) spans /// the whole admitted request. No stage (acquire, probe, walk, content read) /// starts once it is exhausted, and the acquire wait and walk deadline are -/// clamped to the remainder, so an admitted request cannot hold its scarce walk -/// slot past the budget plus the in-flight stage's kill/reap slack and one -/// unclamped probe subprocess. +/// clamped to the remainder. The probe and content-read subprocesses have no +/// duration bound of their own past their pre-start budget check, so a hung +/// git probe holds the request's walk slot for the full duration of the hang. /// /// Scope: this closes the direct unauthenticated scan, including the dangling /// case. A stale-public mirror row still serves withheld content (tracked @@ -188,9 +188,9 @@ pub async fn get_by_cid( // leaving this and every later candidate unproven, never a false 404). A // stage is never started with zero remaining; the probe and read // subprocesses carry no internal duration clamp, so this pre-start check is - // their entire bound (documented overshoot: at most one unclamped cat-file - // plus the in-flight clamped stage's kill/reap slack). The acquire and walk - // stages clamp their deadlines to the returned remainder. + // their entire bound (a hung one holds the request's walk slot for the + // duration of the hang). The acquire and walk stages clamp their deadlines + // to the returned remainder. fn budget_gate( truncated_by: &mut Vec<&'static str>, deadline: std::time::Instant, @@ -944,6 +944,96 @@ mod tests { ); } + /// F2 found-beats-taint on the acquire arm: an acquire timeout taints the + /// scan but must NOT stop it — the loop `continue`s, and a later repo that + /// genuinely carries the object still serves. The NEWER row (visited first + /// under `list_all_repos`' updated_at DESC) is a Tigris-backed ghost whose + /// acquire stalls against the silent endpoint and times out at 1s; the + /// OLDER row is a plain public repo carrying the blob, reached next and + /// served from a cheap probe — found beats taint: 200 with the blob bytes, + /// never the truncation 503. MUTATION (RED): turn the acquire-timeout arm's + /// `continue` into a `break` and the public copy never serves (503). + #[sqlx::test] + async fn get_by_cid_acquire_taint_does_not_block_later_public_copy(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // Seed the blob repo through a LOCAL-ONLY store first, so seeding never + // consults the (deliberately unreachable) Tigris endpoint. + state.repo_store = + crate::git::repo_store::RepoStore::for_testing(repos_dir.clone(), pool.clone()); + let content = b"acquire taint continue proof\n"; + let (_, oid) = + seed_repo_with_blob(&state, tmp.path(), "z6f2acqcont", "pubcopy", content).await; + // Swap in a Tigris-backed store over the SAME repos_dir (the seeded bare + // repo stays a fast local hit) and add a NEWER ghost row with no local + // copy: its acquire consults the silent local endpoint and stalls to the + // 1s timeout (endpoint-pinned test client, no AWS_* env reads). + let endpoint = silent_tigris_endpoint().await; + let tigris = + crate::git::tigris::TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint) + .await; + state.repo_store = crate::git::repo_store::RepoStore::new(repos_dir, Some(tigris), pool); + state + .db + .upsert_mirror_repo("z6f2acqcont", "ghost", "/unused-ghost", None, false) + .await + .unwrap(); + let mut cfg = (*state.config).clone(); + cfg.git_acquire_timeout_secs = 1; + state.config = Arc::new(cfg); + + // Ordering precondition: the ghost must be iterated FIRST (updated_at + // DESC — it was upserted after the blob repo), otherwise the pubcopy + // would serve before the taint ever fires and the continue-vs-break + // distinction would go untested. + let order: Vec = state + .db + .list_all_repos() + .await + .unwrap() + .into_iter() + .map(|r| r.name) + .collect(); + let ghost_pos = order.iter().position(|n| n == "ghost").unwrap(); + let pub_pos = order.iter().position(|n| n == "pubcopy").unwrap(); + assert!( + ghost_pos < pub_pos, + "precondition: the stalling ghost must be iterated before the blob repo; got {order:?}" + ); + + let peer: SocketAddr = "203.0.113.73:5000".parse().unwrap(); + let started = std::time::Instant::now(); + let resp = ipfs_router(state) + .oneshot(get_cid(&cid_for_oid(&oid), Some(peer))) + .await + .unwrap(); + // The taint arm demonstrably FIRED on this run: the response can only + // arrive after the ghost's stalled acquire burned its full 1s timeout + // (a cheap skip or a deny verdict would answer near-instantly). + assert!( + started.elapsed() >= std::time::Duration::from_millis(900), + "the ghost's acquire must stall to its timeout before the scan continues; \ + got {:?}", + started.elapsed() + ); + assert_eq!( + resp.status(), + StatusCode::OK, + "an acquire taint must not stop the scan — the later public copy serves" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + assert_eq!( + &body[..], + content.as_slice(), + "the served body must be the blob content from the later public copy" + ); + } + /// F2 probe taint: a repo row whose local dir does not exist (no Tigris) — /// `RepoStore::acquire` returns the path anyway (local passthrough), and the /// `cat-file -t` probe cannot even spawn (missing working dir), so diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index d564a554..87c5d65c 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -420,11 +420,13 @@ pub struct Config { /// acquire wait and walk deadline are clamped to `min(their own timeout, /// remaining budget)`; a stage is never started with zero remaining. On /// exhaustion the scan stops without a verdict and the request sheds a - /// retryable 503 + Retry-After rather than a false 404. Residual overshoot - /// past the budget is bounded by the kill/reap slack of the one in-flight - /// clamped stage (the walk watchdog's SIGTERM grace + SIGKILL settle) plus - /// the `object_type` / `read_object_content` probe subprocesses, which are - /// budget-checked before they start but carry no internal duration clamp. + /// retryable 503 + Retry-After rather than a false 404. The clamps bound + /// only the acquire and walk stages (overshoot there is the walk watchdog's + /// SIGTERM grace + SIGKILL settle); the `object_type` / + /// `read_object_content` probe subprocesses are budget-checked before they + /// start but have NO duration bound of their own, so a hung git probe + /// (corrupt pack, stuck filesystem) holds the request's walk slot for the + /// full duration of the hang. /// Must be positive. Default: 600s (10 min), matching /// `git_service_timeout_secs` so a single full-length walk still fits. #[arg( From 8d79692555758496126370814579b5b9ec0ff8be Mon Sep 17 00:00:00 2001 From: t Date: Sat, 18 Jul 2026 09:51:10 -0500 Subject: [PATCH 45/58] fix(node): make write-acquire cancellation-safe and connection-affine (#174 F1) acquire_write took the pg advisory lock, then awaited Tigris, then built the guard. An outer tokio::time::timeout firing during the Tigris await dropped the future with the lock held and no guard, so no unlock ran and later pushes to the repo spun 60s. release() also unlocked on an arbitrary pooled connection, not the one that locked (session advisory locks are connection-affine), so even the normal release was unreliable. Pin a PoolConnection in RepoWriteGuard, build the guard before the lock query, run the lock loop through it, unlock on that same connection in release(), and add a Drop backstop that spawns a detached affine unlock (Handle::try_current so an off-runtime drop logs instead of panicking). Two distinct-session tests (drop-without-release, affine release) go RED without the fix, GREEN with it. Pinning a connection per active write is a DB-pool DoS unless the pool clears the concurrent-write cap: the shipped default (20) was below max_concurrent_git_pushes (32). Raise the db_max_connections default to 48 and add Config::validate() to reject db_max_connections < max_concurrent_git_pushes + 8 headroom at boot; wire it into startup. --- crates/gitlawb-node/src/config.rs | 76 ++++++++- crates/gitlawb-node/src/git/repo_store.rs | 199 +++++++++++++++++++--- crates/gitlawb-node/src/main.rs | 7 + 3 files changed, 259 insertions(+), 23 deletions(-) diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index 87c5d65c..5fb66b55 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -206,11 +206,15 @@ pub struct Config { /// Maximum connections in the PostgreSQL pool. This is a cap, not a floor /// (connections open lazily). Size against the database server's - /// max_connections, remembering admin tooling opens its own pool. + /// max_connections, remembering admin tooling opens its own pool. Each + /// concurrent write pins one pooled connection for its whole duration (the + /// advisory lock in `repo_store::acquire_write` is connection-affine), so this + /// must exceed `max_concurrent_git_pushes` by `DB_POOL_APP_HEADROOM` or slow + /// pushes starve every other DB path — enforced by `Config::validate`. #[arg( long, env = "GITLAWB_DB_MAX_CONNECTIONS", - default_value_t = 20, + default_value_t = 48, value_parser = clap::value_parser!(u32).range(1..) )] pub db_max_connections: u32, @@ -461,6 +465,36 @@ impl Config { } PathBuf::from(&self.key_path) } + + /// DB connections reserved for everything other than held write-locks: auth + /// lookups, visibility-rule reads, the post-receive tail's own DB writes, and + /// admin tooling. A write pins one pooled connection for its whole duration, so + /// the pool must clear the concurrent-write cap by at least this margin. + pub const DB_POOL_APP_HEADROOM: u32 = 8; + + /// Cross-field boot validation. Single-field ranges are enforced by clap; this + /// catches combinations that ship a denial-of-service under otherwise-valid + /// values. Call once at startup and fail fast on `Err`. + pub fn validate(&self) -> Result<(), String> { + // A write pins one pooled connection for its whole duration (the + // connection-affine advisory lock in repo_store::acquire_write), and + // concurrent writes are capped at max_concurrent_git_pushes. If the pool + // does not exceed that cap by DB_POOL_APP_HEADROOM, a burst of slow pushes + // drains every connection and every other DB path 503s. (#174 F1) + let floor = (self.max_concurrent_git_pushes as u64) + (Self::DB_POOL_APP_HEADROOM as u64); + if (self.db_max_connections as u64) < floor { + return Err(format!( + "GITLAWB_DB_MAX_CONNECTIONS ({}) must be at least max_concurrent_git_pushes ({}) \ + + {} headroom = {}: each concurrent write pins one pooled connection for its whole \ + duration, so a smaller pool lets a burst of slow pushes starve every other DB path.", + self.db_max_connections, + self.max_concurrent_git_pushes, + Self::DB_POOL_APP_HEADROOM, + floor + )); + } + Ok(()) + } } #[cfg(test)] @@ -666,4 +700,42 @@ mod tests { 1_048_576 ); } + + /// #174 F1: a connection-affine write lock pins a pooled connection per + /// concurrent write, so the pool must clear `max_concurrent_git_pushes` by + /// `DB_POOL_APP_HEADROOM` or a push burst starves every other DB path. + /// `validate()` must reject an under-sized pool at boot. + #[test] + fn db_pool_must_clear_the_git_push_cap() { + // Shipped defaults validate (48 >= 32 + 8). + Config::parse_from(["gitlawb-node"]) + .validate() + .expect("default config must validate"); + + // An under-sized pool relative to the push cap is rejected (20 < 32 + 8). + let under = Config::parse_from([ + "gitlawb-node", + "--db-max-connections", + "20", + "--max-concurrent-git-pushes", + "32", + ]); + assert!( + under.validate().is_err(), + "db_max_connections 20 below max_concurrent_git_pushes 32 + headroom must be rejected" + ); + + // Exactly at the floor validates (40 == 32 + 8). + let at_floor = Config::parse_from([ + "gitlawb-node", + "--db-max-connections", + "40", + "--max-concurrent-git-pushes", + "32", + ]); + assert!( + at_floor.validate().is_ok(), + "db_max_connections at the floor (pushes + headroom) must validate" + ); + } } diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index a5c367e9..86a1ee8b 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -13,7 +13,8 @@ use std::path::{Path, PathBuf}; use std::sync::Arc; use anyhow::{Context, Result}; -use sqlx::PgPool; +use sqlx::pool::PoolConnection; +use sqlx::{PgPool, Postgres}; use tokio::sync::Mutex; use tracing::{debug, info, warn}; @@ -157,13 +158,42 @@ impl RepoStore { let (owner_slug, local_path) = self.local_path(owner_did, repo_name)?; let lock_key = advisory_lock_key(&owner_slug, repo_name); - // Acquire Postgres advisory lock with retry using pg_try_advisory_lock - // to avoid blocking indefinitely on stale locks from crashed connections. + // Pin a dedicated pooled connection and build the guard holding it BEFORE + // issuing the lock query. Session-level pg advisory locks are + // connection-affine (they can only be released on the session that took + // them), so the guard must own the locking connection; and building the + // guard first means any cancellation after the lock is taken — a + // `tokio::time::timeout` firing during the Tigris download below — drops a + // guard that CAN release, closing the leak the outer timeout otherwise + // opened (#174 F1). + let conn = self + .pool + .acquire() + .await + .context("acquiring db connection for the write advisory lock")?; + let mut guard = RepoWriteGuard { + owner_slug: owner_slug.clone(), + repo_name: repo_name.to_string(), + local_path: local_path.clone(), + lock_key, + conn: Some(conn), + locked: false, + released: false, + tigris: self.tigris.clone(), + }; + + // Acquire the advisory lock with retry, through the guard's OWN connection, + // so the matching unlock (in release, or the Drop backstop) runs on the same + // session — pg_advisory_unlock on a different pooled connection is a no-op. let mut acquired = false; for attempt in 0..60 { + let c = guard + .conn + .as_deref_mut() + .expect("write guard holds its connection during acquisition"); let row: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") .bind(lock_key) - .fetch_one(&self.pool) + .fetch_one(&mut *c) .await .context("trying advisory lock")?; if row.0 { @@ -177,9 +207,11 @@ impl RepoStore { if !acquired { anyhow::bail!("could not acquire advisory lock after 60s — possible stale lock for {owner_slug}/{repo_name}"); } + guard.locked = true; - // Always download the latest from Tigris before writing. - // Local disk may be stale if another machine pushed since our last access. + // Always download the latest from Tigris before writing. Local disk may be + // stale if another machine pushed since our last access. The guard already + // owns the lock + its connection, so a cancellation here drops through Drop. if let Some(ref tigris) = self.tigris { if tigris.exists(&owner_slug, repo_name).await.unwrap_or(false) { debug!(repo = %repo_name, "write acquire: downloading latest from tigris"); @@ -197,14 +229,7 @@ impl RepoStore { } } - Ok(RepoWriteGuard { - owner_slug, - repo_name: repo_name.to_string(), - local_path, - lock_key, - pool: self.pool.clone(), - tigris: self.tigris.clone(), - }) + Ok(guard) } /// Initialize a new bare repo on local disk and upload to Tigris. @@ -354,7 +379,16 @@ pub struct RepoWriteGuard { repo_name: String, pub local_path: PathBuf, lock_key: i64, - pool: PgPool, + /// The pooled connection that took the advisory lock. Session-level pg + /// advisory locks are connection-affine, so the guard pins that connection + /// for its whole lifetime and unlocks on it (in `release`, or the `Drop` + /// backstop). `None` only after the connection has been taken to unlock. + conn: Option>, + /// Set once the advisory lock has actually been taken. A guard dropped + /// before the lock is held (or after `release`) performs no unlock. + locked: bool, + /// Set once `release` has run its unlock, making the `Drop` backstop inert. + released: bool, tigris: Option, } @@ -369,7 +403,7 @@ impl RepoWriteGuard { /// half-applied or otherwise inconsistent repo would propagate corruption to /// Tigris (and to every node that later downloads it). The lock is always /// released regardless, to avoid stale locks blocking future writes. - pub async fn release(self, success: bool) { + pub async fn release(mut self, success: bool) { // Upload to Tigris only on success. if success { if let Some(ref tigris) = self.tigris { @@ -384,11 +418,57 @@ impl RepoWriteGuard { warn!(repo = %self.repo_name, "write failed — skipping tigris upload to avoid propagating an inconsistent repo"); } - // Release advisory lock - let _ = sqlx::query("SELECT pg_advisory_unlock($1)") - .bind(self.lock_key) - .execute(&self.pool) - .await; + // Release the advisory lock on the SAME connection that took it (session + // advisory locks are connection-affine). Taking the connection here makes + // the Drop backstop inert. + if self.locked { + if let Some(mut conn) = self.conn.take() { + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(self.lock_key) + .execute(&mut *conn) + .await; + } + } + self.released = true; + } +} + +impl Drop for RepoWriteGuard { + /// Cancellation-safe backstop: if the guard is dropped while still holding the + /// advisory lock (a `tokio::time::timeout` cancelled `acquire_write`, or a + /// handler future was dropped before `release`), unlock on the pinned + /// connection. `Drop` cannot await, so spawn a detached unlock — it runs on the + /// same session (connection-affine). An off-runtime drop falls back to a log; + /// the ~60s stale-lock retry loop in `acquire_write` reclaims it. + fn drop(&mut self) { + if self.released || !self.locked { + return; + } + let Some(mut conn) = self.conn.take() else { + return; + }; + let lock_key = self.lock_key; + let repo_name = self.repo_name.clone(); + match tokio::runtime::Handle::try_current() { + Ok(handle) => { + handle.spawn(async move { + if let Err(e) = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(lock_key) + .execute(&mut *conn) + .await + { + warn!(repo = %repo_name, err = %e, "detached advisory-unlock on write-guard drop failed"); + } + }); + } + Err(_) => { + warn!( + repo = %repo_name, + "RepoWriteGuard dropped off a Tokio runtime; advisory lock not released \ + synchronously — the stale-lock retry loop will reclaim it" + ); + } + } } } @@ -562,4 +642,81 @@ mod tests { ); } } + + // ── advisory-lock cancellation-safety (#174 F1, RED-before/GREEN-after) ── + + /// F1 (P1): dropping a `RepoWriteGuard` WITHOUT calling `release()` — the + /// state a `tokio::time::timeout` cancellation leaves `acquire_write` in when + /// it fires during the Tigris await — must still release the session advisory + /// lock. A checker connection is held OUT of the pool first, so `acquire_write` + /// is forced onto a distinct session; the checker (a different session) then + /// probes the lock, so advisory-lock re-entrancy cannot mask a leak. + /// + /// Load-bearing: RED today (no `Drop` releases the lock → held by + /// `acquire_write`'s session → checker's `pg_try_advisory_lock` returns + /// false). GREEN after the connection-affine `Drop` backstop. + #[sqlx::test] + async fn write_guard_drop_without_release_frees_the_lock(pool: sqlx::PgPool) { + let dir = tempfile::TempDir::new().unwrap(); + let store = RepoStore::for_testing(dir.path().to_path_buf(), pool.clone()); + let owner = "did:key:z6MkDropBackstopProofAAAAAAAAAAAAAAAAAAAAAA"; + let name = "leaktest"; + let slug = owner.replace([':', '/'], "_"); + let key = advisory_lock_key(&slug, name); + + // Distinct session for the probe: hold it out of the pool BEFORE acquiring, + // so acquire_write cannot use it and a re-entrant probe cannot falsely read free. + let mut checker = pool.acquire().await.expect("checker connection"); + + let guard = store.acquire_write(owner, name).await.expect("acquire"); + // The cancellation shape: drop without release(). + drop(guard); + // Let the detached unlock task run. + tokio::time::sleep(std::time::Duration::from_millis(400)).await; + + let (free,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut *checker) + .await + .unwrap(); + assert!( + free, + "advisory lock must be released when the guard is dropped without release()" + ); + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(key) + .execute(&mut *checker) + .await; + } + + /// F1 (latent non-affine release): `release()` must unlock on the SAME + /// session that locked, so the lock is freed regardless of which pooled + /// connection would service a fresh query. Observed from a distinct session. + #[sqlx::test] + async fn write_guard_release_frees_the_lock_from_a_distinct_session(pool: sqlx::PgPool) { + let dir = tempfile::TempDir::new().unwrap(); + let store = RepoStore::for_testing(dir.path().to_path_buf(), pool.clone()); + let owner = "did:key:z6MkAffineReleaseProofBBBBBBBBBBBBBBBBBBBB"; + let name = "affinetest"; + let slug = owner.replace([':', '/'], "_"); + let key = advisory_lock_key(&slug, name); + + let mut checker = pool.acquire().await.expect("checker connection"); + let guard = store.acquire_write(owner, name).await.expect("acquire"); + guard.release(false).await; + + let (free,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut *checker) + .await + .unwrap(); + assert!( + free, + "release() must free the advisory lock via connection-affine unlock" + ); + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(key) + .execute(&mut *checker) + .await; + } } diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index ffac65a2..b7bc758e 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -74,6 +74,13 @@ async fn main() -> Result<()> { // bootstrap peers. Operators can opt out via GITLAWB_BOOTSTRAP_DISABLE_SEEDS. bootstrap::merge_seeds(&mut config); + // Fail fast on config combinations that are individually in-range but jointly + // unsafe — notably a DB pool too small for the concurrent-write cap, which + // would let a push burst starve every other DB path (#174 F1). + config + .validate() + .map_err(|e| anyhow::anyhow!("invalid configuration: {e}"))?; + if !config.public_read { warn!( "GITLAWB_PUBLIC_READ=false is reserved; per-repository private-read enforcement is not wired in alpha" From 65d7af2a741a6c8104a1463da02c6fd1d3cdddef Mon Sep 17 00:00:00 2001 From: t Date: Sat, 18 Jul 2026 10:14:27 -0500 Subject: [PATCH 46/58] fix(node): own the post-receive replication tail in a detached task (#174 F2) The post-receive replication tail (withheld/candidate/full-scan resolution, then the encrypt+recovery spawn and the announce spawn) ran inline in the request future and parked on git_encrypt_semaphore before the durable try_begin gate. A client/proxy disconnect while parked dropped the future and silently lost this push's pins, recovery copy, and announcements, with no reconciliation sweep to recover them (state.rs documented this exact residual). Move the whole tail into an independently owned tokio task and return the git response immediately, so a disconnect can no longer drop the parked work. Each push owns its own tail, including its own always-spawned announce, so the announce is never brought under the per-repo encrypt coalescing (which would drop a coalesced push's per-ref announcements). Update state.rs to retire the disconnect-loses-work residual. Test: receive_pack_landed_push_returns_without_parking_on_scan_pool asserts the handler returns 200 while the scan pool is held (RED with the tail inline: the request future parks and the timeout fires; GREEN detached). The burst serialization test is retimed to poll for scan completion since the withheld walk is now detached too. --- crates/gitlawb-node/src/api/repos.rs | 645 ++++++++++++++------------- crates/gitlawb-node/src/state.rs | 14 +- 2 files changed, 352 insertions(+), 307 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 2770ffb9..98e395f6 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1645,314 +1645,336 @@ pub async fn git_receive_pack( } } - // Replication enforcement (Phase 2): decide once per push whether the public - // may read this repo at all and, if so, which blob OIDs must not leave the - // node. `withheld == None` means replicate nothing (private / mode A / - // undetermined): skip every pin so even commit and tree objects (which - // withheld_blob_oids never lists) stay local. `announce` gates the - // network-facing announcements. Fail closed: a private or undetermined repo - // never leaks. - let rules_opt = state.db.list_visibility_rules(&record.id).await.ok(); - let (announce, withheld) = replication_withheld_set( - state.git_encrypt_semaphore.clone(), - rules_opt.clone(), - &record.owner_did, - record.is_public, - disk_path.clone(), - state.git_bin.clone(), - std::time::Duration::from_secs(state.config.git_service_timeout_secs), - ) - .await; - - // Resolve the per-push pin candidate set once, off the async worker, then - // filter to what may actually replicate. Delta path: the reachable-only - // `withheld` set suffices (delta objects are reachable). Full-scan path: the - // candidate set can include dangling blobs the withheld set never classified, - // so fail closed — replicate a blob only if it is reachable AND - // visibility-allowed (#99). Only computed when something will actually - // replicate; every degraded path logs rather than failing silently. - let object_list: Vec = if let Some(withheld_set) = withheld.clone() { - let new_tips: Vec = ref_updates - .iter() - .map(|u| u.new_sha.clone()) - .filter(|s| s != ZERO_SHA) - .collect(); - let old_tips: Vec = ref_updates - .iter() - .map(|u| u.old_sha.clone()) - .filter(|s| s != ZERO_SHA) - .collect(); - let pin_set = crate::git::push_delta::resolve_candidates_for_push( + // #174 F2: move the whole post-receive replication tail into an independently owned + // task. It parks on `git_encrypt_semaphore` (withheld / candidate / full-scan + // resolution), so leaving it in the request future means a client/proxy disconnect + // while parked silently drops this push's pins, recovery copy, and announcements — + // the residual `state.rs` documented, because the durable `try_begin` gate sat after + // the park. The request future now returns the git response immediately and the tail + // runs detached; a disconnect can no longer drop it. Each push owns its own tail, + // including its own always-spawned announce, so per-push announcements are never + // coalesced away (the announce spawn stays out of the per-repo encrypt coalescing). + let did = did.to_string(); + tokio::spawn(async move { + // Replication enforcement (Phase 2): decide once per push whether the public + // may read this repo at all and, if so, which blob OIDs must not leave the + // node. `withheld == None` means replicate nothing (private / mode A / + // undetermined): skip every pin so even commit and tree objects (which + // withheld_blob_oids never lists) stay local. `announce` gates the + // network-facing announcements. Fail closed: a private or undetermined repo + // never leaks. + let rules_opt = state.db.list_visibility_rules(&record.id).await.ok(); + let (announce, withheld) = replication_withheld_set( state.git_encrypt_semaphore.clone(), + rules_opt.clone(), + &record.owner_did, + record.is_public, disk_path.clone(), - new_tips, - old_tips, state.git_bin.clone(), std::time::Duration::from_secs(state.config.git_service_timeout_secs), - false, ) .await; - if pin_set.full_scan { - fail_closed_full_scan_objects( + + // Resolve the per-push pin candidate set once, off the async worker, then + // filter to what may actually replicate. Delta path: the reachable-only + // `withheld` set suffices (delta objects are reachable). Full-scan path: the + // candidate set can include dangling blobs the withheld set never classified, + // so fail closed — replicate a blob only if it is reachable AND + // visibility-allowed (#99). Only computed when something will actually + // replicate; every degraded path logs rather than failing silently. + let object_list: Vec = if let Some(withheld_set) = withheld.clone() { + let new_tips: Vec = ref_updates + .iter() + .map(|u| u.new_sha.clone()) + .filter(|s| s != ZERO_SHA) + .collect(); + let old_tips: Vec = ref_updates + .iter() + .map(|u| u.old_sha.clone()) + .filter(|s| s != ZERO_SHA) + .collect(); + let pin_set = crate::git::push_delta::resolve_candidates_for_push( state.git_encrypt_semaphore.clone(), disk_path.clone(), - rules_opt.clone().unwrap_or_default(), - record.is_public, - record.owner_did.clone(), - pin_set.candidates, + new_tips, + old_tips, state.git_bin.clone(), std::time::Duration::from_secs(state.config.git_service_timeout_secs), + false, ) - .await - } else { - crate::git::visibility_pack::replicable_objects(pin_set.candidates, &withheld_set) - } - } else { - Vec::new() - }; - - // Pin new git objects to the local IPFS node (no-op if ipfs_api is empty). - // Skipped entirely when the public cannot read the repo (withheld == None). - // - // Coalesce-and-requeue per repo (#174 P2-2 + F5): the spawned task's walks park - // on `git_encrypt_semaphore` (which DEFERS when the pool is full rather than - // dropping the recovery copy). To bound the OUTSTANDING task set, at most one - // task per repo is in flight; a push arriving while one is in flight does NOT - // spawn a duplicate — and is NOT dropped either. The in-flight task pins only - // its own pre-spawn object-list snapshot, so this push's (old, new) tip pairs - // are merged into the in-flight key's pending slot in the same critical section - // as the presence check, and the task loop-drains them (fresh rules, fail - // closed) before releasing the key. Without the requeue a coalesced push's pins - // and recovery copies would be silently absent until an unrelated later push - // (the F5 loss). The guard still releases the key on panic (Drop on unwind), so - // a crashed walk never permanently locks the repo out. - if withheld.is_some() { - let tip_pairs: Vec<(String, String)> = ref_updates - .iter() - .map(|u| (u.old_sha.clone(), u.new_sha.clone())) - .collect(); - match state.encrypt_inflight.try_begin(&record.id, tip_pairs) { - crate::state::BeginOutcome::Coalesced => { - tracing::debug!( - repo = %record.id, - "post-push encryption task already in flight for this repo; coalesced \ - — this push's tip pairs are queued for that task's drain" - ); - } - crate::state::BeginOutcome::Admitted(inflight_guard) => { - let ctx = EncryptTaskCtx { - ipfs_api: state.config.ipfs_api.clone(), - repo_path: disk_path.clone(), - db: state.db.clone(), - repo_id: record.id.clone(), - owner_did: record.owner_did.clone(), - repo_name: record.name.clone(), - irys_url: state.config.irys_url.clone(), - http_client: std::sync::Arc::clone(&state.http_client), - node_did: state.node_did.to_string(), - node_keypair: std::sync::Arc::clone(&state.node_keypair), - git_bin: state.git_bin.clone(), - git_timeout: std::time::Duration::from_secs( - state.config.git_service_timeout_secs, - ), - encrypt_sem: state.git_encrypt_semaphore.clone(), - }; - tokio::spawn(run_encrypt_pin_task( - ctx, - inflight_guard, - object_list.clone(), - rules_opt.clone(), + .await; + if pin_set.full_scan { + fail_closed_full_scan_objects( + state.git_encrypt_semaphore.clone(), + disk_path.clone(), + rules_opt.clone().unwrap_or_default(), record.is_public, - )); - } - } - } - - // Pin new git objects to Pinata, then record branch→CID and gossip. - // - // #174 P2-2 scope note: this SECOND detached spawn is deliberately NOT brought - // under the per-repo encryption coalescing above. Two reasons: (1) it does not - // park on `git_encrypt_semaphore` (or any semaphore) — the Pinata `pin_new_objects` - // is a bounded reqwest round-trip, so it does not form the unbounded PARKED-waiter - // set that is the P2-2 residual; it runs to completion under the HTTP client's - // network timeouts. (2) Unlike the idempotent recovery-copy walk, this task does - // PER-PUSH, PER-REF work — branch→CID upserts, gossip publish, GraphQL subscription - // broadcast, Arweave anchoring, and peer notify, each keyed to THIS push's - // ref_updates. Coalescing it against an in-flight task for the same repo would DROP - // a later push's ref-update announcements (a correctness regression), not merely - // delay a duplicate. So it is scoped out with rationale, not brought under the bound. - { - let pinata_jwt = state.config.pinata_jwt.clone(); - let pinata_upload_url = state.config.pinata_upload_url.clone(); - let repo_path_clone = disk_path.clone(); - let db_clone = state.db.clone(); - let http_client = Arc::clone(&state.http_client); - let node_did_str = state.node_did.to_string(); - let repo_slug = format!( - "{}/{}", - crate::db::normalize_owner_key(&record.owner_did), - record.name - ); - let ref_updates_clone = ref_updates - .iter() - .map(|u| (u.ref_name.clone(), u.old_sha.clone(), u.new_sha.clone())) - .collect::>(); - let p2p_handle = state.p2p.clone(); - let pusher_did_clone = did.to_string(); - let db_for_peers = state.db.clone(); - let ref_update_tx = state.ref_update_tx.clone(); - let irys_url = state.config.irys_url.clone(); - let owner_did_for_arweave = record.owner_did.clone(); - let self_public_url = state.config.public_url.clone(); - let node_keypair = Arc::clone(&state.node_keypair); - let object_list_pinata = object_list; - let do_pinata_replication = withheld.is_some(); - tokio::spawn(async move { - let pinned = if do_pinata_replication { - crate::pinata::pin_new_objects( - &http_client, - &pinata_upload_url, - &pinata_jwt, - &repo_path_clone, - object_list_pinata, - &db_clone, + record.owner_did.clone(), + pin_set.candidates, + state.git_bin.clone(), + std::time::Duration::from_secs(state.config.git_service_timeout_secs), ) .await } else { - Vec::new() - }; + crate::git::visibility_pack::replicable_objects(pin_set.candidates, &withheld_set) + } + } else { + Vec::new() + }; - if !pinned.is_empty() { - tracing::info!(count = pinned.len(), "pinned git objects to Pinata"); + // Pin new git objects to the local IPFS node (no-op if ipfs_api is empty). + // Skipped entirely when the public cannot read the repo (withheld == None). + // + // Coalesce-and-requeue per repo (#174 P2-2 + F5): the spawned task's walks park + // on `git_encrypt_semaphore` (which DEFERS when the pool is full rather than + // dropping the recovery copy). To bound the OUTSTANDING task set, at most one + // task per repo is in flight; a push arriving while one is in flight does NOT + // spawn a duplicate — and is NOT dropped either. The in-flight task pins only + // its own pre-spawn object-list snapshot, so this push's (old, new) tip pairs + // are merged into the in-flight key's pending slot in the same critical section + // as the presence check, and the task loop-drains them (fresh rules, fail + // closed) before releasing the key. Without the requeue a coalesced push's pins + // and recovery copies would be silently absent until an unrelated later push + // (the F5 loss). The guard still releases the key on panic (Drop on unwind), so + // a crashed walk never permanently locks the repo out. + if withheld.is_some() { + let tip_pairs: Vec<(String, String)> = ref_updates + .iter() + .map(|u| (u.old_sha.clone(), u.new_sha.clone())) + .collect(); + match state.encrypt_inflight.try_begin(&record.id, tip_pairs) { + crate::state::BeginOutcome::Coalesced => { + tracing::debug!( + repo = %record.id, + "post-push encryption task already in flight for this repo; coalesced \ + — this push's tip pairs are queued for that task's drain" + ); + } + crate::state::BeginOutcome::Admitted(inflight_guard) => { + let ctx = EncryptTaskCtx { + ipfs_api: state.config.ipfs_api.clone(), + repo_path: disk_path.clone(), + db: state.db.clone(), + repo_id: record.id.clone(), + owner_did: record.owner_did.clone(), + repo_name: record.name.clone(), + irys_url: state.config.irys_url.clone(), + http_client: std::sync::Arc::clone(&state.http_client), + node_did: state.node_did.to_string(), + node_keypair: std::sync::Arc::clone(&state.node_keypair), + git_bin: state.git_bin.clone(), + git_timeout: std::time::Duration::from_secs( + state.config.git_service_timeout_secs, + ), + encrypt_sem: state.git_encrypt_semaphore.clone(), + }; + tokio::spawn(run_encrypt_pin_task( + ctx, + inflight_guard, + object_list.clone(), + rules_opt.clone(), + record.is_public, + )); + } } + } + + // Pin new git objects to Pinata, then record branch→CID and gossip. + // + // #174 P2-2 scope note: this SECOND detached spawn is deliberately NOT brought + // under the per-repo encryption coalescing above. Two reasons: (1) it does not + // park on `git_encrypt_semaphore` (or any semaphore) — the Pinata `pin_new_objects` + // is a bounded reqwest round-trip, so it does not form the unbounded PARKED-waiter + // set that is the P2-2 residual; it runs to completion under the HTTP client's + // network timeouts. (2) Unlike the idempotent recovery-copy walk, this task does + // PER-PUSH, PER-REF work — branch→CID upserts, gossip publish, GraphQL subscription + // broadcast, Arweave anchoring, and peer notify, each keyed to THIS push's + // ref_updates. Coalescing it against an in-flight task for the same repo would DROP + // a later push's ref-update announcements (a correctness regression), not merely + // delay a duplicate. So it is scoped out with rationale, not brought under the bound. + { + let pinata_jwt = state.config.pinata_jwt.clone(); + let pinata_upload_url = state.config.pinata_upload_url.clone(); + let repo_path_clone = disk_path.clone(); + let db_clone = state.db.clone(); + let http_client = Arc::clone(&state.http_client); + let node_did_str = state.node_did.to_string(); + let repo_slug = format!( + "{}/{}", + crate::db::normalize_owner_key(&record.owner_did), + record.name + ); + let ref_updates_clone = ref_updates + .iter() + .map(|u| (u.ref_name.clone(), u.old_sha.clone(), u.new_sha.clone())) + .collect::>(); + let p2p_handle = state.p2p.clone(); + let pusher_did_clone = did.to_string(); + let db_for_peers = state.db.clone(); + let ref_update_tx = state.ref_update_tx.clone(); + let irys_url = state.config.irys_url.clone(); + let owner_did_for_arweave = record.owner_did.clone(); + let self_public_url = state.config.public_url.clone(); + let node_keypair = Arc::clone(&state.node_keypair); + let object_list_pinata = object_list; + let do_pinata_replication = withheld.is_some(); + tokio::spawn(async move { + let pinned = if do_pinata_replication { + crate::pinata::pin_new_objects( + &http_client, + &pinata_upload_url, + &pinata_jwt, + &repo_path_clone, + object_list_pinata, + &db_clone, + ) + .await + } else { + Vec::new() + }; - // Build sha→cid map from pinned objects - let cid_map: std::collections::HashMap = pinned.into_iter().collect(); + if !pinned.is_empty() { + tracing::info!(count = pinned.len(), "pinned git objects to Pinata"); + } - // Record branch→CID for each ref update and publish gossip - for (ref_name, old_sha, new_sha) in &ref_updates_clone { - let cid = cid_map.get(new_sha).map(|s| s.as_str()); + // Build sha→cid map from pinned objects + let cid_map: std::collections::HashMap = + pinned.into_iter().collect(); - if let Some(cid_str) = cid { - let _ = db_clone - .upsert_branch_cid(&repo_slug, ref_name, new_sha, cid_str, &node_did_str) - .await; + // Record branch→CID for each ref update and publish gossip + for (ref_name, old_sha, new_sha) in &ref_updates_clone { + let cid = cid_map.get(new_sha).map(|s| s.as_str()); + + if let Some(cid_str) = cid { + let _ = db_clone + .upsert_branch_cid( + &repo_slug, + ref_name, + new_sha, + cid_str, + &node_did_str, + ) + .await; + } + + if announce { + if let Some(p2p) = &p2p_handle { + p2p.publish_ref_update(crate::p2p::RefUpdateEvent { + node_did: node_did_str.clone(), + pusher_did: pusher_did_clone.clone(), + repo: repo_slug.clone(), + ref_name: ref_name.clone(), + old_sha: old_sha.clone(), + new_sha: new_sha.clone(), + timestamp: chrono::Utc::now().to_rfc3339(), + cert_id: None, + cid: cid.map(|s| s.to_string()), + }) + .await; + } + } } + // Broadcast ref update to GraphQL subscription listeners — one per ref. + // Gated on `announce`: /graphql/ws is unauthenticated (mounted after + // the optional_signature layer), and the subscription resolver has no + // caller to gate against, so only publicly-readable ref updates may + // reach anonymous subscribers. Mirrors the gossip (above) and Arweave + // (below) sends, which are already `announce`-gated. Without this a + // private-repo push would leak live ref metadata over the socket — + // the subscription analog of #112/#114. + let now_ts = chrono::Utc::now().to_rfc3339(); if announce { - if let Some(p2p) = &p2p_handle { - p2p.publish_ref_update(crate::p2p::RefUpdateEvent { - node_did: node_did_str.clone(), - pusher_did: pusher_did_clone.clone(), + for (ref_name, old_sha, new_sha) in &ref_updates_clone { + let _ = ref_update_tx.send(crate::state::RefUpdateBroadcast { repo: repo_slug.clone(), ref_name: ref_name.clone(), old_sha: old_sha.clone(), new_sha: new_sha.clone(), - timestamp: chrono::Utc::now().to_rfc3339(), - cert_id: None, - cid: cid.map(|s| s.to_string()), - }) - .await; + pusher_did: pusher_did_clone.clone(), + node_did: node_did_str.clone(), + timestamp: now_ts.clone(), + }); } } - } - - // Broadcast ref update to GraphQL subscription listeners — one per ref. - // Gated on `announce`: /graphql/ws is unauthenticated (mounted after - // the optional_signature layer), and the subscription resolver has no - // caller to gate against, so only publicly-readable ref updates may - // reach anonymous subscribers. Mirrors the gossip (above) and Arweave - // (below) sends, which are already `announce`-gated. Without this a - // private-repo push would leak live ref metadata over the socket — - // the subscription analog of #112/#114. - let now_ts = chrono::Utc::now().to_rfc3339(); - if announce { - for (ref_name, old_sha, new_sha) in &ref_updates_clone { - let _ = ref_update_tx.send(crate::state::RefUpdateBroadcast { - repo: repo_slug.clone(), - ref_name: ref_name.clone(), - old_sha: old_sha.clone(), - new_sha: new_sha.clone(), - pusher_did: pusher_did_clone.clone(), - node_did: node_did_str.clone(), - timestamp: now_ts.clone(), - }); - } - } - // Arweave permanent anchoring — fire for each ref update. - // Suppressed for repos the public cannot read (public permanent ledger). - if announce && !irys_url.is_empty() { - for (ref_name, old_sha, new_sha) in &ref_updates_clone { - let cid = cid_map.get(new_sha).cloned(); - let anchor = crate::arweave::RefAnchor { - repo: repo_slug.clone(), - owner_did: owner_did_for_arweave.clone(), - ref_name: ref_name.clone(), - old_sha: old_sha.clone(), - new_sha: new_sha.clone(), - cid: cid.clone(), - timestamp: now_ts.clone(), - node_did: node_did_str.clone(), - }; - match crate::arweave::anchor_ref_update(&http_client, &irys_url, &anchor).await - { - Ok(tx_id) if !tx_id.is_empty() => { - let arweave_url = crate::arweave::arweave_url(&tx_id); - let _ = db_clone - .record_arweave_anchor(&crate::db::RecordAnchorInput { - repo: &repo_slug, - owner_did: &owner_did_for_arweave, - ref_name, - old_sha, - new_sha, - cid: cid.as_deref(), - irys_tx_id: &tx_id, - arweave_url: &arweave_url, - node_did: &node_did_str, - }) - .await; + // Arweave permanent anchoring — fire for each ref update. + // Suppressed for repos the public cannot read (public permanent ledger). + if announce && !irys_url.is_empty() { + for (ref_name, old_sha, new_sha) in &ref_updates_clone { + let cid = cid_map.get(new_sha).cloned(); + let anchor = crate::arweave::RefAnchor { + repo: repo_slug.clone(), + owner_did: owner_did_for_arweave.clone(), + ref_name: ref_name.clone(), + old_sha: old_sha.clone(), + new_sha: new_sha.clone(), + cid: cid.clone(), + timestamp: now_ts.clone(), + node_did: node_did_str.clone(), + }; + match crate::arweave::anchor_ref_update(&http_client, &irys_url, &anchor) + .await + { + Ok(tx_id) if !tx_id.is_empty() => { + let arweave_url = crate::arweave::arweave_url(&tx_id); + let _ = db_clone + .record_arweave_anchor(&crate::db::RecordAnchorInput { + repo: &repo_slug, + owner_did: &owner_did_for_arweave, + ref_name, + old_sha, + new_sha, + cid: cid.as_deref(), + irys_tx_id: &tx_id, + arweave_url: &arweave_url, + node_did: &node_did_str, + }) + .await; + } + Ok(_) => {} + Err(e) => { + tracing::warn!(repo=%repo_slug, err=%e, "Arweave anchor failed") + } } - Ok(_) => {} - Err(e) => tracing::warn!(repo=%repo_slug, err=%e, "Arweave anchor failed"), } } - } - // HTTP peer notification — notify all known peers to pull from us. - // This is the reliable fallback when Gossipsub p2p is not yet connected. - // Suppressed for repos the public cannot read. Runs last so a slow or - // unreachable peer cannot delay the local GraphQL broadcast or Arweave - // anchoring above; this is the lowest-priority best-effort step. - if announce { - if let Ok(peers) = db_for_peers.list_peers().await { - for peer in peers { - if peer.http_url.is_empty() { - continue; - } - let peer_url = peer.http_url.trim_end_matches('/'); - if let Some(self_url) = self_public_url.as_deref() { - if peer_url == self_url.trim_end_matches('/') { + // HTTP peer notification — notify all known peers to pull from us. + // This is the reliable fallback when Gossipsub p2p is not yet connected. + // Suppressed for repos the public cannot read. Runs last so a slow or + // unreachable peer cannot delay the local GraphQL broadcast or Arweave + // anchoring above; this is the lowest-priority best-effort step. + if announce { + if let Ok(peers) = db_for_peers.list_peers().await { + for peer in peers { + if peer.http_url.is_empty() { continue; } + let peer_url = peer.http_url.trim_end_matches('/'); + if let Some(self_url) = self_public_url.as_deref() { + if peer_url == self_url.trim_end_matches('/') { + continue; + } + } + let notify_url = format!("{peer_url}{SYNC_NOTIFY_PATH}"); + notify_peer_of_refs( + &http_client, + node_keypair.as_ref(), + &peer.did, + ¬ify_url, + &repo_slug, + &ref_updates_clone, + &node_did_str, + &pusher_did_clone, + ) + .await; } - let notify_url = format!("{peer_url}{SYNC_NOTIFY_PATH}"); - notify_peer_of_refs( - &http_client, - node_keypair.as_ref(), - &peer.did, - ¬ify_url, - &repo_slug, - &ref_updates_clone, - &node_did_str, - &pusher_did_clone, - ) - .await; } } - } - }); - } + }); + } + }); Ok(result) } @@ -5179,9 +5201,23 @@ mod tests { assert_eq!(a.status(), 200, "push A lands 200 despite scan contention"); assert_eq!(b.status(), 200, "push B lands 200 despite scan contention"); - // Let the detached recipients walks drain through the same pool before - // reading the detector files. - tokio::time::sleep(std::time::Duration::from_millis(800)).await; + // Wait for both pushes' detached scan tails to drain through the pool of 1 + // before reading the detector files. The WHOLE tail (withheld walk included) now + // runs detached (#174 F2), so poll until every expected scan has run rather than a + // fixed sleep, which is load-sensitive under a parallel test run. + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(20); + loop { + let ran = std::fs::read_to_string(&ranfile) + .unwrap_or_default() + .lines() + .count(); + if ran >= 6 || std::time::Instant::now() >= deadline { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + // Small settle so the last scan's rmdir has landed before the overlap check. + tokio::time::sleep(std::time::Duration::from_millis(100)).await; assert!( !overlap.exists(), "with a scan pool of 1, no two scans' git may ever be alive at once \ @@ -5252,9 +5288,20 @@ mod tests { /// F4 scenario 4 — landed-push-never-fails: a push whose post-receive walk must /// park (pool held elsewhere) DEFERS and then returns the receive-pack success /// once admission frees; contention never converts the landed push into a 5xx. + /// #174 F2 (RED-before/GREEN-after): the post-receive replication tail parks on + /// `git_encrypt_semaphore` (withheld/candidate/full-scan resolution). Leaving it in the + /// request future means a client/proxy disconnect while parked silently loses this + /// push's pins, recovery copy, and announcements (state.rs documented this residual). + /// The fix moves the whole tail into an independently owned task, so the handler + /// returns its receive-pack 200 WITHOUT waiting on the scan pool and a disconnect can + /// no longer drop the work. + /// + /// Load-bearing: with the tail inline (pre-fix) the handler parks while the pool is + /// held and does NOT return within the bound (RED — the timeout fires). With the + /// detached tail it returns 200 promptly (GREEN). #[cfg(unix)] #[sqlx::test] - async fn receive_pack_landed_push_defers_on_scan_contention_never_errors(pool: sqlx::PgPool) { + async fn receive_pack_landed_push_returns_without_parking_on_scan_pool(pool: sqlx::PgPool) { use axum::extract::{Path, State}; use axum::Extension; use std::net::SocketAddr; @@ -5268,40 +5315,38 @@ mod tests { f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6f4park", "p1", true).await; let sem = Arc::new(Semaphore::new(1)); state.git_encrypt_semaphore = sem.clone(); - // The test holds the pool's only permit: the push's withheld walk must park. + // Hold the pool's only permit: the post-receive scan would park if it ran in the + // request future. let held = sem.clone().acquire_owned().await.unwrap(); let peer: SocketAddr = "203.0.113.74:5000".parse().unwrap(); - let mut handle = tokio::spawn(git_receive_pack( - State(state), - Path(("z6f4park".to_string(), "p1".to_string())), - Extension(crate::auth::AuthenticatedDid( - "did:key:z6MkF4ParkPusherAAAAAAAAAAAAAAAAAAAAAAAAA".to_string(), - )), - crate::rate_limit::PeerAddr(Some(peer)), - axum::http::HeaderMap::new(), - ref_update_body("2222222222222222222222222222222222222222"), - )); - - // Parked, not errored: the handler must NOT complete while the pool is held. - let parked = tokio::time::timeout(std::time::Duration::from_millis(700), &mut handle).await; - assert!( - parked.is_err(), - "the post-receive walk must defer while the scan pool is held; got {parked:?}" - ); - - // Release admission: the SAME push now finishes with the receive-pack 200. - drop(held); - let resp = tokio::time::timeout(std::time::Duration::from_secs(30), handle) - .await - .expect("the deferred push must complete once admission frees") - .expect("the push task must not panic") - .expect("contention must never convert a landed push into an error"); + // The handler must return its receive-pack 200 WITHOUT waiting on the held scan + // pool — the tail is owned by a detached task. Pre-fix this times out. + let resp = tokio::time::timeout( + std::time::Duration::from_secs(5), + git_receive_pack( + State(state), + Path(("z6f4park".to_string(), "p1".to_string())), + Extension(crate::auth::AuthenticatedDid( + "did:key:z6MkF4ParkPusherAAAAAAAAAAAAAAAAAAAAAAAAA".to_string(), + )), + crate::rate_limit::PeerAddr(Some(peer)), + axum::http::HeaderMap::new(), + ref_update_body("2222222222222222222222222222222222222222"), + ), + ) + .await + .expect("the handler must return without parking on the held scan pool") + .expect("contention must never convert a landed push into an error"); assert_eq!( resp.status(), 200, - "the response is the receive-pack success, not a contention 5xx" + "the response is the receive-pack success, returned before the detached tail runs" ); + + // The detached tail is still owned: release admission and let it drain cleanly. + drop(held); + tokio::time::sleep(std::time::Duration::from_millis(300)).await; } // ---- #174 U4 (P2-2): post-push encryption task set bounded by per-repo coalescing ---- diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index c4d2b079..8396a369 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -449,13 +449,13 @@ impl Drop for EncryptInflightGuard { /// recovery copy or silently under-pin it. The returned permit must move into /// the blocking closure so a started scan always completes holding it (a /// disconnect cannot cancel `spawn_blocking` or leak the permit mid-walk). -/// Accepted residuals, stated once for every caller: (1) the park wait is -/// queue-depth multiplied — post-receive tails are no longer admission-bounded -/// once the write permit is released, so N landed pushes can queue N scans and -/// the last waits N scan-durations; (2) a client-timeout disconnect while -/// parked HERE drops the handler future and silently loses this push's -/// replication work — the `encrypt_inflight` coalescing requeue does NOT cover -/// it, because this park precedes the `try_begin` spawn gate. +/// Accepted residual, stated once for every caller: the park wait is queue-depth +/// multiplied — post-receive tails are no longer admission-bounded once the write +/// permit is released, so N landed pushes can queue N scans and the last waits N +/// scan-durations. A client-timeout disconnect no longer loses the work (#174 F2): +/// the whole post-receive replication tail runs in an independently owned task, so +/// dropping the request future cannot drop this parked scan — the park no longer +/// precedes any durable-record gate in a cancellable future. pub async fn acquire_scan_permit( scan_sem: Arc, repo: &std::path::Path, From 759c9b6aa387f15cc0f2f88ed3dfb1f0d13a8cc8 Mon Sep 17 00:00:00 2001 From: t Date: Sat, 18 Jul 2026 10:36:16 -0500 Subject: [PATCH 47/58] fix(node): bound the /ipfs cat-file probe and content read (#174 F3) The GET /ipfs/{cid} object-type probe and content read ran bare synchronous Command::output() on the async worker, budget-checked only before they started. A hung or corrupt object store therefore pinned a Tokio runtime worker and both held IPFS admission permits indefinitely, so enough requests exhausted the route despite GITLAWB_IPFS_REQUEST_BUDGET_SECS. The walk beside them already used spawn_blocking + the reaped bounded runner; these two stages were left bare. Extract run_bounded_git_raw (returns ExitStatus/stdout/stderr so callers can classify exit codes; run_bounded_git stays a thin bail-on-nonzero wrapper for the walk callers) and add store::object_type_bounded / read_object_content_bounded on it, preserving object_type's Ok(None)-on-absence vs Err classification so the serve path's 404-vs-503 semantics are unchanged. Run both /ipfs stages in spawn_blocking under a deadline clamped to min(git timeout, remaining budget), with the IPFS permits held across the awaited spawn_blocking. Test: get_by_cid_hung_probe_is_reaped_and_sheds_503 hangs git on a FIFO alternates with no feeder; the reaped probe sheds a 503 in bounded time (GREEN), while a neutralized deadline blocks the handler forever (RED). The existing probe-error / corrupt-repo tests still assert the absence-vs-error classification at the handler. --- crates/gitlawb-node/src/api/ipfs.rs | 158 +++++++++++------- crates/gitlawb-node/src/git/store.rs | 56 +++++++ .../gitlawb-node/src/git/visibility_pack.rs | 39 ++++- 3 files changed, 185 insertions(+), 68 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index 41080833..43e418ae 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -292,32 +292,52 @@ pub async fn get_by_cid( } }; - // Budget gate for the probe stage (F3). The probe subprocess has no - // internal duration clamp, so this pre-start check is its entire bound. - if budget_gate( + // Budget gate for the probe stage (F3): a probe is never STARTED with zero + // remaining, and a started probe now runs its git child under a deadline + // clamped to the remainder (below), so it can never complete past the budget. + let Some(probe_budget) = budget_gate( &mut truncated_by, request_deadline, state.config.ipfs_request_budget_secs, &repo.name, "object-type probe", - ) - .is_none() - { + ) else { break; - } + }; // Check whether the object exists in this repo before any expensive // reachability walk. This prevents random-CID spray from triggering // full-history git walks on repos that don't carry the object. Absent - // (`Ok(None)`) is a VERDICT; a probe that could not run is not. - let obj_type = match store::object_type(&repo_path, &sha256_hex) { - Ok(Some(t)) => t, - Ok(None) => continue, - Err(e) => { + // (`Ok(None)`) is a VERDICT; a probe that could not run is not. The + // `git cat-file -t` shells out, so run it OFF the async worker under the + // reaped bounded runner (#174 F3) — a hung/corrupt object store cannot pin a + // runtime worker or the held IPFS permits past the deadline. + let probe_deadline = std::time::Instant::now() + + std::cmp::min( + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + probe_budget, + ); + // The probe shells to the real `git` (as `object_type` historically did), + // independent of `state.git_bin` (which tests point at a fake walk git). + let probe_path = repo_path.clone(); + let probe_sha = sha256_hex.clone(); + let obj_type = match tokio::task::spawn_blocking(move || { + store::object_type_bounded("git", &probe_path, &probe_sha, probe_deadline) + }) + .await + { + Ok(Ok(Some(t))) => t, + Ok(Ok(None)) => continue, + Ok(Err(e)) => { tracing::warn!(repo = %repo.name, err = %e, "object-type probe failed during /ipfs scan; skipping repo without a verdict"); taint(&mut truncated_by, "probe"); continue; } + Err(join_err) => { + tracing::warn!(repo = %repo.name, err = %join_err, "object-type probe task panicked during /ipfs scan; skipping repo without a verdict"); + taint(&mut truncated_by, "probe"); + continue; + } }; // Per-blob gating only applies when a path-scoped rule exists (KTD4). @@ -413,28 +433,52 @@ pub async fn get_by_cid( // serving keeps the terminal arm honest and the stop unconditional; the // retryable 503 tells the caller to come back rather than letting an // over-budget request keep spending. - if budget_gate( + let Some(read_budget) = budget_gate( &mut truncated_by, request_deadline, state.config.ipfs_request_budget_secs, &repo.name, "content read", - ) - .is_none() - { + ) else { break; - } + }; // Now that we've passed the gate, read the content. A failed read after a // passed gate is not an absence verdict — the probe just said the object - // exists here — so the skip taints the scan. - let content = match store::read_object_content(&repo_path, &sha256_hex, &obj_type) { - Ok(c) => c, - Err(e) => { + // exists here — so the skip taints the scan. Like the probe, the read shells + // out to `git cat-file `, so run it OFF the async worker under the reaped + // bounded runner clamped to the remaining budget (#174 F3). + let read_deadline = std::time::Instant::now() + + std::cmp::min( + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + read_budget, + ); + // Real `git`, as the read historically used, independent of `state.git_bin`. + let read_path = repo_path.clone(); + let read_sha = sha256_hex.clone(); + let read_type = obj_type.clone(); + let content = match tokio::task::spawn_blocking(move || { + store::read_object_content_bounded( + "git", + &read_path, + &read_sha, + &read_type, + read_deadline, + ) + }) + .await + { + Ok(Ok(c)) => c, + Ok(Err(e)) => { tracing::warn!(repo = %repo.name, err = %e, "object content read failed during /ipfs scan; skipping repo without a verdict"); taint(&mut truncated_by, "read"); continue; } + Err(join_err) => { + tracing::warn!(repo = %repo.name, err = %join_err, "object content read task panicked during /ipfs scan; skipping repo without a verdict"); + taint(&mut truncated_by, "read"); + continue; + } }; // 3. Return the content with IPFS-style headers @@ -1539,22 +1583,22 @@ mod tests { ); } - /// F3 expiry between probe and walk: an object PROBED PRESENT in a - /// path-scoped repo is still not served once the budget is gone; the - /// walk-stage gate taints and stops before any walk starts. The probe is - /// stalled past the 1s budget while still SUCCEEDING via a FIFO at - /// `objects/info/alternates`: real `git cat-file -t` blocks opening it at - /// odb setup until a plain OS thread feeds it an empty alternates list at - /// ~2s, after which the probe finds the genuine loose object and reports - /// "blob". The walk-needing repo thus reaches the walk gate with zero - /// remaining: 503 naming the budget, never a 404 (the walk-gate deny of the - /// fake git's empty allowed-set would have been a verdict) and never a - /// serve, and the fake-git walk log stays EMPTY (no walk ever started). - /// MUTATION (RED): drop the walk-stage budget gate and this decays to the - /// walked 404 with one logged walk. + /// #174 F3 hung-probe reap (RED-before/GREEN-after): the `git cat-file -t` + /// probe runs OFF the async worker under the reaped bounded runner, so a hung or + /// corrupt object store cannot pin a runtime worker or the held IPFS permits. + /// `objects/info/alternates` is a FIFO with no writer, so real `git cat-file -t` + /// blocks at odb setup forever. With the probe bounded to + /// `min(git_service_timeout, remaining budget)` (~1s here), the watchdog tears the + /// git process group down at the deadline and the probe returns Err — a taint, not + /// a verdict — so the scan sheds a retryable 503 naming the probe, no walk ever + /// starts, and the whole request returns in bounded time. + /// + /// Load-bearing: with the probe on the bare async worker (pre-fix) this FIFO blocks + /// the handler forever (no feeder frees it) and the request hangs — the wrapping + /// timeout fires (RED). With the reaped bounded probe it returns 503 promptly. #[cfg(unix)] #[sqlx::test] - async fn get_by_cid_budget_expiry_between_probe_and_walk_sheds_503(pool: sqlx::PgPool) { + async fn get_by_cid_hung_probe_is_reaped_and_sheds_503(pool: sqlx::PgPool) { let tmp = tempfile::TempDir::new().unwrap(); let mut state = crate::test_support::test_state(pool.clone()).await; let repos_dir = tmp.path().join("repos"); @@ -1587,12 +1631,10 @@ mod tests { .await .unwrap(); - // Stall the REAL-git probe past the budget while still letting it - // succeed: `objects/info/alternates` as a FIFO blocks cat-file's odb - // setup until a writer appears. The feeder thread supplies an EMPTY - // alternates list (open + close, no bytes) after 2s, so the probe then - // resolves the genuine loose object. Non-blocking open with retries so - // a handler that never probes cannot wedge the feeder forever. + // Hang the REAL-git probe indefinitely: `objects/info/alternates` as a FIFO + // with no writer blocks `git cat-file -t` at odb setup forever. There is no + // feeder — the reaped bounded runner must tear the git process group down at + // the deadline; a bare unbounded probe would block the handler here. let rec = state .db .get_repo("z6f3probe", "gated") @@ -1611,28 +1653,16 @@ mod tests { 0, "mkfifo(objects/info/alternates) must succeed" ); - let feeder = std::thread::spawn(move || { - use std::os::unix::fs::OpenOptionsExt; - std::thread::sleep(std::time::Duration::from_secs(2)); - for _ in 0..300 { - if std::fs::OpenOptions::new() - .write(true) - .custom_flags(libc::O_NONBLOCK) - .open(&fifo) - .is_ok() - { - return; - } - std::thread::sleep(std::time::Duration::from_millis(50)); - } - }); - let peer: SocketAddr = "203.0.113.72:5000".parse().unwrap(); - let resp = ipfs_router(state) - .oneshot(get_cid(&cid_for_oid(&oid), Some(peer))) - .await - .unwrap(); - feeder.join().unwrap(); + // The request must return in bounded time: the reaped probe sheds a 503; a + // bare unbounded probe would block on the FIFO forever (no feeder frees it). + let resp = tokio::time::timeout( + std::time::Duration::from_secs(15), + ipfs_router(state).oneshot(get_cid(&cid_for_oid(&oid), Some(peer))), + ) + .await + .expect("the hung probe must be reaped, not block the handler") + .unwrap(); assert_eq!( resp.status(), StatusCode::SERVICE_UNAVAILABLE, @@ -1651,8 +1681,8 @@ mod tests { .unwrap(); let body = String::from_utf8_lossy(&body); assert!( - body.contains("budget"), - "the shed must name the budget taint; got: {body}" + body.contains("probe"), + "the shed must name the reaped-probe taint; got: {body}" ); let walks = std::fs::read_to_string(&walk_log) .map(|s| s.lines().count()) diff --git a/crates/gitlawb-node/src/git/store.rs b/crates/gitlawb-node/src/git/store.rs index 5697c229..d5a0d1a1 100644 --- a/crates/gitlawb-node/src/git/store.rs +++ b/crates/gitlawb-node/src/git/store.rs @@ -320,6 +320,62 @@ pub fn read_object_content(repo_path: &Path, sha256_hex: &str, obj_type: &str) - Ok(content_output.stdout) } +/// Bounded, reaped variant of [`object_type`] for the async `/ipfs` serve path +/// (#174 F3): runs `git cat-file -t` off the caller's runtime through the +/// process-group + watchdog reaper, so a hung or corrupt object store cannot pin a +/// runtime worker or an IPFS admission permit past `deadline`. Exit classification +/// is identical to [`object_type`] — a missing object is `Ok(None)`, a store-access +/// failure is `Err` — so the serve path's 404-vs-503 semantics are unchanged. +pub fn object_type_bounded( + git_bin: &str, + repo_path: &Path, + sha256_hex: &str, + deadline: std::time::Instant, +) -> Result> { + let (status, stdout, stderr) = crate::git::visibility_pack::run_bounded_git_raw( + git_bin, + &["cat-file", "-t", sha256_hex], + repo_path, + &[], + deadline, + )?; + if !status.success() { + // Same classification as `object_type` (a nonzero exit is an ABSENCE verdict + // only when git could examine the object store); U5 refines the absence arm + // with an out-of-band readability check. + let stderr = String::from_utf8_lossy(&stderr); + if stderr.contains("not a git repository") + || stderr.lines().any(|l| l.starts_with("error:")) + { + bail!("git cat-file -t failed: {}", stderr.trim()); + } + return Ok(None); + } + Ok(Some(String::from_utf8_lossy(&stdout).trim().to_string())) +} + +/// Bounded, reaped variant of [`read_object_content`] for the async `/ipfs` serve +/// path (#174 F3). Same teardown guarantees as [`object_type_bounded`]. +pub fn read_object_content_bounded( + git_bin: &str, + repo_path: &Path, + sha256_hex: &str, + obj_type: &str, + deadline: std::time::Instant, +) -> Result> { + let (status, stdout, stderr) = crate::git::visibility_pack::run_bounded_git_raw( + git_bin, + &["cat-file", obj_type, sha256_hex], + repo_path, + &[], + deadline, + )?; + if !status.success() { + bail!("git cat-file failed: {}", String::from_utf8_lossy(&stderr)); + } + Ok(stdout) +} + /// Read a git object by its SHA-256 hex object ID. /// /// Returns `(object_type, content_bytes)` where `content_bytes` is the raw diff --git a/crates/gitlawb-node/src/git/visibility_pack.rs b/crates/gitlawb-node/src/git/visibility_pack.rs index 9f07020d..e69c4ceb 100644 --- a/crates/gitlawb-node/src/git/visibility_pack.rs +++ b/crates/gitlawb-node/src/git/visibility_pack.rs @@ -67,13 +67,13 @@ fn child_terminated_without_reaping(pid: i32) -> bool { } #[cfg(unix)] -pub(crate) fn run_bounded_git( +pub(crate) fn run_bounded_git_raw( git_bin: &str, args: &[&str], repo_path: &Path, stdin_bytes: &[u8], deadline: Instant, -) -> Result> { +) -> Result<(std::process::ExitStatus, Vec, Vec)> { use std::io::{Read, Write}; use std::os::unix::process::CommandExt; use std::sync::mpsc::RecvTimeoutError; @@ -191,6 +191,23 @@ pub(crate) fn run_bounded_git( if killed && !status.success() { return Err(crate::git::smart_http::GitServiceTimeout.into()); } + Ok((status, out, err)) +} + +/// Bounded git returning only stdout, `bail!`ing on any nonzero exit. The thin +/// wrapper the walk callers use. Probes that must distinguish exit classes — +/// `git cat-file` absence vs an object-store access failure — call +/// [`run_bounded_git_raw`] and classify the status/stderr themselves. +#[cfg(unix)] +pub(crate) fn run_bounded_git( + git_bin: &str, + args: &[&str], + repo_path: &Path, + stdin_bytes: &[u8], + deadline: Instant, +) -> Result> { + let label = args.first().copied().unwrap_or("git"); + let (status, out, err) = run_bounded_git_raw(git_bin, args, repo_path, stdin_bytes, deadline)?; if !status.success() { anyhow::bail!("git {label} failed: {}", String::from_utf8_lossy(&err)); } @@ -210,13 +227,13 @@ pub(crate) fn run_bounded_git( /// the Unix version's signature and result semantics so every caller compiles on all /// targets (#174). #[cfg(not(unix))] -pub(crate) fn run_bounded_git( +pub(crate) fn run_bounded_git_raw( git_bin: &str, args: &[&str], repo_path: &Path, stdin_bytes: &[u8], deadline: Instant, -) -> Result> { +) -> Result<(std::process::ExitStatus, Vec, Vec)> { use std::io::{Read, Write}; use std::sync::mpsc::RecvTimeoutError; @@ -283,6 +300,20 @@ pub(crate) fn run_bounded_git( if killed && !status.success() { return Err(crate::git::smart_http::GitServiceTimeout.into()); } + Ok((status, out, err)) +} + +/// Non-Unix thin wrapper matching the Unix [`run_bounded_git`] semantics. +#[cfg(not(unix))] +pub(crate) fn run_bounded_git( + git_bin: &str, + args: &[&str], + repo_path: &Path, + stdin_bytes: &[u8], + deadline: Instant, +) -> Result> { + let label = args.first().copied().unwrap_or("git"); + let (status, out, err) = run_bounded_git_raw(git_bin, args, repo_path, stdin_bytes, deadline)?; if !status.success() { anyhow::bail!("git {label} failed: {}", String::from_utf8_lossy(&err)); } From 6dc2b2e912587b14d1b80c8db87426231d29be53 Mon Sep 17 00:00:00 2001 From: t Date: Sat, 18 Jul 2026 10:49:18 -0500 Subject: [PATCH 48/58] fix(node): share one whole-scan deadline across both full-scan phases (#174 F4) fail_closed_full_scan_objects ran phase 1 (replicable_blob_set_bounded) then phase 2 (all_blob_oids) with a FRESH Instant::now() + timeout for phase 2, under one held scan permit. A large-but-successful phase 1 plus a full phase 2 held the permit ~2x the configured budget instead of the one whole-scan budget. Compute one deadline before phase 1 and share it: phase 2 takes the shared Instant directly, phase 1 takes the remaining-until-deadline. Total occupancy is now ~1x. The completeness cost is honest: a repo whose phase 1 nears the budget under-pins this push (fail closed) rather than over-holding, so size the budget so both phases normally fit. Test: full_scan_shares_one_deadline_across_both_phases drives a slow-but- successful phase 1 that eats the budget; the shared deadline reaps phase 2 and the scan fails closed (empty). RED before: phase 2's fresh budget completed and kept the non-blob candidate. --- crates/gitlawb-node/src/api/repos.rs | 64 +++++++++++++++++++++++++++- 1 file changed, 62 insertions(+), 2 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 98e395f6..ede6c5e1 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -139,10 +139,25 @@ async fn fail_closed_full_scan_objects( crate::state::acquire_scan_permit(encrypt_sem, &disk_path, "fail-closed full scan").await; tokio::task::spawn_blocking(move || -> anyhow::Result> { let _permit = permit; + // One whole-scan deadline shared across both phases (#174 F4). A fresh + // `Instant::now() + timeout` for phase 2 let a large-but-successful phase 1 plus + // a full phase 2 hold the scan permit ~2x the configured budget. Sharing the + // deadline caps total occupancy at ~1x: phase 1 runs against the remaining + // budget, and if it consumes the budget phase 2 gets what is left and fails + // closed (pins nothing) rather than over-holding — the safe direction. The cost + // is honest: a genuinely large repo whose phase 1 nears the budget under-pins + // this push rather than the previous silent ~2x hold; size the budget so both + // phases normally fit. + let deadline = std::time::Instant::now() + timeout; let allowed = crate::git::visibility_pack::replicable_blob_set_bounded( - &disk_path, &git_bin, timeout, &rules, is_public, &owner_did, + &disk_path, + &git_bin, + deadline.saturating_duration_since(std::time::Instant::now()), + &rules, + is_public, + &owner_did, )?; - let all_blobs = crate::git::push_delta::all_blob_oids(&disk_path, &git_bin, std::time::Instant::now() + timeout)?; + let all_blobs = crate::git::push_delta::all_blob_oids(&disk_path, &git_bin, deadline)?; Ok(crate::git::visibility_pack::replicable_objects_fail_closed( candidates, &allowed, &all_blobs, )) @@ -5036,6 +5051,51 @@ mod tests { ); } + /// #174 F4 (RED-before/GREEN-after): the two full-scan phases share ONE whole-scan + /// deadline. Phase 1 (`replicable_blob_set_bounded`) succeeds but consumes almost + /// the whole budget; phase 2 (`all_blob_oids`) then gets only the remainder. With a + /// shared deadline phase 2 is reaped and the scan fails closed (pins nothing) — the + /// safe direction. With a FRESH `Instant::now() + timeout` for phase 2 (pre-fix) it + /// gets a full second budget, completes with an empty blob set, and the non-blob + /// candidate is kept — so the result is NON-empty (RED) and the permit is held ~2x. + #[cfg(unix)] + #[tokio::test] + async fn full_scan_shares_one_deadline_across_both_phases() { + use std::sync::Arc; + use std::time::Duration; + use tokio::sync::Semaphore; + + let tmp = tempfile::TempDir::new().unwrap(); + // Phase 1 (ls-tree) sleeps 1.5s and succeeds (empty tree); phase 2 + // (cat-file --batch-all-objects) sleeps 1.5s. With a 2s whole-scan budget the + // shared deadline leaves phase 2 only ~0.5s, so it is reaped; a fresh 2s budget + // would let it finish. + let body = "#!/bin/sh\ncase \"$1\" in\n rev-parse) echo deadbeef ;;\n rev-list) echo aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa ;;\n ls-tree) sleep 1.5 ;;\n cat-file) case \"$*\" in *--batch-all-objects*) sleep 1.5 ;; *) : ;; esac ;;\n *) : ;;\nesac\nexit 0\n"; + let git_bin = write_fake_git(tmp.path(), body); + // A candidate that is NOT a blob (never appears in all_blob_oids): kept by + // replicable_objects_fail_closed only if phase 2 actually ran to completion. + let candidates = vec!["cccccccccccccccccccccccccccccccccccccccc".to_string()]; + + let sem: Arc = Arc::new(Semaphore::new(1)); + let objs = fail_closed_full_scan_objects( + sem, + tmp.path().to_path_buf(), + vec![vis_rule("/secret/**", &[])], + true, + OWNER_DID.to_string(), + candidates, + git_bin, + Duration::from_secs(2), + ) + .await; + assert!( + objs.is_empty(), + "a large-but-successful phase 1 must leave phase 2 only the SHARED whole-scan \ + remainder, so it reaps and the scan fails closed; got {objs:?} (a fresh phase-2 \ + budget completed the scan and kept the candidate — the ~2x-budget bug)" + ); + } + /// Shared fixture for the F4 handler-layer tests: a state whose repo_store and /// git_bin point at the given tempdir/fake-git, plus a seeded on-disk repo, /// optionally with a path-scoped rule (so the post-receive walks actually run). From d39fc8de9cf5a3f15c42d23675fc5741f0412302 Mon Sep 17 00:00:00 2001 From: t Date: Sat, 18 Jul 2026 10:57:16 -0500 Subject: [PATCH 49/58] fix(node): disambiguate absence from an unreadable object store in the /ipfs probe (#174 F5) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit object_type_bounded treated any bare `fatal:` cat-file exit as absence. A packed object whose pack/idx is transiently unreadable (permissions, or a mid-repack race) emits `could not get object info` — byte-identical to a genuine miss — so a present object could return Ok(None) and the /ipfs scan a definitive 404 instead of the intended retryable 503. The stderr cannot separate the two (the CID is pre-validated, so that string is also the only legitimate absence path), so the disambiguation must be out of band. On the collided fatal, probe object_store_readable (objects/ listable, every pack/idx openable). Unreadable -> Err (taint -> 503). Readable -> re-probe once; still absent -> Ok(None), now present -> the type. This narrows but does not close the concurrent-repack TOCTOU (the check samples a different instant), stated honestly rather than claimed deterministic. Test: object_type_bounded_unreadable_pack_is_error_not_absence packs a real repo, chmod 000 the pack, and asserts Err (root-guarded). RED with the readability check disabled: the present blob returned Ok(None). --- crates/gitlawb-node/src/git/store.rs | 199 +++++++++++++++++++++++++-- 1 file changed, 188 insertions(+), 11 deletions(-) diff --git a/crates/gitlawb-node/src/git/store.rs b/crates/gitlawb-node/src/git/store.rs index d5a0d1a1..f9d65e9d 100644 --- a/crates/gitlawb-node/src/git/store.rs +++ b/crates/gitlawb-node/src/git/store.rs @@ -339,19 +339,86 @@ pub fn object_type_bounded( &[], deadline, )?; - if !status.success() { - // Same classification as `object_type` (a nonzero exit is an ABSENCE verdict - // only when git could examine the object store); U5 refines the absence arm - // with an out-of-band readability check. - let stderr = String::from_utf8_lossy(&stderr); - if stderr.contains("not a git repository") - || stderr.lines().any(|l| l.starts_with("error:")) - { - bail!("git cat-file -t failed: {}", stderr.trim()); + if status.success() { + return Ok(Some(String::from_utf8_lossy(&stdout).trim().to_string())); + } + // A nonzero exit whose stderr proves git could not EXAMINE the object store — + // "not a git repository", or any `error:` line (corrupt loose object, bad idx) — + // is not an absence verdict, so surface it as Err (#173/#174: never a false 404). + let stderr = String::from_utf8_lossy(&stderr); + if stderr.contains("not a git repository") || stderr.lines().any(|l| l.starts_with("error:")) { + bail!("git cat-file -t failed: {}", stderr.trim()); + } + // Any other bare `fatal:` ("could not get object info") is the absence-vs- + // unreadable-pack COLLISION (#174 F5): a genuinely missing object and a packed + // object whose pack/idx is unreadable (permissions, or a mid-repack race) emit the + // identical string. Disambiguate OUT OF BAND: if the object store is not readable, + // this is not an absence verdict -> Err (taint -> retryable 503). + if !object_store_readable(repo_path) { + bail!( + "git cat-file -t inconclusive: object store not readable at {} (not an absence verdict)", + repo_path.display() + ); + } + // Store readable: re-probe once. An object still absent on a confirmed-readable + // store is very likely truly absent (Ok(None)); if a mid-repack race resolved and + // the re-probe now finds it, return the type. This narrows, but cannot fully close, + // the concurrent-repack window (the readability check samples a different instant). + let (status2, stdout2, stderr2) = crate::git::visibility_pack::run_bounded_git_raw( + git_bin, + &["cat-file", "-t", sha256_hex], + repo_path, + &[], + deadline, + )?; + if status2.success() { + return Ok(Some(String::from_utf8_lossy(&stdout2).trim().to_string())); + } + let stderr2 = String::from_utf8_lossy(&stderr2); + if stderr2.contains("not a git repository") || stderr2.lines().any(|l| l.starts_with("error:")) + { + bail!("git cat-file -t failed: {}", stderr2.trim()); + } + Ok(None) +} + +/// Best-effort check that a repo's object store is readable, used to disambiguate a +/// genuine missing-object `git cat-file` fatal from an unreadable or racing pack +/// (both emit "could not get object info"). Returns false on any unreadable +/// `objects/` dir or any pack/idx that cannot be opened (EACCES / EIO), so the +/// caller surfaces an error rather than a false absence. Cheap — a couple of readdir +/// plus open probes. It narrows, but does not close, the concurrent-repack TOCTOU: it +/// samples a different instant than the failing cat-file. +fn object_store_readable(repo_path: &Path) -> bool { + let objects = repo_path.join("objects"); + // The objects dir itself must be listable; drain the iterator so a mid-listing + // EACCES/EIO surfaces, not just the initial open. + let Ok(entries) = std::fs::read_dir(&objects) else { + return false; + }; + for entry in entries { + if entry.is_err() { + return false; } - return Ok(None); } - Ok(Some(String::from_utf8_lossy(&stdout).trim().to_string())) + // Every pack file and its index must be openable for read. A loose-only store + // (no pack dir) is fine — the objects readdir above already proved reachability. + if let Ok(pack_entries) = std::fs::read_dir(objects.join("pack")) { + for entry in pack_entries { + let Ok(entry) = entry else { + return false; + }; + let path = entry.path(); + if matches!( + path.extension().and_then(|s| s.to_str()), + Some("pack") | Some("idx") + ) && std::fs::File::open(&path).is_err() + { + return false; + } + } + } + true } /// Bounded, reaped variant of [`read_object_content`] for the async `/ipfs` serve @@ -571,4 +638,114 @@ mod tests { "unchanged file must not appear: {names:?}" ); } + + /// #174 F5 (RED-before/GREEN-after): a packed object whose pack/idx is unreadable + /// makes `git cat-file -t` emit "could not get object info" — byte-identical to a + /// genuine miss. `object_type_bounded` must report absence ONLY when the object + /// store is confirmed readable; an unreadable store is Err (-> retryable 503), + /// never Ok(None) (-> a wrong 404 for a present object). + #[cfg(unix)] + #[test] + fn object_type_bounded_unreadable_pack_is_error_not_absence() { + use std::os::unix::fs::PermissionsExt; + let td = tempfile::TempDir::new().unwrap(); + let work = td.path().join("work"); + let bare = td.path().join("bare.git"); + std::fs::create_dir_all(&work).unwrap(); + let g = |args: &[&str], dir: &Path| { + assert!( + Command::new("git") + .args(args) + .current_dir(dir) + .status() + .unwrap() + .success(), + "git {args:?}" + ); + }; + g(&["init", "-q", "--object-format=sha256", "."], &work); + g(&["config", "user.email", "t@t"], &work); + g(&["config", "user.name", "t"], &work); + std::fs::write(work.join("file.txt"), b"packed f5 content\n").unwrap(); + g(&["add", "file.txt"], &work); + g(&["commit", "-qm", "c1"], &work); + let blob = String::from_utf8( + Command::new("git") + .args(["rev-parse", "HEAD:file.txt"]) + .current_dir(&work) + .output() + .unwrap() + .stdout, + ) + .unwrap() + .trim() + .to_string(); + g( + &[ + "clone", + "-q", + "--bare", + work.to_str().unwrap(), + bare.to_str().unwrap(), + ], + td.path(), + ); + g(&["gc", "-q"], &bare); + + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + + // Readable store: the packed blob probes present; a genuine miss is Ok(None). + assert_eq!( + super::object_type_bounded("git", &bare, &blob, deadline) + .unwrap() + .as_deref(), + Some("blob"), + "a packed blob on a readable store must probe present" + ); + assert!( + super::object_type_bounded("git", &bare, &"0".repeat(64), deadline) + .unwrap() + .is_none(), + "a genuinely-absent object on a readable store must be Ok(None)" + ); + + // Make the pack unreadable: cat-file -t now emits the collided fatal for the + // PRESENT blob. It must surface as Err, not a false Ok(None). + let pack_dir = bare.join("objects").join("pack"); + let set_pack_mode = |mode: u32| { + for e in std::fs::read_dir(&pack_dir).unwrap() { + let p = e.unwrap().path(); + if matches!( + p.extension().and_then(|s| s.to_str()), + Some("pack") | Some("idx") + ) { + let mut perms = std::fs::metadata(&p).unwrap().permissions(); + perms.set_mode(mode); + std::fs::set_permissions(&p, perms).unwrap(); + } + } + }; + set_pack_mode(0o000); + // Root bypasses file permissions, so the chmod won't block reads there; only + // assert the error path when the pack is genuinely unreadable to this process. + let a_pack = std::fs::read_dir(&pack_dir) + .unwrap() + .filter_map(|e| e.ok()) + .map(|e| e.path()) + .find(|p| p.extension().and_then(|s| s.to_str()) == Some("pack")); + let genuinely_unreadable = a_pack + .as_ref() + .map(|p| std::fs::File::open(p).is_err()) + .unwrap_or(false); + let res = super::object_type_bounded("git", &bare, &blob, deadline); + set_pack_mode(0o644); // restore so TempDir cleanup succeeds + + if genuinely_unreadable { + assert!( + res.is_err(), + "an unreadable pack must surface as Err (-> retryable 503), not Ok(None) \ + (-> a wrong 404 for a present object); got {res:?}" + ); + } + } } From 448d38458defead064141573626410a3d68ff9bd Mon Sep 17 00:00:00 2001 From: t Date: Sat, 18 Jul 2026 11:13:32 -0500 Subject: [PATCH 50/58] fix(node): bound concurrent pin-loop memory; document the per-repo task cap (#174 F6) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit EncryptInflight bounds the outstanding post-push encryption-task set to one per repo, but its state.rs doc implied a global cap it does not deliver, and each pin loop (ipfs_pin + pinata pin_new_objects) holds a full per-push object-id list while walking it — so N distinct authenticated first-pushes could hold N MB-scale lists at once, unbounded. Add a global pin_semaphore (GITLAWB_MAX_CONCURRENT_PIN_TASKS, default 8) and gate BOTH pin loops through it: they DEFER when the pool is full, never drop a pin. Tighten the state.rs comment to state the bound is PER-REPO, with the cross-repo residual (throttled by auth + rate limits, memory bounded by the pin permit) called out honestly. Also correct .env.example GITLAWB_DB_MAX_CONNECTIONS (20 -> 48): with the #174 F1 boot validation, 20 is below max_concurrent_git_pushes(32)+headroom(8) and would fail startup — closing the INV-24 doc gap from the F1 change. Tests: pin_new_objects_gated_defers_when_pin_pool_exhausted (RED without the permit acquire); max_concurrent_pin_tasks range/default. --- .env.example | 14 ++++- crates/gitlawb-node/src/api/repos.rs | 83 ++++++++++++++++++++++++- crates/gitlawb-node/src/auth/mod.rs | 1 + crates/gitlawb-node/src/config.rs | 38 +++++++++++ crates/gitlawb-node/src/main.rs | 3 + crates/gitlawb-node/src/state.rs | 29 ++++++--- crates/gitlawb-node/src/test_support.rs | 1 + 7 files changed, 155 insertions(+), 14 deletions(-) diff --git a/.env.example b/.env.example index 8f9bfa2e..9913b41b 100644 --- a/.env.example +++ b/.env.example @@ -24,8 +24,11 @@ DATABASE_URL=postgresql://gitlawb:changeme@localhost:5432/gitlawb # ── Database pool & startup resilience ──────────────────────────────────── # Maximum connections in the PostgreSQL pool. A cap, not a floor — # connections open lazily. Size against the DB server's max_connections, -# remembering admin tooling opens its own pool. -GITLAWB_DB_MAX_CONNECTIONS=20 +# remembering admin tooling opens its own pool. Each concurrent write pins one +# connection for its whole duration (the connection-affine advisory lock), so the +# node REJECTS at boot any value below GITLAWB_MAX_CONCURRENT_GIT_PUSHES + 8 +# headroom — keep this comfortably above that (default 48 for pushes 32). +GITLAWB_DB_MAX_CONNECTIONS=48 # Seconds a request waits for a pool connection before failing with 503. GITLAWB_DB_ACQUIRE_TIMEOUT_SECS=5 # Upper bound on each startup connect+migrate attempt, in seconds. Keep it @@ -141,6 +144,13 @@ GITLAWB_MAX_CONCURRENT_GIT_OPS=128 # Retry-After. Default 32. GITLAWB_MAX_CONCURRENT_GIT_PUSHES=32 +# Max concurrent post-push pin loops (IPFS + Pinata pin_new_objects) across all +# repos. Each loop holds a full per-push object-id list while pinning, so this +# bounds that MB-scale memory even though the per-repo encrypt-task set already +# caps the task COUNT. A loop DEFERS (waits) when the pool is full, never drops a +# pin. Default 8. +GITLAWB_MAX_CONCURRENT_PIN_TASKS=8 + # Max concurrent read ops (upload-pack + the upload-pack info/refs advertisement) # a single caller may hold, so one caller cannot monopolize the read pool. Keyed # on the resolved SOURCE IP, never the DID: a signature does not move a caller off diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index ede6c5e1..6e6d5f3a 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -849,6 +849,7 @@ struct EncryptTaskCtx { git_bin: String, git_timeout: std::time::Duration, encrypt_sem: Arc, + pin_sem: Arc, } /// The detached post-push pin/encrypt task (#174 P2-2 + F5): run this push's own @@ -990,14 +991,40 @@ async fn resolve_drain_object_list( /// The pin/encrypt pipeline shared by the snapshot iteration and the /// coalesced-drain iterations: local IPFS pin, then (path-scoped rules only) the /// admission-gated recipients walk → encrypt-then-pin → Arweave manifest anchor. +/// Pin `object_list` to the local IPFS node under the global pin-admission permit +/// (#174 F6). `EncryptInflight` bounds the pin-task COUNT to one per repo, but each +/// pin loop holds a full per-push object-id list while walking it; this permit bounds +/// how many such MB-scale lists are held at once across all repos. DEFERS (waits) when +/// the pool is full — never drops, since a dropped pin loses the replication copy. +async fn pin_new_objects_gated( + pin_sem: &Arc, + ipfs_api: &str, + repo_path: &std::path::Path, + object_list: Vec, + db: &Arc, +) -> Vec<(String, String)> { + let _permit = pin_sem + .clone() + .acquire_owned() + .await + .expect("pin_semaphore is never closed"); + crate::ipfs_pin::pin_new_objects(ipfs_api, repo_path, object_list, db).await +} + async fn pin_and_encrypt_objects( ctx: &EncryptTaskCtx, object_list: Vec, rules: Option>, is_public: bool, ) { - let pinned = - crate::ipfs_pin::pin_new_objects(&ctx.ipfs_api, &ctx.repo_path, object_list, &ctx.db).await; + let pinned = pin_new_objects_gated( + &ctx.pin_sem, + &ctx.ipfs_api, + &ctx.repo_path, + object_list, + &ctx.db, + ) + .await; if !pinned.is_empty() { tracing::info!(count = pinned.len(), "pinned git objects to IPFS"); for (sha, cid) in &pinned { @@ -1782,6 +1809,7 @@ pub async fn git_receive_pack( state.config.git_service_timeout_secs, ), encrypt_sem: state.git_encrypt_semaphore.clone(), + pin_sem: state.pin_semaphore.clone(), }; tokio::spawn(run_encrypt_pin_task( ctx, @@ -1833,8 +1861,16 @@ pub async fn git_receive_pack( let node_keypair = Arc::clone(&state.node_keypair); let object_list_pinata = object_list; let do_pinata_replication = withheld.is_some(); + // Same global pin-admission bound as the IPFS loop (#174 F6): the Pinata pin + // loop also holds this push's full object-id list while pinning it, so it must + // share the cap. It DEFERS on a full pool rather than dropping the pin. + let pin_sem_pinata = state.pin_semaphore.clone(); tokio::spawn(async move { let pinned = if do_pinata_replication { + let _pin_permit = pin_sem_pinata + .acquire_owned() + .await + .expect("pin_semaphore is never closed"); crate::pinata::pin_new_objects( &http_client, &pinata_upload_url, @@ -5096,6 +5132,48 @@ mod tests { ); } + /// #174 F6 (RED-before/GREEN-after): a post-push pin loop holds this push's full + /// object-id list while walking it, so concurrent pin loops across many repos must + /// be bounded by a global permit, not just the per-repo task count. `pin_new_objects_gated` + /// DEFERS (waits) when the pin pool is exhausted rather than running unbounded. + /// + /// Load-bearing: without the permit acquire the pin loop runs immediately even with + /// the pool held (RED — the deferral assertion fails). With it, it parks. + #[sqlx::test] + async fn pin_new_objects_gated_defers_when_pin_pool_exhausted(pool: sqlx::PgPool) { + use std::sync::Arc; + use tokio::sync::Semaphore; + + let state = crate::test_support::test_state(pool).await; + let db = state.db.clone(); + let tmp = tempfile::TempDir::new().unwrap(); + let pin_sem = Arc::new(Semaphore::new(1)); + // Hold the only pin permit. + let held = pin_sem.clone().acquire_owned().await.unwrap(); + + // Empty ipfs_api makes the pin itself a no-op, but the loop must still DEFER on + // the exhausted pin pool rather than run. + let blocked = tokio::time::timeout( + std::time::Duration::from_millis(500), + pin_new_objects_gated(&pin_sem, "", tmp.path(), vec![], &db), + ) + .await; + assert!( + blocked.is_err(), + "a pin loop must defer while the pin pool is exhausted (#174 F6)" + ); + + // Release admission: the SAME call now completes. + drop(held); + let out = tokio::time::timeout( + std::time::Duration::from_secs(5), + pin_new_objects_gated(&pin_sem, "", tmp.path(), vec![], &db), + ) + .await + .expect("the pin loop completes once admission frees"); + assert!(out.is_empty(), "an empty ipfs_api pins nothing"); + } + /// Shared fixture for the F4 handler-layer tests: a state whose repo_store and /// git_bin point at the given tempdir/fake-git, plus a seeded on-disk repo, /// optionally with a path-scoped rule (so the post-receive walks actually run). @@ -5884,6 +5962,7 @@ mod tests { git_bin: git_bin.to_string(), git_timeout: std::time::Duration::from_secs(600), encrypt_sem: sem, + pin_sem: std::sync::Arc::new(tokio::sync::Semaphore::new(64)), } } diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index 912108a0..c1fa6624 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -524,6 +524,7 @@ mod tests { git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + pin_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), encrypt_inflight: crate::state::EncryptInflight::new(), git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), git_push_advert_per_caller: diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index 5fb66b55..d70be5ea 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -319,6 +319,24 @@ pub struct Config { )] pub max_concurrent_git_pushes: usize, + /// Max concurrent post-push pin loops (`ipfs_pin` and `pinata` + /// `pin_new_objects`) across all repos. `EncryptInflight` bounds the outstanding + /// pin-task COUNT to one per repo, but each pin loop holds a full per-push + /// object-id list (up to `git_max_pack_bytes` worth of OIDs) while it walks it, + /// so N distinct repos could hold N such lists at once. This caps how many run + /// concurrently, bounding that MB-scale memory regardless of how many repos an + /// authenticated actor pushes to (#174 F6). Beyond it a pin loop DEFERS (waits), + /// never drops — a dropped pin would lose the object's replication copy. + /// + /// Default: 8. Must be between 1 and 1_048_576. + #[arg( + long, + env = "GITLAWB_MAX_CONCURRENT_PIN_TASKS", + default_value_t = 8, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub max_concurrent_pin_tasks: usize, + /// Maximum concurrent read operations (`upload-pack` and the upload-pack /// `info/refs` advertisement) a single caller may hold at once, so one caller /// cannot monopolize the `max_concurrent_git_ops` read pool (#174). Callers are @@ -518,6 +536,26 @@ mod tests { ); } + #[test] + fn max_concurrent_pin_tasks_defaults_and_rejects_out_of_range() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).max_concurrent_pin_tasks, + 8 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--max-concurrent-pin-tasks", "2"]) + .max_concurrent_pin_tasks, + 2 + ); + assert!( + Config::try_parse_from(["gitlawb-node", "--max-concurrent-pin-tasks", "0"]).is_err() + ); + assert!( + Config::try_parse_from(["gitlawb-node", "--max-concurrent-pin-tasks", "1048577"]) + .is_err() + ); + } + #[test] fn max_concurrent_git_ops_defaults_and_rejects_out_of_range() { assert_eq!( diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index b7bc758e..4c4c1da8 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -402,6 +402,9 @@ async fn main() -> Result<()> { git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new( config.max_concurrent_git_pushes, )), + // Bounds concurrent post-push pin loops (their MB-scale object-id lists) across + // all repos (#174 F6), independent of the per-repo encrypt-task coalescing below. + pin_semaphore: Arc::new(tokio::sync::Semaphore::new(config.max_concurrent_pin_tasks)), // Coalesces the DETACHED post-push encryption tasks per repo so a rapid pusher // cannot grow the outstanding parked-waiter set past one task per repo (#174 // P2-2). No knob: it is a natural cap (one entry per distinct repo), not a diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 8396a369..77799bd0 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -127,16 +127,25 @@ pub struct AppState { /// walk must not hold a foreground write slot, and a handler already holding a /// write permit that needed a second would self-deadlock at pool size 1. pub git_encrypt_semaphore: Arc, - /// Bounds the OUTSTANDING post-push encryption-task set by per-repo coalescing - /// (#174 P2-2). `git_encrypt_semaphore` caps *active* walks; this caps how many - /// detached tasks *spawn and park* on that semaphore's `acquire_owned().await`. - /// Before spawning a per-push encryption task, the receive-pack handler consults - /// this set: if the repo already has a task in flight it coalesces (skips the - /// duplicate spawn) rather than parking a new unbounded waiter, and its tip pairs - /// are recorded for that task's drain loop (#174 F5). Coalescing only delays the - /// coalesced push's walk — it never drops the withheld-blob recovery copy, which - /// `2a54c15` deliberately kept fail-closed (there is no reconciliation sweep to - /// re-derive a dropped copy). See [`EncryptInflight`]. + /// Bounds concurrent post-push pin loops (`ipfs_pin` / `pinata` `pin_new_objects`) + /// across all repos (#174 F6). `encrypt_inflight` caps the pin-task COUNT to one + /// per repo, but each pin loop holds a full per-push object-id list while walking + /// it, so N distinct repos could hold N such MB-scale lists at once. This caps how + /// many run concurrently; a loop DEFERS (waits) when the pool is full, never drops. + pub pin_semaphore: Arc, + /// Bounds the outstanding post-push encryption-task set to at most one PER REPO by + /// coalescing (#174 P2-2). This is NOT a global cap: N distinct repos still admit N + /// tasks; the cross-repo residual (an authenticated actor pushing to many repos + /// leaves many parked tasks) is throttled by auth plus the per-IP/per-DID rate + /// limits, and its real cost — the MB-scale per-push object-id list each pin loop + /// holds — is bounded by `pin_semaphore`, not this. `git_encrypt_semaphore` caps + /// *active* walks; this caps duplicate SPAWNS per repo. Before spawning a per-push + /// encryption task, the receive-pack handler consults this set: if the repo already + /// has a task in flight it coalesces (skips the duplicate spawn) rather than parking + /// a new waiter, and its tip pairs are recorded for that task's drain loop (#174 F5). + /// Coalescing only delays the coalesced push's walk — it never drops the withheld-blob + /// recovery copy, which `2a54c15` deliberately kept fail-closed (there is no + /// reconciliation sweep to re-derive a dropped copy). See [`EncryptInflight`]. pub encrypt_inflight: EncryptInflight, /// Per-caller concurrency sub-cap on the read pool: each caller (keyed on the /// resolved source IP, #174 U1) may hold at most `max_concurrent_reads_per_caller` diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index df095b44..cd58f19e 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -87,6 +87,7 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + pin_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), encrypt_inflight: crate::state::EncryptInflight::new(), git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys( From 93cdf308baad7b31043e4f8c96e96ab2e24f4d44 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 20 Jul 2026 09:14:01 -0500 Subject: [PATCH 51/58] fix(node): make RepoWriteGuard::release cancellation-safe (#174 F4) Await pg_advisory_unlock while the pooled connection is still owned by self; take it and set released only after the await resolves. A cancel mid-unlock now leaves the Drop backstop armed (conn still Some), so the session-level advisory lock is no longer leaked onto a pooled connection. Adds a cancel-mid-unlock regression test (via a test-only pre-unlock gate) and an append-only inv22 gate row asserting the take/released orderings. --- crates/gitlawb-node/src/git/repo_store.rs | 151 +++++++++++++++++++++- crates/gitlawb-node/tests/inv22_gates.rs | 47 +++++++ 2 files changed, 195 insertions(+), 3 deletions(-) diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 86a1ee8b..d88594e7 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -180,6 +180,8 @@ impl RepoStore { locked: false, released: false, tigris: self.tigris.clone(), + #[cfg(test)] + test_pre_unlock_gate: None, }; // Acquire the advisory lock with retry, through the guard's OWN connection, @@ -390,6 +392,13 @@ pub struct RepoWriteGuard { /// Set once `release` has run its unlock, making the `Drop` backstop inert. released: bool, tigris: Option, + /// Test-only seam: when set, `release` parks on this gate at the exact point + /// it is about to await `pg_advisory_unlock` (connection still owned, not yet + /// released). Dropping the `release` future while it is parked reproduces a + /// mid-unlock cancellation, so a test can assert the `Drop` backstop still + /// frees the session lock. Never set outside tests. + #[cfg(test)] + test_pre_unlock_gate: Option>, } impl RepoWriteGuard { @@ -419,10 +428,22 @@ impl RepoWriteGuard { } // Release the advisory lock on the SAME connection that took it (session - // advisory locks are connection-affine). Taking the connection here makes - // the Drop backstop inert. + // advisory locks are connection-affine). Unlock through the connection + // while it is STILL owned by `self` — do not `take()` it first. If this + // future is cancelled during the unlock await, `self` is dropped with + // `conn == Some(..)` and `released == false`, so the `Drop` backstop still + // runs the detached unlock. `released` is set only AFTER the await + // resolves, so a cancellation cannot make the backstop inert (#174 F4). if self.locked { - if let Some(mut conn) = self.conn.take() { + #[cfg(test)] + let pre_unlock_gate = self.test_pre_unlock_gate.clone(); + if let Some(conn) = self.conn.as_deref_mut() { + // Test-only: park right before the unlock await so a test can drop + // this future mid-unlock (connection owned, not yet released). + #[cfg(test)] + if let Some(gate) = pre_unlock_gate { + gate.notified().await; + } let _ = sqlx::query("SELECT pg_advisory_unlock($1)") .bind(self.lock_key) .execute(&mut *conn) @@ -719,4 +740,128 @@ mod tests { .execute(&mut *checker) .await; } + + // ── cancellation-safe unlock (#174 F4, RED-before/GREEN-after) ────────── + + /// F4 (P1): a cancellation DURING the unlock await must still free the session + /// advisory lock. `release` unlocks through the connection while `self` still + /// owns it, so if the future is dropped mid-unlock, `Drop` sees `conn == Some` + /// + `locked && !released` and runs its detached-unlock backstop. A test-only + /// gate parks `release` at the exact pre-unlock point; dropping the future + /// there reproduces the cancellation. + /// + /// Load-bearing: RED on the original ordering (`self.conn.take()` before the + /// await → at cancellation `self.conn == None` → `Drop` skips → the local + /// connection returns to the pool with the session lock still held → the + /// checker's `pg_try_advisory_lock` returns false). GREEN after the reorder. + #[sqlx::test] + async fn write_guard_release_cancelled_mid_unlock_frees_the_lock(pool: sqlx::PgPool) { + let dir = tempfile::TempDir::new().unwrap(); + let store = RepoStore::for_testing(dir.path().to_path_buf(), pool.clone()); + let owner = "did:key:z6MkCancelMidUnlockProofCCCCCCCCCCCCCCCCCC"; + let name = "canceltest"; + let slug = owner.replace([':', '/'], "_"); + let key = advisory_lock_key(&slug, name); + + // Distinct session for the probe, held out of the pool before acquiring. + let mut checker = pool.acquire().await.expect("checker connection"); + + let mut guard = store.acquire_write(owner, name).await.expect("acquire"); + // Arm the pre-unlock gate; it is never notified, so `release` parks on it + // with the connection still owned and `released` still false. + let gate = Arc::new(tokio::sync::Notify::new()); + guard.test_pre_unlock_gate = Some(gate); + + // Box the future so we can drop it ourselves (tokio::pin! keeps it alive to + // end of scope, which would defer the guard's Drop past the assertions). + let mut fut = Box::pin(guard.release(false)); + let parked = + tokio::time::timeout(std::time::Duration::from_millis(300), fut.as_mut()).await; + assert!( + parked.is_err(), + "release should park on the pre-unlock gate, not complete" + ); + // Cancel mid-unlock: dropping the boxed future runs RepoWriteGuard::drop. + drop(fut); + + // Let the Drop backstop's detached unlock task run. + tokio::time::sleep(std::time::Duration::from_millis(400)).await; + + let (free,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut *checker) + .await + .unwrap(); + assert!( + free, + "advisory lock must be freed when release is cancelled mid-unlock — the \ + connection must stay owned by the guard so Drop's backstop can run" + ); + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(key) + .execute(&mut *checker) + .await; + } + + /// F4: the ordinary success path still frees the lock, and a second write + /// acquire on the same repo returns promptly (no stale-lock retry loop). + #[sqlx::test] + async fn write_guard_release_true_frees_lock_and_second_acquire_succeeds(pool: sqlx::PgPool) { + let dir = tempfile::TempDir::new().unwrap(); + let store = RepoStore::for_testing(dir.path().to_path_buf(), pool.clone()); + let owner = "did:key:z6MkReleaseTrueProofDDDDDDDDDDDDDDDDDDDDDD"; + let name = "reltruetest"; + + let guard = store + .acquire_write(owner, name) + .await + .expect("first acquire"); + guard.release(true).await; + + let again = tokio::time::timeout( + std::time::Duration::from_secs(2), + store.acquire_write(owner, name), + ) + .await + .expect("second acquire_write must not hit the ~60s stale-lock retry loop") + .expect("second acquire"); + again.release(true).await; + } + + /// F4: releasing a guard that never took the lock (`locked == false`, the state + /// acquire_write leaves after a failed acquisition) must not unlock or panic. + #[sqlx::test] + async fn write_guard_release_when_not_locked_does_not_unlock_or_panic(pool: sqlx::PgPool) { + let dir = tempfile::TempDir::new().unwrap(); + let owner = "did:key:z6MkNotLockedProofEEEEEEEEEEEEEEEEEEEEEE"; + let name = "notlockedtest"; + let slug = owner.replace([':', '/'], "_"); + let key = advisory_lock_key(&slug, name); + + let guard = RepoWriteGuard { + owner_slug: slug, + repo_name: name.to_string(), + local_path: dir.path().to_path_buf(), + lock_key: key, + conn: Some(pool.acquire().await.expect("conn")), + locked: false, + released: false, + tigris: None, + test_pre_unlock_gate: None, + }; + // Must complete without panic and issue no unlock. + guard.release(false).await; + + let mut checker = pool.acquire().await.expect("checker"); + let (free,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut *checker) + .await + .unwrap(); + assert!(free, "release on an unlocked guard must not touch the key"); + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(key) + .execute(&mut *checker) + .await; + } } diff --git a/crates/gitlawb-node/tests/inv22_gates.rs b/crates/gitlawb-node/tests/inv22_gates.rs index cbfa0d4c..d21ec194 100644 --- a/crates/gitlawb-node/tests/inv22_gates.rs +++ b/crates/gitlawb-node/tests/inv22_gates.rs @@ -153,3 +153,50 @@ fn inv22_concurrency_gates_present_and_not_bypassed() { ); } } + +/// F4 (repo_store advisory-unlock cancellation safety): `RepoWriteGuard::release` +/// must await `pg_advisory_unlock` while `self` still owns the pooled connection, +/// and must not mark itself `released` until that await resolves. Either shape, +/// reintroduced, re-opens the mid-unlock cancellation leak: taking the connection +/// early leaves `Drop` with `conn == None`, and setting `released = true` early +/// leaves the `Drop` backstop inert — both strand the session lock on cancellation. +/// +/// Scoped to the `release` fn body: the `Drop` impl legitimately takes the +/// connection and unlocks, so a whole-file scan would match it and read as a false +/// pass. Reverting either ordering turns this red (proven load-bearing). +#[test] +fn f4_release_keeps_conn_owned_until_unlock_resolves() { + let repo_store = src("git/repo_store.rs"); + + let rel_start = repo_store + .find("pub async fn release(mut self") + .expect("F4 gate: repo_store.rs no longer defines RepoWriteGuard::release"); + let rel_end = repo_store[rel_start..] + .find("impl Drop for RepoWriteGuard") + .map(|off| rel_start + off) + .expect("F4 gate: release fn / Drop impl markers moved — update this guard"); + let release_body = &repo_store[rel_start..rel_end]; + + let unlock = release_body + .find("pg_advisory_unlock") + .expect("F4 gate: release must still issue pg_advisory_unlock"); + let before_unlock = &release_body[..unlock]; + + // (a) the connection must still be owned by `self` at the unlock await. + assert!( + !before_unlock.contains("self.conn.take()"), + "F4 regression: RepoWriteGuard::release takes self.conn BEFORE awaiting \ + pg_advisory_unlock. A cancellation during the unlock await then strands the \ + session advisory lock (Drop sees conn == None and skips its backstop). \ + Unlock through the still-owned connection instead." + ); + // (b) `released` must not be set before the unlock await — the other + // reintroduction shape a single-reorder check on (a) alone is blind to. + assert!( + !before_unlock.contains("released = true"), + "F4 regression: RepoWriteGuard::release sets `released = true` BEFORE awaiting \ + pg_advisory_unlock. A cancellation during the await then leaves the Drop \ + backstop inert (it early-returns on released). Set released only AFTER the \ + unlock await resolves." + ); +} From e0e207e6455dd3b51aa85085611e6b06c2c3f770 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 20 Jul 2026 09:36:09 -0500 Subject: [PATCH 52/58] fix(node): classify /ipfs object probe with a version-stable signal (#174 F5) Probe existence via `git cat-file --batch-check` (structured ` missing` on exit 0) instead of matching English `cat-file -t` prose, so a git wording change cannot flip a miss into an error or vice-versa. A bad-config/corrupt repo on a readable store is now a terminal, non-retryable 500 (not a false 404, and not a retryable 503 that would fan out cat-file retries); a transient unreadable store stays 503. Error bodies are opaque (no raw git stderr). Layers onto the existing object_store_readable + re-probe disambiguation. --- crates/gitlawb-node/src/api/ipfs.rs | 126 ++++++++- crates/gitlawb-node/src/git/store.rs | 369 +++++++++++++++++++++++---- 2 files changed, 445 insertions(+), 50 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index 43e418ae..1910a14b 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -61,6 +61,13 @@ use crate::visibility::{visibility_check, Decision}; /// the truncation sources — existing content is never misreported absent /// because of unrelated repos or transient faults. /// +/// Deterministic fault (F5/U4): a candidate repo that is persistently broken (a +/// corrupt repo, a bad `.git/config`) also yields no absence verdict, but a retry +/// cannot fix it, so a scan that found nothing sheds a TERMINAL, non-retryable 500 +/// (opaque body) rather than the retryable 503 — checked first so a deterministic +/// fault is never downgraded, and gated on nothing-served so a healthy repo that +/// carries the object still serves. +/// /// Request budget (F3): one absolute clock (`ipfs_request_budget_secs`) spans /// the whole admitted request. No stage (acquire, probe, walk, content read) /// starts once it is exhausted, and the acquire wait and walk deadline are @@ -181,6 +188,13 @@ pub async fn get_by_cid( truncated_by.push(source); } } + // A DETERMINISTIC probe fault (a corrupt repo / bad `.git/config`; #174 F5/U4) is + // separate from a transient taint: a retry cannot fix it, so a scan that found + // nothing must NOT shed the retryable 503 (which would invite a conformant client + // to retry-storm a fresh `git cat-file` per attempt against the broken repo). It + // sheds a terminal, non-retryable 500 instead — but only if nothing served, so one + // corrupt repo never masks a healthy repo that carries the object. + let mut deterministic_fault = false; // Budget gate shared by the four per-stage checks (F3): the remaining // request budget, or — once exhausted — None, after logging the stage and @@ -328,11 +342,22 @@ pub async fn get_by_cid( { Ok(Ok(Some(t))) => t, Ok(Ok(None)) => continue, - Ok(Err(e)) => { - tracing::warn!(repo = %repo.name, err = %e, "object-type probe failed during /ipfs scan; skipping repo without a verdict"); + // Transient probe fault (unreadable/mid-repack store): unproven, retryable. + Ok(Err(store::ProbeError::Transient(e))) => { + tracing::warn!(repo = %repo.name, err = %e, "object-type probe hit a transient store fault during /ipfs scan; skipping repo without a verdict"); taint(&mut truncated_by, "probe"); continue; } + // Deterministic probe fault (corrupt repo / bad config): a retry cannot fix + // it, so it does NOT taint (which would shed a retryable 503). It records a + // terminal condition that the terminal arm renders as a non-retryable 500 + // only if nothing served. The raw git detail stays in the log; the client + // body is opaque. + Ok(Err(store::ProbeError::Deterministic(e))) => { + tracing::warn!(repo = %repo.name, err = %e, "object-type probe hit a deterministic fault (corrupt repo/config) during /ipfs scan; skipping repo without a verdict"); + deterministic_fault = true; + continue; + } Err(join_err) => { tracing::warn!(repo = %repo.name, err = %join_err, "object-type probe task panicked during /ipfs scan; skipping repo without a verdict"); taint(&mut truncated_by, "probe"); @@ -500,6 +525,21 @@ pub async fn get_by_cid( return Ok((StatusCode::OK, headers, content).into_response()); } + // Deterministic fault (F5/U4): a candidate repo is persistently broken (corrupt + // repo / bad `.git/config`), so the object is not proven absent AND a retry cannot + // change that. Shed a TERMINAL, non-retryable 500 rather than the retryable 503 + // below — a 503 would let a conformant client retry-storm a fresh `git cat-file` + // per attempt against the broken repo. This is checked before the transient 503 so + // a deterministic fault is never downgraded to a retryable status. The body is + // opaque (a generic message via `AppError::Git` -> 500, no Retry-After): the raw + // git stderr — which leaks filesystem paths / config — was logged at the probe, and + // never reaches the client. + if deterministic_fault { + return Err(AppError::Git( + "ipfs object probe could not complete: a candidate repository is corrupt".into(), + )); + } + // Truncated scan (F2): at least one candidate repo yielded no verdict, so the // object is not proven absent. A 404 here would misreport existing content, so // shed retryable instead — Overloaded is the single 503 + Retry-After site in @@ -1195,6 +1235,88 @@ mod tests { ); } + /// #174 F5/U4 (RED-before/GREEN-after): a candidate repo with a corrupt + /// `.git/config` makes `git cat-file` die with `fatal: bad config line N` while + /// `objects/` stays readable. That is a DETERMINISTIC fault, not an absence, and a + /// retry cannot fix it — so the scan must shed a TERMINAL, non-retryable 500, never + /// the old false 404 (`Ok(None)` fell through) and never the retryable 503 (which + /// would invite a conformant client to retry-storm a fresh `cat-file` per attempt). + /// A second, healthy repo probes clean (absent verdict); the corrupt row is what + /// forces the 500. The body must be OPAQUE — no raw git stderr, no filesystem path. + /// MUTATION (RED): route the deterministic fault back to `Ok(None)` in + /// `object_type_bounded` and this decays to a 404; classify it Transient and it + /// decays to a retryable 503. + #[sqlx::test] + async fn get_by_cid_bad_config_repo_is_terminal_500_not_404_or_503(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + // A healthy repo that probes clean (would give a definitive 404 on its own) plus + // a repo whose bare git dir has a corrupt config (objects/ intact). + seed_repo_with_blob(&state, tmp.path(), "z6f5clean", "real", b"probe clean\n").await; + state + .db + .upsert_mirror_repo("z6f5badcfg", "broken", "/unused-badcfg", None, false) + .await + .unwrap(); + let rec = state + .db + .get_repo("z6f5badcfg", "broken") + .await + .unwrap() + .unwrap(); + let bare = state + .repo_store + .acquire(&rec.owner_did, &rec.name) + .await + .unwrap(); + std::fs::create_dir_all(&bare).unwrap(); + run_git(&["init", "-q", "--bare", "--object-format=sha256"], &bare); + // Corrupt the config; leave objects/ readable (the readable-store + git-fails + // combination is exactly what makes this deterministic, not transient). + { + use std::io::Write; + let mut cfg = std::fs::OpenOptions::new() + .append(true) + .open(bare.join("config")) + .unwrap(); + cfg.write_all(b"\n[broken section\nnot a valid = = = line\n") + .unwrap(); + } + + let peer: SocketAddr = "203.0.113.69:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&valid_cid(), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::INTERNAL_SERVER_ERROR, + "a bad-config (deterministic) repo fault must shed a terminal 500, never a \ + 404 (false absence) or a retryable 503" + ); + assert!( + resp.headers().get("retry-after").is_none(), + "a terminal 500 must NOT advertise a retry (that is the whole point vs 503)" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + let body = String::from_utf8_lossy(&body); + assert!( + !body.contains("bad config") + && !body.contains(tmp.path().to_str().unwrap()) + && !body.contains(".git") + && !body.contains("fatal"), + "the 500 body must be opaque — no raw git stderr / config text / filesystem \ + path; got: {body}" + ); + } + /// F2 read taint: the gate passes (the probe reads the truncated loose object's /// intact "blob 64" header) but the content read fails (`cat-file blob` dies on /// the deflate stream cut mid-content) — the probe just said the object EXISTS diff --git a/crates/gitlawb-node/src/git/store.rs b/crates/gitlawb-node/src/git/store.rs index f9d65e9d..a18ab472 100644 --- a/crates/gitlawb-node/src/git/store.rs +++ b/crates/gitlawb-node/src/git/store.rs @@ -320,66 +320,164 @@ pub fn read_object_content(repo_path: &Path, sha256_hex: &str, obj_type: &str) - Ok(content_output.stdout) } -/// Bounded, reaped variant of [`object_type`] for the async `/ipfs` serve path -/// (#174 F3): runs `git cat-file -t` off the caller's runtime through the -/// process-group + watchdog reaper, so a hung or corrupt object store cannot pin a -/// runtime worker or an IPFS admission permit past `deadline`. Exit classification -/// is identical to [`object_type`] — a missing object is `Ok(None)`, a store-access -/// failure is `Err` — so the serve path's 404-vs-503 semantics are unchanged. -pub fn object_type_bounded( +/// Why an `/ipfs` existence probe could not return an absence verdict (#174 F5/U4). +/// The caller (`api::ipfs`) maps the variant to an HTTP status, and the split exists +/// only for that mapping: a `Transient` fault is retryable (503), a `Deterministic` +/// fault is terminal (500). +/// +/// The discriminator is object-store readability, NOT any English `git` wording, so a +/// future `git` message change cannot silently reclassify a fault (KTD-4): if the +/// store cannot be read (an unreadable or mid-repack pack, a removed `objects/` dir, +/// a permissions fault) the fault may clear on its own -> `Transient`; if the store IS +/// readable yet `git` still fails (a corrupt repo, a bad `.git/config`) a retry cannot +/// fix it -> `Deterministic`, so a conformant client is told not to retry-storm a +/// fresh `git cat-file` per attempt against a persistently broken repo. +#[derive(Debug)] +pub enum ProbeError { + /// Retryable (-> 503): the object store could not be read right now. + Transient(anyhow::Error), + /// Terminal (-> 500): a persistent, deterministic fault a retry cannot fix. + Deterministic(anyhow::Error), +} + +impl std::fmt::Display for ProbeError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ProbeError::Transient(e) => write!(f, "transient probe fault: {e}"), + ProbeError::Deterministic(e) => write!(f, "deterministic probe fault: {e}"), + } + } +} + +impl std::error::Error for ProbeError {} + +/// Structured outcome of one `git cat-file --batch-check` existence probe. +enum BatchProbe { + /// Present: git printed ` ` on exit 0. + Present(String), + /// A structured, CLEAN absence: git printed ` missing` on exit 0 with no + /// `error:` diagnostics on stderr. This is the ONLY signal that can become an + /// `Ok(None)` (404) absence verdict — and even then only after the store-readable + /// disambiguation below, since an unreadable pack ALSO prints a clean `missing`. + Missing, + /// git could not honestly examine the object store: a hard exit (bad config, not a + /// git repo) OR an exit-0 `missing` accompanied by `error:` diagnostics (a corrupt + /// loose object still prints `missing` on stdout but complains on stderr). Never an + /// absence verdict. Carries the raw git detail for the server log only. + Fault(anyhow::Error), +} + +/// One bounded, reaped `git cat-file --batch-check` probe (the oid is fed on stdin, so +/// no prose is matched to decide presence — the `missing` token and the exit code are +/// the structured signal). See [`object_type_bounded`] for the surrounding teardown +/// guarantees. +fn batch_check_probe( git_bin: &str, repo_path: &Path, sha256_hex: &str, deadline: std::time::Instant, -) -> Result> { +) -> std::result::Result { + let stdin = format!("{sha256_hex}\n"); let (status, stdout, stderr) = crate::git::visibility_pack::run_bounded_git_raw( git_bin, - &["cat-file", "-t", sha256_hex], + &["cat-file", "--batch-check"], repo_path, - &[], + stdin.as_bytes(), deadline, - )?; - if status.success() { - return Ok(Some(String::from_utf8_lossy(&stdout).trim().to_string())); - } - // A nonzero exit whose stderr proves git could not EXAMINE the object store — - // "not a git repository", or any `error:` line (corrupt loose object, bad idx) — - // is not an absence verdict, so surface it as Err (#173/#174: never a false 404). + ) + // A spawn/timeout failure of the reaped child is not deterministic — retry it. + .map_err(ProbeError::Transient)?; + let stderr = String::from_utf8_lossy(&stderr); - if stderr.contains("not a git repository") || stderr.lines().any(|l| l.starts_with("error:")) { - bail!("git cat-file -t failed: {}", stderr.trim()); - } - // Any other bare `fatal:` ("could not get object info") is the absence-vs- - // unreadable-pack COLLISION (#174 F5): a genuinely missing object and a packed - // object whose pack/idx is unreadable (permissions, or a mid-repack race) emit the - // identical string. Disambiguate OUT OF BAND: if the object store is not readable, - // this is not an absence verdict -> Err (taint -> retryable 503). - if !object_store_readable(repo_path) { - bail!( - "git cat-file -t inconclusive: object store not readable at {} (not an absence verdict)", - repo_path.display() - ); + // A corrupt object makes `--batch-check` print `missing` on stdout (exit 0) yet + // emit `error:` lines on stderr; those diagnostics disqualify a clean-absence read + // regardless of the exit code, so they are checked before anything else. + let has_error_diag = stderr.lines().any(|l| l.starts_with("error:")); + + if status.success() && !has_error_diag { + let line = String::from_utf8_lossy(&stdout); + let line = line.trim(); + // ` missing` is the structured absence token; ` ` is a + // hit. Anything else on a "success" exit is unexpected and not an absence. + if line + .rsplit(' ') + .next() + .is_some_and(|last| last == "missing") + { + return Ok(BatchProbe::Missing); + } + let mut parts = line.split_whitespace(); + if let (Some(_oid), Some(ty), Some(_size)) = (parts.next(), parts.next(), parts.next()) { + return Ok(BatchProbe::Present(ty.to_string())); + } + return Ok(BatchProbe::Fault(anyhow::anyhow!( + "unexpected git cat-file --batch-check output: {line:?}" + ))); } - // Store readable: re-probe once. An object still absent on a confirmed-readable - // store is very likely truly absent (Ok(None)); if a mid-repack race resolved and - // the re-probe now finds it, return the type. This narrows, but cannot fully close, - // the concurrent-repack window (the readability check samples a different instant). - let (status2, stdout2, stderr2) = crate::git::visibility_pack::run_bounded_git_raw( - git_bin, - &["cat-file", "-t", sha256_hex], - repo_path, - &[], - deadline, - )?; - if status2.success() { - return Ok(Some(String::from_utf8_lossy(&stdout2).trim().to_string())); + + Ok(BatchProbe::Fault(anyhow::anyhow!( + "git cat-file --batch-check failed (exit {:?}): {}", + status.code(), + stderr.trim() + ))) +} + +/// Bounded, reaped variant of [`object_type`] for the async `/ipfs` serve path +/// (#174 F3/F5): runs `git cat-file --batch-check` off the caller's runtime through the +/// process-group + watchdog reaper, so a hung or corrupt object store cannot pin a +/// runtime worker or an IPFS admission permit past `deadline`. +/// +/// Absence is keyed on `--batch-check`'s STRUCTURED ` missing` token on exit 0, +/// never on any English `fatal:` wording (KTD-4): a genuinely-absent object is the only +/// `Ok(None)` (404) path. A probe that could not honestly examine the store is a +/// [`ProbeError`], split by object-store readability into `Transient` (retryable 503) +/// and `Deterministic` (terminal 500) so the serve path can shed the right status. +pub fn object_type_bounded( + git_bin: &str, + repo_path: &Path, + sha256_hex: &str, + deadline: std::time::Instant, +) -> std::result::Result, ProbeError> { + match batch_check_probe(git_bin, repo_path, sha256_hex, deadline)? { + BatchProbe::Present(ty) => Ok(Some(ty)), + BatchProbe::Fault(detail) => Err(classify_store_fault(repo_path, detail)), + BatchProbe::Missing => { + // A clean `missing` is the absence-vs-unreadable-pack COLLISION (#174 F5): + // a genuinely missing object AND a packed object whose pack/idx is + // unreadable (permissions, or a mid-repack race) both print an identical + // clean `missing`. Disambiguate OUT OF BAND on store readability — an + // unreadable store is not an absence verdict (taint -> retryable 503). + if !object_store_readable(repo_path) { + return Err(ProbeError::Transient(anyhow::anyhow!( + "git cat-file inconclusive: object store not readable at {} (not an absence verdict)", + repo_path.display() + ))); + } + // Store readable: re-probe once. Still `missing` on a confirmed-readable + // store is very likely truly absent (Ok(None)); a mid-repack race that + // resolved returns the type. This narrows, but cannot fully close, the + // concurrent-repack window (the readability check samples a different + // instant than the failing probe). + match batch_check_probe(git_bin, repo_path, sha256_hex, deadline)? { + BatchProbe::Present(ty) => Ok(Some(ty)), + BatchProbe::Fault(detail) => Err(classify_store_fault(repo_path, detail)), + BatchProbe::Missing => Ok(None), + } + } } - let stderr2 = String::from_utf8_lossy(&stderr2); - if stderr2.contains("not a git repository") || stderr2.lines().any(|l| l.starts_with("error:")) - { - bail!("git cat-file -t failed: {}", stderr2.trim()); +} + +/// Classify a probe fault by object-store readability (#174 F5/U4). An unreadable store +/// may be a transient permissions/mid-repack condition (retryable 503); a readable +/// store on which git still fails is a persistent, deterministic fault — a corrupt repo +/// or a bad `.git/config` — that a retry cannot fix (terminal 500). The `detail` is +/// carried for the server log; the client-facing body is opaque (set by the caller). +fn classify_store_fault(repo_path: &Path, detail: anyhow::Error) -> ProbeError { + if object_store_readable(repo_path) { + ProbeError::Deterministic(detail) + } else { + ProbeError::Transient(detail) } - Ok(None) } /// Best-effort check that a repo's object store is readable, used to disambiguate a @@ -748,4 +846,179 @@ mod tests { ); } } + + /// Shared setup: a bare sha256 repo carrying one committed blob. Returns the repo + /// path and the blob's oid. + #[cfg(unix)] + fn bare_repo_with_blob(td: &std::path::Path) -> (std::path::PathBuf, String) { + let work = td.join("work"); + let bare = td.join("bare.git"); + std::fs::create_dir_all(&work).unwrap(); + let g = |args: &[&str], dir: &Path| { + assert!( + Command::new("git") + .args(args) + .current_dir(dir) + .status() + .unwrap() + .success(), + "git {args:?}" + ); + }; + g(&["init", "-q", "--object-format=sha256", "."], &work); + g(&["config", "user.email", "t@t"], &work); + g(&["config", "user.name", "t"], &work); + std::fs::write(work.join("file.txt"), b"f5 u4 content\n").unwrap(); + g(&["add", "file.txt"], &work); + g(&["commit", "-qm", "c1"], &work); + let blob = String::from_utf8( + Command::new("git") + .args(["rev-parse", "HEAD:file.txt"]) + .current_dir(&work) + .output() + .unwrap() + .stdout, + ) + .unwrap() + .trim() + .to_string(); + g( + &[ + "clone", + "-q", + "--bare", + work.to_str().unwrap(), + bare.to_str().unwrap(), + ], + td, + ); + (bare, blob) + } + + /// #174 F5/U4 (RED-before/GREEN-after, the CORE regression guard): a repo with a + /// corrupt `.git/config` makes `git cat-file` die with `fatal: bad config line N` + /// (exit 128, NO `error:` line) while `objects/` stays fully readable. The old + /// `-t` path let that fall through to `Ok(None)` — a false 404 for content that + /// may well exist. `object_type_bounded` must instead classify it as a + /// DETERMINISTIC fault (a retry cannot fix it) so the serve path renders a + /// terminal 500, never a 404 and never a retryable 503. + /// + /// LOAD-BEARING: revert the classification (route `BatchProbe::Fault` on a readable + /// store back to `Ok(None)`, or drop the `has_error_diag`/exit checks so a hard + /// `fatal:` is read as `missing`) and this goes RED — the probe reports the corrupt + /// repo as an absent object. + #[cfg(unix)] + #[test] + fn object_type_bounded_bad_config_is_deterministic_not_absence() { + let td = tempfile::TempDir::new().unwrap(); + let (bare, blob) = bare_repo_with_blob(td.path()); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + + // Baseline on the healthy repo: present blob probes present, genuine miss is + // the clean Ok(None) absence verdict. + assert_eq!( + super::object_type_bounded("git", &bare, &blob, deadline) + .unwrap() + .as_deref(), + Some("blob"), + "a present blob on a healthy readable store must probe present" + ); + assert!( + super::object_type_bounded("git", &bare, &"0".repeat(64), deadline) + .unwrap() + .is_none(), + "a genuinely-absent object on a healthy readable store must be Ok(None) (404)" + ); + + // Corrupt the config; objects/ is untouched (and stays readable). + { + use std::io::Write; + let mut cfg = std::fs::OpenOptions::new() + .append(true) + .open(bare.join("config")) + .unwrap(); + cfg.write_all(b"\n[broken section\nnot a valid = = = line\n") + .unwrap(); + } + assert!( + super::object_store_readable(&bare), + "config corruption must leave objects/ readable (that is the whole point: \ + a readable store + a git failure == deterministic, not transient)" + ); + + // Probing the PRESENT blob under the bad config must be a DETERMINISTIC fault, + // never Ok(None) (the old false 404) and never a Transient (retryable 503). + let res = super::object_type_bounded("git", &bare, &blob, deadline); + assert!( + matches!(res, Err(super::ProbeError::Deterministic(_))), + "a bad-config fatal on a readable store must be a terminal Deterministic \ + fault (-> 500), never Ok(None) (-> false 404) or Transient (-> 503); got {res:?}" + ); + // And a genuinely-absent oid under the bad config is ALSO not an absence verdict. + let res_absent = super::object_type_bounded("git", &bare, &"0".repeat(64), deadline); + assert!( + matches!(res_absent, Err(super::ProbeError::Deterministic(_))), + "even a would-be-absent oid must not read as Ok(None) once the config is \ + corrupt; got {res_absent:?}" + ); + } + + /// #174 F5/U4: a corrupt LOOSE object makes `git cat-file --batch-check` print + /// ` missing` on stdout (exit 0) yet emit `error:` diagnostics on stderr. The + /// clean-`missing` absence path must NOT fire here — the `error:` line disqualifies + /// a clean-absence read — so the probe surfaces a fault, not a false Ok(None) 404. + /// The object store is readable (a corrupt object file still opens), so this is a + /// Deterministic fault. LOAD-BEARING: drop the `has_error_diag` guard and a corrupt + /// object reads as `missing` -> Ok(None) -> false 404 (RED). + #[cfg(unix)] + #[test] + fn object_type_bounded_corrupt_loose_object_is_fault_not_absence() { + use std::os::unix::fs::PermissionsExt; + let td = tempfile::TempDir::new().unwrap(); + let work = td.path().join("loose"); + std::fs::create_dir_all(&work).unwrap(); + let g = |args: &[&str]| { + assert!( + Command::new("git") + .args(args) + .current_dir(&work) + .status() + .unwrap() + .success(), + "git {args:?}" + ); + }; + g(&["init", "-q", "--object-format=sha256", "."]); + g(&["config", "user.email", "t@t"]); + g(&["config", "user.name", "t"]); + std::fs::write(work.join("f.txt"), b"loose object content\n").unwrap(); + g(&["add", "f.txt"]); + g(&["commit", "-qm", "c1"]); + let blob = String::from_utf8( + Command::new("git") + .args(["rev-parse", "HEAD:f.txt"]) + .current_dir(&work) + .output() + .unwrap() + .stdout, + ) + .unwrap() + .trim() + .to_string(); + + // Overwrite the loose object file with non-zlib garbage (it is 0o444 by default). + let obj = work.join(".git/objects").join(&blob[0..2]).join(&blob[2..]); + let mut perms = std::fs::metadata(&obj).unwrap().permissions(); + perms.set_mode(0o644); + std::fs::set_permissions(&obj, perms).unwrap(); + std::fs::write(&obj, b"garbage not a zlib stream").unwrap(); + + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + let res = super::object_type_bounded("git", &work, &blob, deadline); + assert!( + res.is_err(), + "a corrupt loose object (error: on stderr, `missing` on stdout) must be Err, \ + never a false Ok(None) 404; got {res:?}" + ); + } } From 9180467973c5b25b3e2a20f401c7bd532d91abf1 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 20 Jul 2026 10:00:40 -0500 Subject: [PATCH 53/58] fix(node): clamp the initial /ipfs metadata queries to the request budget (#174 F6) list_all_repos and list_visibility_rules_for_repos ran as bare awaits while holding the scarce IPFS walk permits, so a query blocked in Postgres pinned the slots past GITLAWB_IPFS_REQUEST_BUDGET_SECS and shed later requests as 503s. Wrap both in the remaining request-deadline budget; on timeout the RAII permits drop and the handler sheds a retryable budget 503. Fails closed: a timeout on the visibility-rules query denies the request rather than serving an unfiltered listing. Appends an inv22 gate row asserting both queries stay deadline-wrapped. --- crates/gitlawb-node/src/api/ipfs.rs | 263 ++++++++++++++++++++++- crates/gitlawb-node/tests/inv22_gates.rs | 41 ++++ 2 files changed, 293 insertions(+), 11 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index 1910a14b..7f977a81 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -148,22 +148,65 @@ pub async fn get_by_cid( None => None, }; - // 2. Search all repos for an object with this SHA-256 - let repos = state - .db - .list_all_repos() - .await - .map_err(AppError::Internal)?; + // 2. Search all repos for an object with this SHA-256. + // + // F6/KTD-5: both initial metadata queries run while the scarce walk permits + // acquired above are ALREADY held (RAII, for the whole request) and BEFORE the + // per-repo loop's first budget gate. With no pool statement_timeout, a query + // blocked in Postgres would otherwise pin those walk slots for the entire stall + // — past the request budget — capacity-503'ing later requests. Clamp BOTH to the + // remaining request budget. On timeout return the same retryable budget 503 the + // later stages shed (the "budget" source); returning here drops the RAII permits, + // freeing the slot. list_visibility_rules_for_repos is the access-control query, + // so its timeout returns BEFORE the loop — the scan can NEVER run with an empty + // rule map and serve an unfiltered listing that exposes private repos (FAIL CLOSED). + let repos = match tokio::time::timeout( + request_deadline.saturating_duration_since(std::time::Instant::now()), + state.db.list_all_repos(), + ) + .await + { + Ok(Ok(repos)) => repos, + Ok(Err(e)) => return Err(AppError::Internal(e)), + Err(_elapsed) => { + tracing::warn!( + budget_secs = state.config.ipfs_request_budget_secs, + "/ipfs list_all_repos exceeded the request budget \ + (GITLAWB_IPFS_REQUEST_BUDGET_SECS); shedding a retryable 503 and freeing the walk permit" + ); + return Err(AppError::Overloaded(format!( + "ipfs scan incomplete (budget) for CID {cid_str}; retry shortly" + ))); + } + }; // Fetch every repo's visibility rules in one query rather than one per row // (the gate runs each row against its OWN rules — KTD2a). A row absent from // the map has no rules. let repo_ids: Vec = repos.iter().map(|r| r.id.clone()).collect(); - let rules_by_repo = state - .db - .list_visibility_rules_for_repos(&repo_ids) - .await - .map_err(AppError::Internal)?; + let rules_by_repo = match tokio::time::timeout( + request_deadline.saturating_duration_since(std::time::Instant::now()), + state.db.list_visibility_rules_for_repos(&repo_ids), + ) + .await + { + Ok(Ok(rules)) => rules, + Ok(Err(e)) => return Err(AppError::Internal(e)), + // FAIL CLOSED (security-critical): a timeout on the access-control query must + // DENY. Returning here — before the loop — means the scan can never fall + // through and apply an empty rule map, which would serve an unfiltered listing + // exposing private repos. It sheds the same retryable budget 503 as above. + Err(_elapsed) => { + tracing::warn!( + budget_secs = state.config.ipfs_request_budget_secs, + "/ipfs list_visibility_rules_for_repos exceeded the request budget \ + (GITLAWB_IPFS_REQUEST_BUDGET_SECS); denying (fail closed) and freeing the walk permit" + ); + return Err(AppError::Overloaded(format!( + "ipfs scan incomplete (budget) for CID {cid_str}; retry shortly" + ))); + } + }; // Request-scoped memo of the per-repo allowed-blob set (KTD1, #126). The // caller is constant for one request, so `repo.id` alone is a safe, @@ -2416,4 +2459,202 @@ mod tests { "a different IP must not be braked by another IP's exhausted bucket" ); } + + /// F6/KTD-5: the two initial metadata queries (`list_all_repos`, + /// `list_visibility_rules_for_repos`) run AFTER the scarce walk permits are + /// acquired (held RAII for the whole request) but BEFORE the per-repo loop's + /// first budget gate. Pre-fix they were bare awaits with no deadline, so a query + /// blocked in Postgres pinned the walk slot for the whole stall, past the request + /// budget. Here we hold an ACCESS EXCLUSIVE lock on `repos` so `list_all_repos` + /// blocks; with the budget clamp the request sheds a retryable budget 503 within + /// ~budget and FREES the walk permit, and a follow-up (lock released) is served. + /// + /// Load-bearing: pre-fix the bare await blocks on the lock until the 10s wrapping + /// timeout fires (RED — "never returned within budget"). After the fix it returns + /// the 503 at ~1s and the permit is free again. MUTATION (RED): drop the + /// `tokio::time::timeout` around `list_all_repos` and this hangs past the wrap. + #[sqlx::test] + async fn get_by_cid_stalled_metadata_query_frees_walk_permit(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool.clone()).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // Global walk pool of 1 so the held/freed permit is directly observable; + // per-source cap permissive so only the global pool matters. + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(1)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + let mut cfg = (*state.config).clone(); + cfg.ipfs_request_budget_secs = 1; + state.config = Arc::new(cfg); + + let sem = state.git_ipfs_walk_semaphore.clone(); + let router = ipfs_router(state); + + // Hold an ACCESS EXCLUSIVE lock on `repos` on a dedicated pooled connection: + // `list_all_repos`' SELECT needs ACCESS SHARE, which conflicts, so it blocks + // at lock acquisition regardless of row count. + let mut lock_conn = pool.acquire().await.unwrap(); + sqlx::raw_sql("BEGIN; LOCK TABLE repos IN ACCESS EXCLUSIVE MODE;") + .execute(&mut *lock_conn) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.80:5000".parse().unwrap(); + let started = std::time::Instant::now(); + let resp = tokio::time::timeout( + std::time::Duration::from_secs(10), + router.clone().oneshot(get_cid(&valid_cid(), Some(peer))), + ) + .await + .expect("the budget clamp must return within budget; a bare await hangs on the lock") + .unwrap(); + let elapsed = started.elapsed(); + + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a metadata query blocked past the request budget must shed a retryable 503" + ); + assert!( + elapsed < std::time::Duration::from_secs(3), + "the clamp must end the request at ~budget (1s); got {elapsed:?} \ + (pre-fix the bare await blocks on the lock for the whole stall)" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + let body = String::from_utf8_lossy(&body); + assert!( + body.contains("budget"), + "the shed must name the budget taint so it maps to \ + GITLAWB_IPFS_REQUEST_BUDGET_SECS; got: {body}" + ); + // The scarce walk permit was RAII-dropped on the early return, not pinned for + // the stall: the slot is free again the instant the request returns. + assert_eq!( + sem.available_permits(), + 1, + "the walk permit must be freed on the budget-shed path, not held for the stall" + ); + + // Release the lock; a follow-up request is now SERVED (404 — empty DB), never + // capacity-503'd, proving the slot was not left pinned. + sqlx::raw_sql("ROLLBACK") + .execute(&mut *lock_conn) + .await + .unwrap(); + drop(lock_conn); + let resp2 = router + .oneshot(get_cid(&valid_cid(), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp2.status(), + StatusCode::NOT_FOUND, + "with the permit freed and the lock released, a follow-up is served (404), \ + not capacity-503'd" + ); + } + + /// F6/KTD-5 FAIL CLOSED (security-critical): `list_visibility_rules_for_repos` is + /// the access-control query. If its timeout let the handler fall through with an + /// empty rule map, the loop would apply no visibility rules and serve an unfiltered + /// listing — exposing a public repo's path-restricted blob. Here a PUBLIC repo + /// carries the blob under a path-scoped rule that denies anon; `visibility_rules` + /// is locked ACCESS EXCLUSIVE so the rule query blocks. The fix returns the budget + /// 503 BEFORE the loop, so the handler NEVER serves (never 200). + /// + /// Load-bearing: pre-fix the bare await blocks on the lock until the 10s wrap fires + /// (RED). After the fix it sheds the 503 at ~1s. The `assert_ne!(200)` is the + /// fail-closed guard: a naive fix that `unwrap_or_default()`s the rules on timeout + /// and falls through would serve the blob 200 and trip it. + #[sqlx::test] + async fn get_by_cid_visibility_rule_timeout_fails_closed(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool.clone()); + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(1)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + + // A PUBLIC repo (upsert_mirror_repo sets is_public=true) carrying the blob, + // with a path-scoped rule restricting src/** to a reader that is NOT the anon + // caller. Rules applied => the blob is denied; rules skipped (fall-through) => + // the public repo serves it (the exposure this guard forbids). + let (repo_id, oid) = + seed_repo_with_blob(&state, tmp.path(), "z6f3failclosed", "gated", b"private\n").await; + state + .db + .set_visibility_rule( + &repo_id, + "src/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkU3IpfsReaderDDDDDDDDDDDDDDDDDDDDDDDD".to_string()], + "z6f3failclosed", + ) + .await + .unwrap(); + + let mut cfg = (*state.config).clone(); + cfg.ipfs_request_budget_secs = 1; + state.config = Arc::new(cfg); + let sem = state.git_ipfs_walk_semaphore.clone(); + let router = ipfs_router(state); + + // Lock `visibility_rules` ACCESS EXCLUSIVE: list_all_repos (on `repos`) still + // succeeds, but list_visibility_rules_for_repos blocks on the rule query. + let mut lock_conn = pool.acquire().await.unwrap(); + sqlx::raw_sql("BEGIN; LOCK TABLE visibility_rules IN ACCESS EXCLUSIVE MODE;") + .execute(&mut *lock_conn) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.81:5000".parse().unwrap(); + let started = std::time::Instant::now(); + let resp = tokio::time::timeout( + std::time::Duration::from_secs(10), + router.oneshot(get_cid(&cid_for_oid(&oid), Some(peer))), + ) + .await + .expect("the budget clamp must return within budget; a bare await hangs on the lock") + .unwrap(); + let elapsed = started.elapsed(); + + let status = resp.status(); + // Fail closed: the handler must NEVER emit the listing with no rules applied. + assert_ne!( + status, + StatusCode::OK, + "a visibility-rule query timeout must DENY, never serve the path-restricted \ + blob from the public repo (that would expose private content)" + ); + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "a visibility-rule query blocked past the budget must shed the retryable budget 503" + ); + assert!( + elapsed < std::time::Duration::from_secs(3), + "the clamp must end the request at ~budget (1s); got {elapsed:?}" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + let body = String::from_utf8_lossy(&body); + assert!( + body.contains("budget"), + "the fail-closed shed must name the budget taint; got: {body}" + ); + assert_eq!( + sem.available_permits(), + 1, + "the walk permit must be freed on the fail-closed budget-shed path" + ); + + sqlx::raw_sql("ROLLBACK") + .execute(&mut *lock_conn) + .await + .unwrap(); + drop(lock_conn); + } } diff --git a/crates/gitlawb-node/tests/inv22_gates.rs b/crates/gitlawb-node/tests/inv22_gates.rs index d21ec194..756806c0 100644 --- a/crates/gitlawb-node/tests/inv22_gates.rs +++ b/crates/gitlawb-node/tests/inv22_gates.rs @@ -200,3 +200,44 @@ fn f4_release_keeps_conn_owned_until_unlock_resolves() { unlock await resolves." ); } + +/// F6/KTD-5 (initial IPFS metadata queries deadline-wrapped): `get_by_cid` acquires +/// the scarce walk permits (RAII, held for the whole request) BEFORE its two initial +/// metadata queries, and the per-repo loop's first budget gate runs only later. So +/// both `list_all_repos` and `list_visibility_rules_for_repos` must be clamped to the +/// remaining request budget — otherwise a query blocked in Postgres pins the walk slot +/// for the whole stall, past the budget. This scans the PRODUCTION half of `api/ipfs.rs` +/// (the `mod tests` half names the same calls in its own harness and would make the +/// check vacuous) and asserts each query call is immediately preceded by a +/// `tokio::time::timeout(` wrapper. Removing either wrapper turns this red (proven +/// load-bearing). +#[test] +fn f6_ipfs_metadata_queries_are_deadline_wrapped() { + let ipfs = src("api/ipfs.rs"); + let production = ipfs + .split("#[cfg(test)]") + .next() + .expect("split always yields a first chunk"); + + for call in [".list_all_repos()", ".list_visibility_rules_for_repos("] { + assert_eq!( + production.matches(call).count(), + 1, + "F6 guard stale: `{call}` must appear exactly once in the production half \ + of api/ipfs.rs (the deadline-wrapped handler call) — update this guard" + ); + let idx = production + .find(call) + .unwrap_or_else(|| panic!("F6 gate: api/ipfs.rs no longer calls `{call}`")); + // The wrapper opens a few lines above the call (match tokio::time::timeout( ... + // remaining budget ..., )); a 240-char lookback covers it without + // reaching the previous statement. + let window = &production[idx.saturating_sub(240)..idx]; + assert!( + window.contains("tokio::time::timeout("), + "F6 gate missing: `{call}` must be wrapped in tokio::time::timeout(...) \ + clamped to the remaining request budget. An unwrapped await pins the held \ + walk permit for the whole DB stall, past GITLAWB_IPFS_REQUEST_BUDGET_SECS." + ); + } +} From a2be8bab136396c382d8810d77355290a49c16bb Mon Sep 17 00:00:00 2001 From: t Date: Mon, 20 Jul 2026 10:19:50 -0500 Subject: [PATCH 54/58] fix(node): bound post-receive Pinata replication memory (#174 F2) The detached Pinata task moved each push's full object-id list into the closure and parked it on pin_semaphore.acquire_owned(); a tokio semaphore bounds active holders, not the waiter queue, so a slow Pinata backend let N queued pushes each retain an MB-scale OID list (unbounded memory/tasks). Capture only the small ref-update tuples and re-derive the object set via the existing reaped/deadline-bounded git helpers AFTER the pin permit is acquired, so retained memory is O(ref tuples), not O(object lists). Per-ref effects are unchanged (still one spawn per push, no coalescing or shedding, so no dropped announcements). Adds an inv22 gate row forbidding a retained object list across the acquire, plus re-derivation-equivalence and reaping tests. --- crates/gitlawb-node/src/api/repos.rs | 276 +++++++++++++++++++++-- crates/gitlawb-node/tests/inv22_gates.rs | 65 ++++++ 2 files changed, 327 insertions(+), 14 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 6e6d5f3a..0f12e720 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -988,6 +988,96 @@ async fn resolve_drain_object_list( Some((object_list, rules_opt, record.is_public)) } +/// Re-derive the Pinata replication object set for a push from its ref-update +/// tuples (#174 F2 / KTD-3). +/// +/// The detached Pinata task used to MOVE the push's full pre-resolved object list +/// into its closure and hold it across the `pin_semaphore` await. Under a slow +/// Pinata backend every later push then parked a fresh task each still retaining +/// an MB-scale OID list, so outstanding memory grew O(pushes x object-list) — +/// unbounded. The task now captures only the small `(ref, old, new)` tuples and +/// calls this once a pin slot frees, re-deriving the SAME OID set via +/// `git rev-list` (the delta scan) filtered by the current withheld set. Retained +/// memory is O(ref tuples); the object list is materialized only inside the +/// pin-bounded section, so at most `pin_semaphore` permits' worth exist at once. +/// +/// Coalescing / shedding were rejected because the task's per-ref work is +/// non-idempotent (branch->CID upsert, gossip, GraphQL broadcast, Arweave anchor, +/// peer-notify); dropping a later push's task drops its announcements. Only the +/// retained object list is dropped here, not the task, so every push's effects +/// still fire exactly once. +/// +/// Fresh by design, exactly like `resolve_drain_object_list`: the withheld set +/// and candidate set are recomputed from the current rules, so a rule tightened +/// since the push is honored and fails closed (a newly-withheld blob is not +/// pinned; a no-longer-announceable repo pins nothing). Every git child runs +/// through the same INV-22 bounded, process-group-reaped helpers the sibling +/// post-receive scans use (`replication_withheld_set`, +/// `resolve_candidates_for_push`, `fail_closed_full_scan_objects`). +#[allow(clippy::too_many_arguments)] +async fn pinata_object_list_for_refs( + encrypt_sem: Arc, + disk_path: std::path::PathBuf, + ref_updates: &[(String, String, String)], + rules_opt: Option>, + is_public: bool, + owner_did: String, + git_bin: String, + timeout: std::time::Duration, +) -> Vec { + let (_announce, withheld) = replication_withheld_set( + encrypt_sem.clone(), + rules_opt.clone(), + &owner_did, + is_public, + disk_path.clone(), + git_bin.clone(), + timeout, + ) + .await; + // Not announceable, or the withheld walk failed: replicate nothing (fail + // closed) — mirrors the receive-pack tail's `withheld == None` handling. + let withheld_set = match withheld { + Some(w) => w, + None => return Vec::new(), + }; + let new_tips: Vec = ref_updates + .iter() + .map(|(_, _, new)| new.clone()) + .filter(|s| s != ZERO_SHA) + .collect(); + let old_tips: Vec = ref_updates + .iter() + .map(|(_, old, _)| old.clone()) + .filter(|s| s != ZERO_SHA) + .collect(); + let pin_set = crate::git::push_delta::resolve_candidates_for_push( + encrypt_sem.clone(), + disk_path.clone(), + new_tips, + old_tips, + git_bin.clone(), + timeout, + false, + ) + .await; + if pin_set.full_scan { + fail_closed_full_scan_objects( + encrypt_sem, + disk_path, + rules_opt.unwrap_or_default(), + is_public, + owner_did, + pin_set.candidates, + git_bin, + timeout, + ) + .await + } else { + crate::git::visibility_pack::replicable_objects(pin_set.candidates, &withheld_set) + } +} + /// The pin/encrypt pipeline shared by the snapshot iteration and the /// coalesced-drain iterations: local IPFS pin, then (path-scoped rules only) the /// admission-gated recipients walk → encrypt-then-pin → Arweave manifest anchor. @@ -1825,16 +1915,20 @@ pub async fn git_receive_pack( // Pin new git objects to Pinata, then record branch→CID and gossip. // // #174 P2-2 scope note: this SECOND detached spawn is deliberately NOT brought - // under the per-repo encryption coalescing above. Two reasons: (1) it does not - // park on `git_encrypt_semaphore` (or any semaphore) — the Pinata `pin_new_objects` - // is a bounded reqwest round-trip, so it does not form the unbounded PARKED-waiter - // set that is the P2-2 residual; it runs to completion under the HTTP client's - // network timeouts. (2) Unlike the idempotent recovery-copy walk, this task does - // PER-PUSH, PER-REF work — branch→CID upserts, gossip publish, GraphQL subscription - // broadcast, Arweave anchoring, and peer notify, each keyed to THIS push's - // ref_updates. Coalescing it against an in-flight task for the same repo would DROP - // a later push's ref-update announcements (a correctness regression), not merely - // delay a duplicate. So it is scoped out with rationale, not brought under the bound. + // under the per-repo encryption coalescing above, because unlike the idempotent + // recovery-copy walk it does PER-PUSH, PER-REF work — branch→CID upserts, gossip + // publish, GraphQL subscription broadcast, Arweave anchoring, and peer notify, each + // keyed to THIS push's ref_updates. Coalescing (or shedding) it against an in-flight + // task for the same repo would DROP a later push's ref-update announcements (a + // correctness regression), not merely delay a duplicate. So the task stays one per + // push and every push's effects fire exactly once. + // + // #174 F2 / KTD-3: {bounded memory, no dropped effects, no handler latency} are + // jointly unsatisfiable by coalesce/shed/block, so instead of retaining the full + // object list we bound the thing that actually accumulates. The task captures only + // the small ref tuples and RE-DERIVES the object set inside the worker once a pin + // slot frees (see `pinata_object_list_for_refs`); the MB-scale OID list is never + // held by a parked task. { let pinata_jwt = state.config.pinata_jwt.clone(); let pinata_upload_url = state.config.pinata_upload_url.clone(); @@ -1859,11 +1953,22 @@ pub async fn git_receive_pack( let owner_did_for_arweave = record.owner_did.clone(); let self_public_url = state.config.public_url.clone(); let node_keypair = Arc::clone(&state.node_keypair); - let object_list_pinata = object_list; let do_pinata_replication = withheld.is_some(); + // #174 F2 / KTD-3: capture only the small inputs the re-derivation needs; the + // MB-scale object list is NOT moved in. `pinata_object_list_for_refs` recomputes + // it from these once a pin slot frees. rules/owner/is_public drive the fresh + // fail-closed withheld filter; encrypt_sem + git_bin + timeout keep the re-derive + // git children under the same INV-22 bounded, group-reaped scan admission. + let pinata_rules_opt = rules_opt.clone(); + let pinata_owner_did = record.owner_did.clone(); + let pinata_is_public = record.is_public; + let pinata_git_bin = state.git_bin.clone(); + let pinata_git_timeout = + std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let pinata_encrypt_sem = state.git_encrypt_semaphore.clone(); // Same global pin-admission bound as the IPFS loop (#174 F6): the Pinata pin - // loop also holds this push's full object-id list while pinning it, so it must - // share the cap. It DEFERS on a full pool rather than dropping the pin. + // loop holds a re-derived object-id list while pinning it, so it shares the cap. + // It DEFERS on a full pool rather than dropping the pin. let pin_sem_pinata = state.pin_semaphore.clone(); tokio::spawn(async move { let pinned = if do_pinata_replication { @@ -1871,12 +1976,28 @@ pub async fn git_receive_pack( .acquire_owned() .await .expect("pin_semaphore is never closed"); + // Re-derive the object set now that a pin slot is free (#174 F2 / + // KTD-3). A parked task retained only `ref_updates_clone` (O(ref + // tuples)), never this list, so a slow Pinata backend cannot grow + // outstanding memory O(pushes x object-list). Fresh + fail-closed; + // each git child is INV-22 bounded and process-group reaped. + let object_list = pinata_object_list_for_refs( + pinata_encrypt_sem, + repo_path_clone.clone(), + &ref_updates_clone, + pinata_rules_opt, + pinata_is_public, + pinata_owner_did, + pinata_git_bin, + pinata_git_timeout, + ) + .await; crate::pinata::pin_new_objects( &http_client, &pinata_upload_url, &pinata_jwt, &repo_path_clone, - object_list_pinata, + object_list, &db_clone, ) .await @@ -6127,6 +6248,133 @@ mod tests { ); } + /// #174 F2 / KTD-3 re-derivation equivalence: the object set the Pinata worker + /// re-derives from ONLY the ref tuples (`pinata_object_list_for_refs`, run once a + /// pin slot frees) must equal exactly what the old retained `object_list` would + /// have pinned — the inline-resolved delta, filtered by the withheld set. If the + /// two differ, the memory fix changed what gets pinned; they must not. + #[tokio::test] + async fn f2_pinata_rederivation_equals_retained_object_list() { + let tmp = tempfile::TempDir::new().unwrap(); + u5_init_repo(tmp.path()); + let c1 = u5_commit_file(tmp.path(), "a.txt", "one\n"); + let c2 = u5_commit_file(tmp.path(), "b.txt", "two\n"); + let sem = std::sync::Arc::new(tokio::sync::Semaphore::new(4)); + let timeout = std::time::Duration::from_secs(600); + + // What the OLD retained-list task WOULD have pinned: the inline pipeline the + // receive-pack tail ran before moving `object_list` into the closure — the + // delta for main c1 -> c2, filtered by the (empty) withheld set. + let candidates = crate::git::push_delta::resolve_candidates_for_push( + sem.clone(), + tmp.path().to_path_buf(), + vec![c2.clone()], + vec![c1.clone()], + "git".to_string(), + timeout, + false, + ) + .await; + assert!( + !candidates.full_scan, + "the c1 -> c2 push is a delta, not a full scan" + ); + let retained: std::collections::HashSet = + crate::git::visibility_pack::replicable_objects( + candidates.candidates, + &std::collections::HashSet::new(), + ) + .into_iter() + .collect(); + + // What the worker re-derives from only the (ref, old, new) tuples. Empty rules + // + is_public => announceable, withheld = {} (the common Pinata case). + let ref_updates = vec![("refs/heads/main".to_string(), c1.clone(), c2.clone())]; + let rederived: std::collections::HashSet = pinata_object_list_for_refs( + sem.clone(), + tmp.path().to_path_buf(), + &ref_updates, + Some(Vec::new()), + true, + "z6MkPinataOwnerAAAAAAAAAAAAAAAAAAAAAAAAAAAA".to_string(), + "git".to_string(), + timeout, + ) + .await + .into_iter() + .collect(); + + let new_blob = u5_git(tmp.path(), &["rev-parse", "HEAD:b.txt"]); + assert!( + retained.contains(&c2) && retained.contains(&new_blob), + "the push introduced the new commit and blob" + ); + assert_eq!( + rederived, retained, + "the worker's git rev-list re-derivation must yield exactly the object set \ + the retained list would have pinned — the memory fix must not change what pins" + ); + } + + /// #174 F2 / KTD-3 reaped + deadline-bounded: the worker's re-derivation git children + /// run through the same INV-22 bounded, process-group-reaped helpers the sibling scans + /// use. On a git that hangs on both `rev-list` and `--batch-all-objects`, + /// `pinata_object_list_for_refs` must RETURN within the watchdog budget (the group is + /// SIGKILLed + reaped at the deadline), not block. A bare `Command::output()` here + /// would hang past the ceiling (RED). + #[cfg(unix)] + #[tokio::test] + async fn f2_pinata_rederivation_is_deadline_bounded_and_reaped() { + use std::os::unix::fs::PermissionsExt; + + let dir = tempfile::TempDir::new().unwrap(); + // Empty rules => replication_withheld_set short-circuits (no git). The tip + // peel (`cat-file -t`) reports a commit so the delta stage proceeds; rev-list + // and the full-scan `cat-file --batch-all-objects` both hang (bounded 30s so a + // broken test cannot leak a permanent orphan). + let fake = dir.path().join("fakegit"); + std::fs::write( + &fake, + "#!/bin/sh\ncase \"$1\" in\n \ + cat-file) case \"$*\" in *--batch-all-objects*) i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done ;; *) echo commit ;; esac ;;\n \ + rev-list) i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done ;;\n \ + *) : ;;\nesac\nexit 0\n", + ) + .unwrap(); + let mut perm = std::fs::metadata(&fake).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&fake, perm).unwrap(); + let git_bin = fake.to_str().unwrap().to_string(); + + let ref_updates = vec![( + "refs/heads/main".to_string(), + ZERO_SHA.to_string(), + "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef".to_string(), + )]; + let got = tokio::time::timeout( + std::time::Duration::from_secs(10), + pinata_object_list_for_refs( + std::sync::Arc::new(tokio::sync::Semaphore::new(4)), + dir.path().to_path_buf(), + &ref_updates, + Some(Vec::new()), + true, + "z6MkPinataOwnerAAAAAAAAAAAAAAAAAAAAAAAAAAAA".to_string(), + git_bin, + std::time::Duration::from_millis(400), + ), + ) + .await + .expect( + "pinata_object_list_for_refs must return within the watchdog budget — a hang \ + means the re-derivation git is not deadline-bounded / group-reaped (RED)", + ); + assert!( + got.is_empty(), + "a hung git yields nothing this push (the reconciliation sweep backstops)" + ); + } + /// Hot-repo drain at encrypt-pool size 1: the task loop holds NO task-level /// permit, so per-iteration helper acquires (withheld walk, candidate scan, /// recipients walk) each get the pool's single permit in turn and BOTH the diff --git a/crates/gitlawb-node/tests/inv22_gates.rs b/crates/gitlawb-node/tests/inv22_gates.rs index 756806c0..2962dc4b 100644 --- a/crates/gitlawb-node/tests/inv22_gates.rs +++ b/crates/gitlawb-node/tests/inv22_gates.rs @@ -241,3 +241,68 @@ fn f6_ipfs_metadata_queries_are_deadline_wrapped() { ); } } + +/// #174 F2 / KTD-3: the detached post-receive Pinata replication task must enqueue +/// only the push's `(ref, old, new)` tuples and RE-DERIVE its object set inside the +/// worker once a pin slot frees — it must NOT move the full per-push object list into +/// the closure and hold it across the `pin_semaphore` acquire. Retaining the list makes +/// every parked task (under a slow Pinata backend) hold an MB-scale OID list, so +/// outstanding memory grows O(pushes x object-list). Coalescing/shedding the task is +/// forbidden (its per-ref effects are non-idempotent), so the fix bounds the retained +/// data, not the task count. +/// +/// Two load-bearing checks, both against the PRODUCTION half of `api/repos.rs` (the +/// `mod tests` half names the same identifiers in its own harness and would make the +/// scan vacuous): (a) the closure-local `object_list_pinata` binding — the retain form — +/// must be gone; reintroducing `let object_list_pinata = object_list;` turns this red. +/// (b) the re-derivation (`pinata_object_list_for_refs`) must appear AFTER the Pinata +/// pin permit is acquired, so the object list is materialized only inside the pin-bounded +/// section and a parked task holds O(ref tuples). +#[test] +fn f2_pinata_enqueues_refs_not_retained_object_lists() { + let repos = src("api/repos.rs"); + let production = repos + .split("#[cfg(test)]") + .next() + .expect("split always yields a first chunk"); + + // (a) the retained-list form must be gone from production. + assert!( + !production.contains("object_list_pinata"), + "F2/KTD-3 regression: the Pinata task retains a full per-push object list \ + (`object_list_pinata`) across the pin-permit acquire. Enqueue only the ref \ + tuples and re-derive the object set inside the worker (pinata_object_list_for_refs)." + ); + + // (b) the re-derivation runs AFTER the pin permit is acquired. Anchor on the + // Pinata pin-admission clone so the window is the Pinata task, not the sibling + // IPFS/encrypt spawn (which shares `pin_semaphore` but never re-derives). + let anchor = production + .find("let pin_sem_pinata") + .expect("F2 gate stale: the Pinata pin-admission clone (pin_sem_pinata) moved"); + let tail = &production[anchor..]; + let acquire = tail + .find(".acquire_owned()") + .expect("F2 gate: the Pinata task no longer acquires a pin permit (acquire_owned)"); + let rederive = tail.find("pinata_object_list_for_refs(").expect( + "F2 gate missing: the Pinata task must re-derive its object set via \ + pinata_object_list_for_refs; it can no longer move a pre-resolved list into the closure", + ); + assert!( + acquire < rederive, + "F2 gate bypassed: the Pinata object-set re-derivation must run AFTER the pin \ + permit is acquired, so a parked task never holds the MB-scale object list" + ); + + // The re-derivation is driven by the push's ref tuples (the enqueued unit), not a + // retained object list — tie "enqueue ref_updates" to the call explicitly. + assert!( + tail[rederive..].starts_with("pinata_object_list_for_refs(") + && tail[rederive..] + .get(..400) + .map(|w| w.contains("ref_updates_clone")) + .unwrap_or(false), + "F2 gate: pinata_object_list_for_refs must re-derive from the push's ref tuples \ + (ref_updates_clone), the small unit the parked task retains" + ); +} From a333ddc1fc7b68830ad6d2d7a55967f64256b0b0 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 20 Jul 2026 10:58:37 -0500 Subject: [PATCH 55/58] fix(node): serialize a disconnected push's repo until its group is reaped (#174 F3) On client disconnect, RepoWriteGuard::Drop released the per-repo advisory lock immediately (it holds no child/pgid), while KillGroupOnDrop's detached reaper was still tearing down the git process group for ~4s. A second same-node push could then acquire the repo and race the first's still-writing objects/ dir and Tigris upload. Add an in-process per-repo write lease (block-and-wait, keyed like the advisory lock) that SUPPLEMENTS the retained cluster-wide PG lock. The lease rides into the disconnect reaper via a clone on the write-path AdmissionGuard (None on all read paths) and a second clone held across guard.release() for the clean-path Tigris upload, so it frees only after the group is reaped. A bounded-wait steal reclaims a leaked lease (block-and-wait has no coalesce degradation). The lease is always taken before the PG lock and released after, so the two serializers cannot invert. Deterministic fake-git race test + inv22 gate row. --- crates/gitlawb-node/src/api/repos.rs | 262 ++++++++++++++++++++- crates/gitlawb-node/src/auth/mod.rs | 1 + crates/gitlawb-node/src/git/smart_http.rs | 20 ++ crates/gitlawb-node/src/main.rs | 5 + crates/gitlawb-node/src/state.rs | 274 ++++++++++++++++++++++ crates/gitlawb-node/src/test_support.rs | 1 + crates/gitlawb-node/tests/inv22_gates.rs | 50 ++++ 7 files changed, 612 insertions(+), 1 deletion(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 0f12e720..9774148e 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1626,6 +1626,25 @@ pub async fn git_receive_pack( } } + // Per-repo in-process write lease (#174 U2/F3): SUPPLEMENTS the cluster-wide pg + // advisory lock. Acquire it BEFORE acquire_write (one consistent order everywhere, + // so the two serializers can never invert into a self-hang) so a second SAME-NODE + // push to this repo blocks here rather than racing a disconnected first push's git + // group while its detached reaper is still tearing it down over the shared local + // objects/ dir. Taking it before the pg lock also means a blocked second writer pins + // no pooled pg connection while it waits. The lease rides the write-path + // AdmissionGuard into the reaper (clone (a)) and spans the clean-path Tigris upload + // in guard.release (clone (b)); it frees only when the LAST clone drops. steal_after + // is set well above any legitimate hold (a full receive-pack under + // git_service_timeout + the ~4s reaper cap + the Tigris upload), so the bounded-wait + // reclaim only ever fires for a genuinely leaked lease, never a merely-slow push. + let lease_steal_after = + std::time::Duration::from_secs(state.config.git_service_timeout_secs * 2 + 60); + let lease = state + .repo_write_leases + .acquire(&record.id, lease_steal_after) + .await; + tracing::debug!(repo = %name, "acquiring write lock"); // Bound the write acquire under `git_acquire_timeout_secs`. acquire_write's // advisory-lock loop already caps at ~60s, but its per-iteration @@ -1661,7 +1680,13 @@ pub async fn git_receive_pack( // instant a disconnect drops this future while the detached reaper runs (#174 P1-a). // The handler keeps no copy. This is independent of the write-lock `guard.release` // below: admission tracks the git process lifetime, the write lock tracks the repo. - let admission = smart_http::AdmissionGuard::new(_permit, _caller_permit); + // Clone (a) of the write lease rides this AdmissionGuard: on a client disconnect the + // guard moves into KillGroupOnDrop's detached reaper, so the lease frees only after + // the receive-pack group is reaped — NOT at the disconnect instant (which is exactly + // when RepoWriteGuard::Drop frees the pg lock). Tying the lease to RepoWriteGuard + // instead would drop it at disconnect and reopen the F3 race. + let admission = + smart_http::AdmissionGuard::new(_permit, _caller_permit).with_lease(lease.clone()); let receive_result = smart_http::receive_pack( &state.git_bin, &disk_path, @@ -1675,6 +1700,12 @@ pub async fn git_receive_pack( // from blocking subsequent pushes. Only upload to Tigris when the push // succeeded; uploading a half-applied repo would propagate corruption. guard.release(receive_result.is_ok()).await; + // Clean path: clone (a) already dropped inside run_git_service when the receive-pack + // group was reaped; clone (b) held here spanned the success-only Tigris upload that + // ran inside release() above. Drop it now so a second same-repo push proceeds the + // moment this write is durable, rather than at end of the (longer) handler tail. On + // the disconnect path this line is never reached — clone (a) rides the reaper (F3). + drop(lease); let result = receive_result.map_err(|e| { let app = git_service_app_error(&e); @@ -6665,4 +6696,233 @@ mod tests { "repo creation must be IP-throttled before signature verification" ); } + + // ── #174 U2 / F3: second same-repo push serialized until a disconnected first ── + // push's git process GROUP is reaped (RepoWriteLease riding the disconnect reaper). + + /// `kill(pid, 0)` liveness probe (same-uid here, so EPERM never applies). + #[cfg(unix)] + fn f3_alive(pid: i32) -> bool { + unsafe { libc::kill(pid, 0) == 0 } + } + + /// F3 (P1, RED-before/GREEN-after): on a client disconnect DURING receive-pack the + /// disconnected push's git group is torn down by KillGroupOnDrop's detached reaper + /// (~4s TERM/grace/KILL/reap), while RepoWriteGuard::Drop releases the pg advisory + /// lock at the disconnect INSTANT. Without the in-process write lease, a second + /// same-node push then acquires the freed pg lock and mutates the shared local repo + /// WHILE the first group is still writing — a torn snapshot. The lease is held by the + /// write-path AdmissionGuard, which rides that reaper, so the second push must not run + /// its receive-pack (mutate the repo) until the first group is reaped. + /// + /// The fake git labels the pushes by receive-pack arrival order (atomic mkdir): the + /// first (push A) forks a SIGTERM-IGNORING descendant, records its pid, then hangs + /// (so A can be dropped mid-transfer and its group survives the SIGTERM grace); the + /// second (push B, a DIFFERENT source) records that its receive-pack ran — i.e. that + /// B mutated the repo. The load-bearing invariant is strictly ordered, not + /// time-windowed: B's marker must NEVER appear while A's descendant is still alive. + /// + /// Load-bearing: pre-fix (no lease) A's disconnect frees the pg lock, B's + /// acquire_write succeeds within its ~1s retry, and B's receive-pack runs (marker + /// appears) WHILE A's descendant is still alive — RED. With the lease the reaper + /// holds it until the group is ESRCH-gone, so B's marker appears only AFTER — GREEN. + #[cfg(unix)] + #[sqlx::test] + async fn f3_second_push_serialized_until_disconnected_group_reaped(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + + let tmp = tempfile::TempDir::new().unwrap(); + let seq_a = tmp.path().join("seq_a"); // first receive-pack wins this mkdir = push A + let descfile = tmp.path().join("desc.pid"); // A's SIGTERM-ignoring descendant pid + let b_ran = tmp.path().join("b.ran"); // set when B's receive-pack runs (B mutates) + // receive-pack: first invocation (A) forks a TERM-ignoring descendant (bounded + // loop so a RED run leaks no permanent orphan), records its pid, and hangs in + // `wait`; second (B) records that it ran. rev-parse feeds any tail probe. + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + receive-pack)\n\ + cat >/dev/null 2>/dev/null\n\ + if mkdir \"{seq}\" 2>/dev/null; then\n\ + sh -c 'trap \"\" TERM; echo $$ > \"{desc}\"; i=0; while [ $i -lt 60 ]; do sleep 0.1; i=$((i+1)); done' &\n\ + wait\n\ + else\n\ + echo 1 > \"{bran}\"\n\ + fi ;;\n\ + rev-parse) echo deadbeef ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + seq = seq_a.display(), + desc = descfile.display(), + bran = b_ran.display(), + ); + let git_bin = write_fake_git(tmp.path(), &body); + // One repo; A and B push to it (same record.id -> same lease key). Non-path-scoped + // + flush-only body -> no post-receive scans to muddy the observation. + let state = + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6f3repo", "r1", false).await; + let did = "did:key:z6MkF3PusherAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + + // Push A: drive its handler future in slices until it reaches receive-pack and the + // fake records its descendant pid (A now holds the lease and is hung). + let mut fut_a = Box::pin(git_receive_pack( + State(state.clone()), + Path(("z6f3repo".to_string(), "r1".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some("203.0.113.81:5000".parse::().unwrap())), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + )); + let mut desc: Option = None; + for _ in 0..1000 { + let _ = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut_a).await; + if let Some(p) = std::fs::read_to_string(&descfile) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + desc = Some(p); + break; + } + } + let desc = desc.expect("push A must reach receive-pack and record its descendant pid"); + assert!( + f3_alive(desc), + "A's descendant must be alive before the disconnect" + ); + + // Push B: a DIFFERENT source, same repo. It blocks on the lease A holds. + let state_b = state.clone(); + let handle_b = tokio::spawn(async move { + git_receive_pack( + State(state_b), + Path(("z6f3repo".to_string(), "r1".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some( + "203.0.113.82:5000".parse::().unwrap(), + )), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await + }); + // Give B time to reach and block on the lease acquire. + tokio::time::sleep(std::time::Duration::from_millis(200)).await; + assert!( + !b_ran.exists(), + "B must not have mutated the repo while A legitimately holds the lease" + ); + + // Client disconnect on A: drop its future. RepoWriteGuard::Drop frees the pg lock + // immediately; the write-path AdmissionGuard (carrying the lease's clone (a)) + // rides KillGroupOnDrop's detached reaper, which now tears down A's group. + drop(fut_a); + + // Load-bearing ordering invariant: while A's descendant is still alive (group not + // yet reaped), B must NOT have run its receive-pack. Poll until the descendant is + // gone; every step it is alive, B's marker must be absent. Pre-fix, B's marker + // appears here (RED); with the lease it can only appear after the reap (GREEN). + let mut reaped = false; + for _ in 0..800 { + if !f3_alive(desc) { + reaped = true; + break; + } + assert!( + !b_ran.exists(), + "F3 RED: push B mutated the repo while push A's disconnected git group \ + was still alive (descendant pid {desc}) — the second writer must be \ + serialized until the first group is reaped" + ); + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + // Safety net so a RED run never leaks the orphan. + unsafe { + libc::kill(desc, libc::SIGKILL); + } + assert!( + reaped, + "A's disconnected group must be reaped within the teardown cap" + ); + + // GREEN tail: once the group is reaped the lease frees and B proceeds — its + // receive-pack runs (marker appears) and it returns 200. + let mut b_mutated = false; + for _ in 0..1000 { + if b_ran.exists() { + b_mutated = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + assert!( + b_mutated, + "push B must proceed and mutate the repo once A's group is reaped" + ); + let resp = tokio::time::timeout(std::time::Duration::from_secs(30), handle_b) + .await + .expect("push B must complete once the lease frees") + .expect("push B task must not panic") + .expect("push B must succeed"); + assert_eq!(resp.status(), 200, "push B lands 200 after serialization"); + } + + /// F3 clean-path no-regression: a clean push (no disconnect) releases the lease after + /// the receive-pack group is reaped and the (success-only) Tigris upload in + /// guard.release runs, so the per-repo lease entry is GC'd and a second same-repo + /// push proceeds immediately. A lease that failed to free on the clean path would + /// wedge every subsequent push to the repo. + #[cfg(unix)] + #[sqlx::test] + async fn f3_clean_push_frees_lease_and_second_push_proceeds(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + + let tmp = tempfile::TempDir::new().unwrap(); + // Clean receive-pack: drain stdin, exit 0. No hang, no descendant. + let body = "#!/bin/sh\ncase \"$1\" in\n receive-pack) cat >/dev/null 2>/dev/null ;;\n rev-parse) echo deadbeef ;;\n *) : ;;\nesac\nexit 0\n"; + let git_bin = write_fake_git(tmp.path(), body); + let state = + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6f3clean", "c1", false).await; + let did = "did:key:z6MkF3CleanPusherAAAAAAAAAAAAAAAAAAAAAAAA"; + + let push = |st: AppState, peer: &'static str| async move { + tokio::time::timeout( + std::time::Duration::from_secs(30), + git_receive_pack( + State(st), + Path(("z6f3clean".to_string(), "c1".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(peer.parse::().unwrap())), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ), + ) + .await + .expect("a clean push must not wedge on the lease") + .expect("the push must succeed") + }; + + let a = push(state.clone(), "203.0.113.91:5000").await; + assert_eq!(a.status(), 200, "clean push A lands 200"); + // The clean push freed its lease (both clones dropped) -> the entry GC'd. + assert!( + state.repo_write_leases.is_empty(), + "a clean push must free the per-repo lease (Drop-frees-key) so it never wedges" + ); + + let b = push(state.clone(), "203.0.113.92:5000").await; + assert_eq!( + b.status(), + 200, + "a second same-repo push proceeds after a clean first" + ); + assert!( + state.repo_write_leases.is_empty(), + "the lease entry must be freed again after the second clean push" + ); + } } diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index c1fa6624..431ac983 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -526,6 +526,7 @@ mod tests { git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), pin_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), encrypt_inflight: crate::state::EncryptInflight::new(), + repo_write_leases: crate::state::RepoWriteLeases::new(), git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), diff --git a/crates/gitlawb-node/src/git/smart_http.rs b/crates/gitlawb-node/src/git/smart_http.rs index f8b46ce8..9ac742ec 100644 --- a/crates/gitlawb-node/src/git/smart_http.rs +++ b/crates/gitlawb-node/src/git/smart_http.rs @@ -33,17 +33,37 @@ pub struct AdmissionGuard { // 'static` so the guard can move into the detached reaper task. _global: Option>, _caller: Option>, + // Per-repo write lease (#174 U2/F3), `Some` ONLY on the receive-pack write path + // (via [`with_lease`](Self::with_lease)); `None` on every read path and every + // non-receive-pack write path. It rides this guard into `KillGroupOnDrop`'s detached + // reaper, so on a client disconnect the lease frees only after the disconnected + // push's git group is reaped — serializing a second same-node push against the + // still-writing group. A `RepoWriteLease` clone is `Send + 'static` and holds no pg + // connection, so it travels with the reaper cleanly. A stray `Some` on a READ path + // would wrongly serialize upload-pack against pushes, hence the None-everywhere-else + // discipline. + _lease: Option, } impl AdmissionGuard { /// Take ownership of the global permit and an optional per-caller permit. Both are /// erased to `Box` — the guard's only job is to hold them until it drops. + /// No lease is attached here; the receive-pack write path adds one via + /// [`with_lease`](Self::with_lease), so every other call site is lease-free. pub fn new(global: impl Send + 'static, caller: Option) -> Self { Self { _global: Some(Box::new(global)), _caller: caller.map(|c| Box::new(c) as Box), + _lease: None, } } + + /// Attach the per-repo write lease (#174 U2/F3). Called ONLY on the receive-pack + /// write path, so the lease rides the disconnect reaper; read paths never call this. + pub fn with_lease(mut self, lease: crate::state::RepoWriteLease) -> Self { + self._lease = Some(lease); + self + } } /// Handle `GET /:owner/:repo/info/refs?service=git-upload-pack` diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 4c4c1da8..d6c0df46 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -410,6 +410,11 @@ async fn main() -> Result<()> { // P2-2). No knob: it is a natural cap (one entry per distinct repo), not a // sized pool. encrypt_inflight: crate::state::EncryptInflight::new(), + // Per-repo in-process write-lease serializer (#174 U2/F3): supplements the pg + // advisory lock so a disconnected push's still-reaping git group can't be raced + // by a second same-node push. Natural cap (one entry per contended repo, freed + // when unreferenced), no sized knob. + repo_write_leases: crate::state::RepoWriteLeases::new(), git_read_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( config.max_concurrent_reads_per_caller, ), diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 77799bd0..213b8b35 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -147,6 +147,20 @@ pub struct AppState { /// recovery copy, which `2a54c15` deliberately kept fail-closed (there is no /// reconciliation sweep to re-derive a dropped copy). See [`EncryptInflight`]. pub encrypt_inflight: EncryptInflight, + /// Per-repo in-process write serializer that SUPPLEMENTS the cluster-wide pg + /// advisory lock on the receive-pack path (#174 U2/F3). On a client disconnect + /// mid-`receive-pack`, `RepoWriteGuard::Drop` releases the pg advisory lock at the + /// disconnect instant, but the disconnected push's git process GROUP is still + /// being torn down by `KillGroupOnDrop`'s detached reaper (~4s TERM/grace/KILL/reap) + /// over the shared LOCAL objects/ dir — so a second SAME-NODE push could acquire + /// the repo and race the still-writing group into a torn snapshot. This lease is + /// held by the write-path `AdmissionGuard`, which rides that reaper, so a second + /// same-repo push blocks until the first group is reaped. It is per-NODE (the + /// corruption is same-node: shared local objects/ + in-process reaper, and the + /// disconnect path uploads nothing to Tigris), so it needs no cross-node counterpart + /// and does NOT replace the pg lock (which stays the genuine cluster-wide serializer). + /// See [`RepoWriteLeases`]. + pub repo_write_leases: RepoWriteLeases, /// Per-caller concurrency sub-cap on the read pool: each caller (keyed on the /// resolved source IP, #174 U1) may hold at most `max_concurrent_reads_per_caller` /// in-flight read ops, so one caller cannot monopolize `git_read_semaphore` @@ -453,6 +467,182 @@ impl Drop for EncryptInflightGuard { } } +/// Per-repo in-process write-lease serializer (#174 U2/F3). Keyed by the repo's DB +/// id (1:1 with the pg advisory lock's owner/name key), each entry is a one-permit +/// semaphore: the receive-pack handler takes it BEFORE `acquire_write` (see the acquire +/// order note on [`acquire`](Self::acquire)) and a second same-repo push BLOCKS on it — +/// block-and-wait, NOT coalesce. It mirrors [`EncryptInflight`]'s keyed-map + guard + +/// Drop-frees-key STRUCTURE; the semantics differ (block-and-wait, so there is no +/// lossy-coalesce degradation to fall back on). +#[derive(Clone, Default)] +pub struct RepoWriteLeases { + // std::sync::Mutex: held only for O(1) map ops (get-or-create + refcount) in a sync + // context, never across an await — the semaphore wait happens OUTSIDE this lock. + repos: Arc>>, +} + +/// A per-repo lease entry: the one-permit semaphore plus a refcount of the handlers +/// currently referencing it (holding or waiting). While `refs > 0` every acquirer +/// shares the SAME semaphore, so mutual exclusion holds; the entry is removed only when +/// `refs` hits 0 (no one references it), so a fresh entry can never split serialization. +struct LeaseSlot { + sem: Arc, + refs: usize, +} + +impl RepoWriteLeases { + pub fn new() -> Self { + Self::default() + } + + /// Acquire the per-repo write lease, blocking until it is free (a second same-repo + /// writer waits). `steal_after` bounds that wait: past it the acquirer STEALS + /// (proceeds permit-less) rather than block forever. + /// + /// Why a bounded steal: block-and-wait has no degradation of its own (unlike the + /// coalescing [`EncryptInflight`], whose lost key merely delays a best-effort copy), + /// and unlike the pg advisory lock (60s stale reclaim) an in-process waiter has no + /// reclaim — so a leaked/never-run Drop (runtime teardown without unwind, task abort, + /// `mem::forget`) would otherwise wedge the repo permanently. A stealer takes NO + /// permit and touches NO count, so a merely-slow holder that later drops can never + /// leave the semaphore over-counted; the caller must therefore set `steal_after` + /// safely ABOVE any legitimate hold (a full receive-pack under + /// `git_service_timeout_secs` + the ~4s reaper cap + the Tigris upload). + /// + /// Acquire order (one consistent order everywhere, so no inversion self-hang): the + /// lease is taken BEFORE the pg advisory lock (`acquire_write`) and released AFTER + /// it. Nothing anywhere takes the pg lock before this lease, so the two serializers + /// can never deadlock; taking the lease first also means a blocked second writer + /// pins no pooled pg connection while it waits. + pub async fn acquire(&self, repo_id: &str, steal_after: std::time::Duration) -> RepoWriteLease { + // Take the entry refcount BEFORE the await, so the entry cannot be GC'd out from + // under a waiter (a fresh entry for a new acquirer would split serialization). + let sem = { + let mut map = self.repos.lock().expect("repo_write_leases mutex poisoned"); + let slot = map.entry(repo_id.to_string()).or_insert_with(|| LeaseSlot { + sem: Arc::new(tokio::sync::Semaphore::new(1)), + refs: 0, + }); + slot.refs += 1; + Arc::clone(&slot.sem) + }; + // Cancellation-safe refcount: hold a reservation across the (cancellable) wait so + // that if this acquire future is DROPPED mid-wait — a client disconnect while a + // second same-repo push is blocked here — the reservation's Drop still decrements + // the ref it just took, rather than stranding it (which would defeat the + // Drop-frees-key GC). On success the reservation is `forget`-transferred into the + // returned guard, which then owns the single decrement. + let reservation = RefReservation { + repos: Arc::clone(&self.repos), + repo_id: repo_id.to_string(), + }; + let permit = match tokio::time::timeout(steal_after, Arc::clone(&sem).acquire_owned()).await + { + Ok(Ok(p)) => Some(p), + // The semaphore is never closed; treat the (unreachable) closed case as a + // steal so acquire always makes forward progress. + Ok(Err(_closed)) => None, + Err(_elapsed) => { + tracing::warn!( + repo = %repo_id, + steal_after_secs = steal_after.as_secs(), + "repo write-lease wait exceeded the steal bound; presuming a leaked \ + lease and proceeding permit-less (in-process serializer reclaim)" + ); + None + } + }; + // Transfer the ref from the reservation to the guard: forget the reservation (so + // it does NOT decrement) and let the guard own the single decrement on its Drop. + std::mem::forget(reservation); + RepoWriteLease(Arc::new(LeaseGuardInner { + repos: Arc::clone(&self.repos), + repo_id: repo_id.to_string(), + _permit: permit, + })) + } + + /// Number of repos with a live lease entry. Test/metrics observability. + #[allow(dead_code)] + pub fn len(&self) -> usize { + self.repos + .lock() + .expect("repo_write_leases mutex poisoned") + .len() + } + + #[allow(dead_code)] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +/// Shared-ownership handle to a held repo write lease (#174 U2/F3). `Clone` hands a +/// second holder a handle to the SAME inner guard; the lease (permit + map refcount) +/// frees only when the LAST clone drops. The receive-pack handler makes two: +/// * clone (a) rides the write-path [`AdmissionGuard`] into `KillGroupOnDrop`'s +/// detached reaper, so on a client disconnect it drops only AFTER the git group is +/// reaped (this is the F3 fix — a lease tied to `RepoWriteGuard` would instead drop +/// at the disconnect instant, reopening the race); +/// * clone (b) is held by the handler across `guard.release()`, so on the clean path +/// it spans the success-only Tigris upload that runs inside `release`, AFTER +/// `receive_pack` has already dropped clone (a) inside `run_git_service`. +/// +/// `Send + 'static` with NO pg connection (just an `Arc`), so it can ride the reaper. +#[derive(Clone)] +pub struct RepoWriteLease(#[allow(dead_code)] Arc); + +struct LeaseGuardInner { + repos: Arc>>, + repo_id: String, + // `None` only on the steal path (the bounded wait elapsed). Dropping `None` releases + // no permit, so a stealer never corrupts the semaphore's permit count. + _permit: Option, +} + +impl Drop for LeaseGuardInner { + fn drop(&mut self) { + // Runs exactly ONCE per handler acquisition — when the last `RepoWriteLease` + // clone drops (the Arc strong count hits 0) — so the refcount decrements once, + // however many clones existed. `_permit` drops after this body, releasing the + // semaphore permit so a waiting acquirer proceeds. + release_lease_ref(&self.repos, &self.repo_id); + } +} + +/// Holds the entry refcount across the cancellable wait inside +/// [`RepoWriteLeases::acquire`]. If that acquire future is dropped mid-wait, this Drop +/// decrements the ref it took; on success `acquire` `forget`s it and hands the ref to +/// the returned [`LeaseGuardInner`], so the ref is decremented exactly once either way. +struct RefReservation { + repos: Arc>>, + repo_id: String, +} + +impl Drop for RefReservation { + fn drop(&mut self) { + release_lease_ref(&self.repos, &self.repo_id); + } +} + +/// Decrement a lease entry's refcount and remove it once no handler references it, so +/// the map cannot grow without bound (Drop-frees-key, like `EncryptInflight`). Safe +/// under block-and-wait: while `refs > 0` every acquirer shares the SAME semaphore, and +/// a fresh entry is created only after `refs` hits 0, when no one references the old one. +fn release_lease_ref( + repos: &Arc>>, + repo_id: &str, +) { + if let Ok(mut map) = repos.lock() { + if let Some(slot) = map.get_mut(repo_id) { + slot.refs = slot.refs.saturating_sub(1); + if slot.refs == 0 { + map.remove(repo_id); + } + } + } +} + /// Admit a post-receive git scan to the shared `git_encrypt_semaphore` pool /// (#174 F4): DEFER (await), never shed — a dropped scan would lose the push's /// recovery copy or silently under-pin it. The returned permit must move into @@ -483,3 +673,87 @@ pub async fn acquire_scan_permit( ); permit } + +#[cfg(test)] +mod repo_write_lease_tests { + use super::RepoWriteLeases; + use std::time::Duration; + + /// #174 U2/F3 lease mechanics: block-and-wait serialization on the same repo, + /// no serialization across distinct repos, Drop-frees-key GC, and the bounded-wait + /// steal reclaim so a leaked (never-run Drop) holder cannot wedge the repo forever. + #[tokio::test] + async fn serializes_same_repo_frees_key_and_steals_on_leak() { + let leases = RepoWriteLeases::new(); + let big = Duration::from_secs(3600); + + // Block-and-wait: a second same-repo acquire waits while the first is held. + let a = leases.acquire("repo1", big).await; + let blocked = + tokio::time::timeout(Duration::from_millis(200), leases.acquire("repo1", big)).await; + assert!( + blocked.is_err(), + "a second same-repo acquire must block while the first lease is held" + ); + // ... and proceeds once the first frees. + drop(a); + let b = tokio::time::timeout(Duration::from_millis(500), leases.acquire("repo1", big)) + .await + .expect("the second acquire must proceed once the first lease frees"); + drop(b); + + // Drop-frees-key: with no holders the entry is removed (bounded map growth). + assert!( + leases.is_empty(), + "the lease entry must be removed once no handler references it" + ); + + // Distinct repos never serialize against each other. + let x = leases.acquire("repoX", big).await; + let _y = tokio::time::timeout(Duration::from_millis(200), leases.acquire("repoY", big)) + .await + .expect("distinct repos must not serialize"); + drop(x); + drop(_y); + + // Steal-on-timeout reclaim: a leaked holder (never-run Drop, simulated by + // mem::forget) must NOT wedge the repo — the bounded wait proceeds permit-less. + let leaked = leases.acquire("repoZ", big).await; + std::mem::forget(leaked); + let stolen = tokio::time::timeout( + Duration::from_secs(2), + leases.acquire("repoZ", Duration::from_millis(150)), + ) + .await + .expect("a leaked lease must be reclaimed by the bounded-wait steal, not hang forever"); + drop(stolen); + } + + /// Cancellation safety: dropping an acquire future while it is BLOCKED waiting for + /// the lease (a client disconnect on a second same-repo push) must not strand the + /// entry refcount — after the holder frees and the waiter is cancelled, the key GCs. + #[tokio::test] + async fn cancelled_waiter_does_not_strand_the_refcount() { + let leases = RepoWriteLeases::new(); + let big = Duration::from_secs(3600); + + let a = leases.acquire("repoC", big).await; + // A waiter blocks, then is cancelled (its acquire future dropped) mid-wait. + let cancelled = + tokio::time::timeout(Duration::from_millis(150), leases.acquire("repoC", big)).await; + assert!( + cancelled.is_err(), + "the waiter must be blocked, then cancelled" + ); + + // Release the holder. If the cancelled waiter had stranded its ref, the entry + // would never GC; assert it does once the holder frees. + drop(a); + // Let any pending Drop bookkeeping settle. + tokio::task::yield_now().await; + assert!( + leases.is_empty(), + "a cancelled waiter must not strand the entry refcount (key must GC)" + ); + } +} diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index cd58f19e..b88ae463 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -89,6 +89,7 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), pin_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), encrypt_inflight: crate::state::EncryptInflight::new(), + repo_write_leases: crate::state::RepoWriteLeases::new(), git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys( 8, diff --git a/crates/gitlawb-node/tests/inv22_gates.rs b/crates/gitlawb-node/tests/inv22_gates.rs index 2962dc4b..0ce54088 100644 --- a/crates/gitlawb-node/tests/inv22_gates.rs +++ b/crates/gitlawb-node/tests/inv22_gates.rs @@ -306,3 +306,53 @@ fn f2_pinata_enqueues_refs_not_retained_object_lists() { (ref_updates_clone), the small unit the parked task retains" ); } + +/// #174 U2 / F3 (second same-repo writer serialized until a disconnected first push's +/// git group is reaped): `git_receive_pack` must take the per-repo in-process write +/// lease and CARRY it on the write-path `AdmissionGuard` (via `.with_lease`), so the +/// lease rides `KillGroupOnDrop`'s detached reaper and frees only after the group is +/// reaped — supplementing the pg advisory lock, which `RepoWriteGuard::Drop` releases at +/// the disconnect instant. Tying the lease to `RepoWriteGuard` instead (or dropping the +/// `.with_lease` carry) reopens the race; the behavioral RED/GREEN test +/// (`f3_second_push_serialized_until_disconnected_group_reaped`) is the real bar, this +/// is the completeness tripwire. Scanned against the PRODUCTION half of `api/repos.rs` +/// (the `mod tests` half names these identifiers in its own harness). +#[test] +fn f3_second_writer_leased_until_reap() { + let repos = src("api/repos.rs"); + let smart_http = src("git/smart_http.rs"); + let repos_production = repos + .split("#[cfg(test)]") + .next() + .expect("split always yields a first chunk"); + + // The lease is acquired, then carried by the AdmissionGuard (.with_lease), before + // receive_pack runs the write. Severing any of the three turns this red. + let lease_acquire = repos_production.find("repo_write_leases").expect( + "F3 gate missing: git_receive_pack no longer takes the per-repo write lease \ + (state.repo_write_leases)", + ); + let with_lease = repos_production.find(".with_lease(").expect( + "F3 gate missing: the write-path AdmissionGuard no longer carries the lease \ + (.with_lease). The lease must ride the disconnect reaper via the AdmissionGuard, \ + NOT RepoWriteGuard (which drops at the disconnect instant, reopening F3).", + ); + let receive = repos_production + .find("smart_http::receive_pack(") + .expect("F3 gate stale: git_receive_pack no longer calls smart_http::receive_pack"); + assert!( + lease_acquire < with_lease && with_lease < receive, + "F3 gate bypassed: the write lease must be acquired, then carried by the \ + AdmissionGuard (.with_lease), BEFORE receive_pack runs the write" + ); + + // The AdmissionGuard must actually hold the lease (Option) via a + // with_lease setter, so it travels into the detached reaper. Removing the field or + // method turns this red. + assert!( + smart_http.contains("_lease: Option") + && smart_http.contains("pub fn with_lease("), + "F3 gate missing: AdmissionGuard must hold an Option set via \ + with_lease, so the lease rides the guard into KillGroupOnDrop's reaper" + ); +} From f02c5e1ebd6c365b2d29bdb1890e1398e8011338 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 20 Jul 2026 11:54:09 -0500 Subject: [PATCH 56/58] fix(node): take the write permits after the per-repo lease, not before (#174 F3 review) Code review found a DoS the F3 lease introduced: git_receive_pack acquired the global and per-source write permits before block-waiting on the per-repo lease, so a second same-repo push that parked on the lease pinned a scarce global write-pool slot (1 of 32) for up to steal_after while sending zero bytes. A few hostile sources stacking same-repo pushes could hold every slot on idle lease-waiters and shed 503 on every push to every other repo node-wide. Move both permit acquisitions to after the lease acquire so a lease-blocked waiter pins no write-pool slot, and keep a cheap non-holding availability peek before the DB lookup so a push flood on a saturated pool still sheds 503 without touching Postgres (the authoritative held permit is taken after the lease). Adds a blocked-waiter-holds-no-permit regression test. --- crates/gitlawb-node/src/api/repos.rs | 192 ++++++++++++++++++++++++--- 1 file changed, 171 insertions(+), 21 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 9774148e..836e5625 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1536,27 +1536,16 @@ pub async fn git_receive_pack( body: Bytes, ) -> Result { let name = smart_http_repo_name(&repo)?; - // Per-source write sub-cap (#174 P1-d): before the global write permit so one - // source IP cannot occupy the whole write pool via many slow authenticated pushes - // and 503 every other source. Owner enforcement defaults off, so any valid did:key - // is accepted (auth != authz), and the 600/hour push limiter bounds arrival RATE, - // not in-flight concurrency — so without this a single host minting disposable DIDs - // saturates the pool. Keyed on the resolved source IP, NEVER the signed DID (a DID - // farm defeats a DID key); no resolvable key -> global write pool only. - let caller_key = read_caller_key(&headers, peer, state.push_limiter_trust); - let _caller_permit = acquire_read_caller_permit( - &state.git_write_per_caller, - caller_key.as_deref(), - name, - "receive-pack", - )?; - // Shed with a 503 before spawning git when the concurrency cap is saturated. - // Pushes draw from the dedicated WRITE pool, separate from reads, so a flood of - // anonymous reads cannot shed an authenticated push (#174). Taken after the - // per-source cap above so one source cannot occupy global slots it would be - // sub-cap-denied for; still before the Tigris acquire_write, bounding concurrent - // fresh acquires (INV-10); held for the whole op. - let _permit = git_permit(&state.git_write_semaphore)?; + // Fast-path shed BEFORE the DB lookup when the write pool is saturated, so a push + // flood on a full pool does not hit Postgres per request. Best-effort (racy) and + // NON-holding: the authoritative, held permit is taken after the per-repo lease below + // (so a lease-blocked waiter pins no write slot — #174 F3 review). This peek only + // restores the cheap pre-DB shed the permit reorder would otherwise have lost. + if state.git_write_semaphore.available_permits() == 0 { + return Err(AppError::Overloaded( + "git service at capacity, retry shortly".into(), + )); + } tracing::info!(owner = %owner, repo = %name, "receive-pack request"); let record = state .db @@ -1645,6 +1634,31 @@ pub async fn git_receive_pack( .acquire(&record.id, lease_steal_after) .await; + // Admission permits are taken HERE, AFTER the per-repo lease and BEFORE acquire_write. + // Ordering is the fix (#174 P2 DoS): the lease is a block-and-wait serializer, so a + // second same-repo push can park on `acquire` above for up to steal_after. Taking the + // scarce write permits only once we own the lease means a lease-blocked waiter pins NO + // write-pool slot while it waits. Otherwise a few hostile sources could stack same-repo + // pushes, hold every global slot on zero-byte lease-waiters, and shed 503 on every push + // to every OTHER repo node-wide. Still before acquire_write, so the git op stays + // admission-gated (INV-10) and a saturated pool sheds 503 before spawning git. + // + // Per-source sub-cap first (#174 P1-d): one source IP cannot occupy the whole write + // pool via many slow pushes. Owner enforcement defaults off, so any valid did:key is + // accepted (auth != authz) and the push rate limiter bounds arrival RATE, not in-flight + // concurrency. Keyed on the resolved source IP, NEVER the signed DID (a DID farm defeats + // a DID key); no resolvable key -> global write pool only. Then the global write permit: + // pushes draw from the dedicated WRITE pool, separate from reads, and it is held for the + // whole op (moved into the AdmissionGuard below). + let caller_key = read_caller_key(&headers, peer, state.push_limiter_trust); + let _caller_permit = acquire_read_caller_permit( + &state.git_write_per_caller, + caller_key.as_deref(), + name, + "receive-pack", + )?; + let _permit = git_permit(&state.git_write_semaphore)?; + tracing::debug!(repo = %name, "acquiring write lock"); // Bound the write acquire under `git_acquire_timeout_secs`. acquire_write's // advisory-lock loop already caps at ~60s, but its per-iteration @@ -6925,4 +6939,140 @@ mod tests { "the lease entry must be freed again after the second clean push" ); } + + /// F3 DoS (P2, RED-before/GREEN-after): a second same-repo push that BLOCKS on the + /// per-repo write lease must hold NO global write permit while it waits. The lease + /// is a block-and-wait serializer, so a lease-blocked waiter can sit for up to + /// steal_after (~a full git_service_timeout window). If it grabs a scarce global + /// write-pool slot BEFORE blocking, a handful of hostile sources can stack same-repo + /// pushes, pin every write slot on lease-waiters sending zero bytes, and shed 503 on + /// every push to every OTHER repo node-wide. The fix acquires the lease BEFORE the + /// two write permits, so a blocked waiter pins no slot. + /// + /// Load-bearing invariant: with the write pool sized to 2, push A holds the lease and + /// is in-flight in receive-pack (1 permit held), and same-repo push B is blocked on + /// the lease, `git_write_semaphore.available_permits()` must stay 1 (only A holds). + /// Pre-fix B takes its permit BEFORE blocking on the lease, draining the pool to 0 + /// (RED). With the reorder B blocks before any permit, so the pool stays at 1 (GREEN). + #[cfg(unix)] + #[sqlx::test] + async fn f3_lease_blocked_waiter_holds_no_write_permit(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + + let tmp = tempfile::TempDir::new().unwrap(); + let seq_a = tmp.path().join("seq_a"); // first receive-pack wins this mkdir = push A + let a_inpack = tmp.path().join("a.inpack"); // set when A reaches receive-pack (holds lease+permit) + let b_ran = tmp.path().join("b.ran"); // set when B's receive-pack runs (B got past the lease) + // receive-pack: first invocation (A) marks that it reached the pack and hangs in a + // bounded loop (so a RED run leaks no permanent orphan); second (B) marks it ran. + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + receive-pack)\n\ + cat >/dev/null 2>/dev/null\n\ + if mkdir \"{seq}\" 2>/dev/null; then\n\ + echo 1 > \"{ainp}\"\n\ + i=0; while [ $i -lt 100 ]; do sleep 0.1; i=$((i+1)); done\n\ + else\n\ + echo 1 > \"{bran}\"\n\ + fi ;;\n\ + rev-parse) echo deadbeef ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + seq = seq_a.display(), + ainp = a_inpack.display(), + bran = b_ran.display(), + ); + let git_bin = write_fake_git(tmp.path(), &body); + let mut state = + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6f3dos", "d1", false).await; + // Size the write pool to 2 so one in-flight holder (A) leaves exactly one slot + // free, and a pool-holding waiter (B, pre-fix) would drain it to zero. Sizing to + // 1 would 503 B on the pool before it could block on the lease, hiding the bug. + state.git_write_semaphore = Arc::new(Semaphore::new(2)); + let did = "did:key:z6MkF3DosPusherAAAAAAAAAAAAAAAAAAAAAAAAAA"; + + // Push A: drive its handler future in slices until it reaches receive-pack (it now + // holds the lease and one write permit and is hung). available_permits() drops to 1. + let mut fut_a = Box::pin(git_receive_pack( + State(state.clone()), + Path(("z6f3dos".to_string(), "d1".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some("203.0.113.71:5000".parse::().unwrap())), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + )); + for _ in 0..1000 { + let _ = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut_a).await; + if a_inpack.exists() { + break; + } + } + assert!( + a_inpack.exists(), + "push A must reach receive-pack and hold the lease" + ); + assert_eq!( + state.git_write_semaphore.available_permits(), + 1, + "with the pool sized to 2, the single in-flight holder (A) leaves one slot free" + ); + + // Push B: a DIFFERENT source, same repo. It blocks on the lease A holds. + let state_b = state.clone(); + let handle_b = tokio::spawn(async move { + git_receive_pack( + State(state_b), + Path(("z6f3dos".to_string(), "d1".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some( + "203.0.113.72:5000".parse::().unwrap(), + )), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await + }); + + // Load-bearing check: while B is a lease-blocked waiter the pool must stay at 1 + // (only A holds a permit). Poll the invariant across a full window; pre-fix B + // grabs the last slot within ms and the pool falls to 0 (RED), post-fix it never + // does (GREEN). A stable state, not a one-shot race: B stays blocked on the lease + // (steal_after is far larger than this window) so once it settles the pool holds. + for _ in 0..100 { + assert_eq!( + state.git_write_semaphore.available_permits(), + 1, + "F3 DoS RED: a lease-blocked same-repo waiter took a global write permit \ + while sending zero bytes, draining the pool — a blocked waiter must pin \ + no write-pool slot" + ); + assert!( + !b_ran.exists(), + "B must not have run receive-pack while A holds the lease" + ); + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + // Non-vacuous: B is genuinely parked on the lease, not returned early. + assert!( + !handle_b.is_finished(), + "push B must still be blocked on the lease at this point" + ); + + // Client disconnect on A: drop its future. The write-path AdmissionGuard rides the + // reaper, freeing the lease once A's group is reaped; B then proceeds. + drop(fut_a); + let resp = tokio::time::timeout(std::time::Duration::from_secs(30), handle_b) + .await + .expect("push B must complete once A's group is reaped and the lease frees") + .expect("push B task must not panic") + .expect("push B must succeed"); + assert_eq!(resp.status(), 200, "push B lands 200 after serialization"); + assert!(b_ran.exists(), "push B ran its receive-pack once unblocked"); + } } From e2be6f3019a3fe2b6d941f0d94722fb10843ef2d Mon Sep 17 00:00:00 2001 From: t Date: Mon, 20 Jul 2026 11:54:09 -0500 Subject: [PATCH 57/58] fix(node): don't 500 an /ipfs miss when a transient taint co-occurs (#174 F5 review) The deterministic-fault terminal 500 was checked before the transient truncated_by 503, unconditionally. When one repo is corrupt (deterministic) while a DIFFERENT repo is transiently skipped in the same scan, the requested CID may live in the transient repo -- a retryable 503 is correct, but the handler returned a terminal 500 and a conformant client won't retry, hiding retrievable content until the unrelated corrupt repo is repaired. Gate the 500 on truncated_by.is_empty(): it fires only when a deterministic fault is the SOLE reason nothing served; a co-occurring transient taint falls through to the retryable 503. A pure deterministic fault still terminally 500s an absent lookup by design. Adds a co-occurrence regression test. --- crates/gitlawb-node/src/api/ipfs.rs | 126 ++++++++++++++++++++++++++-- 1 file changed, 120 insertions(+), 6 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index 7f977a81..5e7ae05a 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -572,12 +572,19 @@ pub async fn get_by_cid( // repo / bad `.git/config`), so the object is not proven absent AND a retry cannot // change that. Shed a TERMINAL, non-retryable 500 rather than the retryable 503 // below — a 503 would let a conformant client retry-storm a fresh `git cat-file` - // per attempt against the broken repo. This is checked before the transient 503 so - // a deterministic fault is never downgraded to a retryable status. The body is - // opaque (a generic message via `AppError::Git` -> 500, no Retry-After): the raw - // git stderr — which leaks filesystem paths / config — was logged at the probe, and - // never reaches the client. - if deterministic_fault { + // per attempt against the broken repo. The body is opaque (a generic message via + // `AppError::Git` -> 500, no Retry-After): the raw git stderr — which leaks + // filesystem paths / config — was logged at the probe, and never reaches the client. + // + // Gate on `truncated_by.is_empty()`: the terminal 500 fires only when a + // deterministic fault is the SOLE reason nothing served. When a TRANSIENT taint + // co-occurs (a DIFFERENT repo was skipped by budget / acquire / walk-cap / probe / + // visit-ceiling), the requested object may live in that transiently-skipped repo, + // so fall through to the retryable 503 below — a retry can re-probe it and serve + // the content. Reporting a terminal 500 there would wrongly tell a conformant + // client not to retry, leaving reachable content unreachable until the unrelated + // broken repo is repaired. + if deterministic_fault && truncated_by.is_empty() { return Err(AppError::Git( "ipfs object probe could not complete: a candidate repository is corrupt".into(), )); @@ -1360,6 +1367,113 @@ mod tests { ); } + /// #174 F5 co-occurrence (RED-before/GREEN-after): a deterministic fault on ONE + /// repo and a TRANSIENT taint on a DIFFERENT repo occur in the same scan, and the + /// requested CID is served by neither. The transiently-skipped repo could hold the + /// object, so a retry can surface it — the correct shed is the RETRYABLE 503, not + /// the terminal 500. Two broken repos drive it, both local so the outcome is + /// deterministic: a bad-`config` repo whose `objects/` stays readable is a + /// DETERMINISTIC probe fault (`deterministic_fault = true`), while a repo whose + /// `objects/` dir is removed is a TRANSIENT probe fault (taints "probe"). A third + /// healthy repo probes clean (absent verdict) so nothing serves. Before the fix the + /// terminal `if deterministic_fault` arm fired first and shed 500 unconditionally, + /// hiding the transiently-skipped repo behind a non-retryable status. MUTATION + /// (RED): drop the `&& truncated_by.is_empty()` gate and this shes 500 again. + #[sqlx::test] + async fn get_by_cid_deterministic_fault_with_cooccurring_transient_taint_is_503_not_500( + pool: sqlx::PgPool, + ) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + // Healthy repo that probes clean (definitive absence on its own). + seed_repo_with_blob(&state, tmp.path(), "z6f5coclean", "real", b"probe clean\n").await; + + // Bad-config repo: objects/ readable, config corrupt -> DETERMINISTIC fault. + state + .db + .upsert_mirror_repo("z6f5cobadcfg", "broken", "/unused-badcfg", None, false) + .await + .unwrap(); + let rec = state + .db + .get_repo("z6f5cobadcfg", "broken") + .await + .unwrap() + .unwrap(); + let bare = state + .repo_store + .acquire(&rec.owner_did, &rec.name) + .await + .unwrap(); + std::fs::create_dir_all(&bare).unwrap(); + run_git(&["init", "-q", "--bare", "--object-format=sha256"], &bare); + { + use std::io::Write; + let mut cfg = std::fs::OpenOptions::new() + .append(true) + .open(bare.join("config")) + .unwrap(); + cfg.write_all(b"\n[broken section\nnot a valid = = = line\n") + .unwrap(); + } + + // Corrupt-dir repo: objects/ removed -> TRANSIENT probe fault (taints "probe"), + // a DIFFERENT repo than the deterministic one above. + state + .db + .upsert_mirror_repo("z6f5cocorrupt", "broken", "/unused-corrupt", None, false) + .await + .unwrap(); + let rec2 = state + .db + .get_repo("z6f5cocorrupt", "broken") + .await + .unwrap() + .unwrap(); + let bare2 = state + .repo_store + .acquire(&rec2.owner_did, &rec2.name) + .await + .unwrap(); + std::fs::create_dir_all(&bare2).unwrap(); + run_git(&["init", "-q", "--bare", "--object-format=sha256"], &bare2); + std::fs::remove_dir_all(bare2.join("objects")).unwrap(); + std::fs::write(bare2.join("HEAD"), b"junk\n").unwrap(); + + let peer: SocketAddr = "203.0.113.71:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&valid_cid(), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a deterministic fault co-occurring with a transient taint on a DIFFERENT \ + repo must shed the retryable 503 (a retry can surface the object in the \ + transiently-skipped repo), never the terminal 500" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the co-occurrence 503 must carry Retry-After" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + let body = String::from_utf8_lossy(&body); + assert!( + body.contains("probe"), + "the shed must name the transient probe taint; got: {body}" + ); + } + /// F2 read taint: the gate passes (the probe reads the truncated loose object's /// intact "blob 64" header) but the content read fails (`cat-file blob` dies on /// the deflate stream cut mid-content) — the probe just said the object EXISTS From e6f1fec861bb840e6883c92db1315cccce3aadb1 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 20 Jul 2026 12:02:46 -0500 Subject: [PATCH 58/58] docs(node): correct three lease/replication comments from code review (#174) - steal_after: the reclaim is NOT a guarantee that only a leaked lease is reclaimed. A waiter's timeout starts at acquire(), not at the FIFO head, so a same-repo backlog whose cumulative wait exceeds steal_after can steal while an earlier waiter still writes; correctness rests on the retained pg advisory lock (which serializes the stealer at acquire_write), not on the bound. - pinata_object_list_for_refs: recomputes from the tail-start rules snapshot, not a fresh read at pin-worker time, so a rule tightened AFTER tail-start is not reflected (matches the old retained-list behavior; reconciliation sweep is the backstop). Removed the overstated 'rule tightened since the push is honored'. - RepoWriteGuard::Drop: note that on runtime shutdown the detached unlock task may be dropped before it polls, bounded by pool teardown / connection close. --- crates/gitlawb-node/src/api/repos.rs | 22 +++++++++++++++------- crates/gitlawb-node/src/git/repo_store.rs | 5 ++++- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 836e5625..f258a4bc 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1007,10 +1007,13 @@ async fn resolve_drain_object_list( /// retained object list is dropped here, not the task, so every push's effects /// still fire exactly once. /// -/// Fresh by design, exactly like `resolve_drain_object_list`: the withheld set -/// and candidate set are recomputed from the current rules, so a rule tightened -/// since the push is honored and fails closed (a newly-withheld blob is not -/// pinned; a no-longer-announceable repo pins nothing). Every git child runs +/// Exactly like `resolve_drain_object_list`: the withheld and candidate sets are +/// recomputed from the rules snapshot captured at post-receive tail start (NOT a +/// fresh read at pin-worker time), so a rule tightened up to that point is honored +/// and the filter always fails closed (a newly-withheld blob is not pinned; a +/// no-longer-announceable repo pins nothing). A tightening AFTER tail-start — before +/// a slow re-derivation runs — is not reflected, matching the old retained-list +/// behavior; the reconciliation sweep is the backstop. Every git child runs /// through the same INV-22 bounded, process-group-reaped helpers the sibling /// post-receive scans use (`replication_withheld_set`, /// `resolve_candidates_for_push`, `fail_closed_full_scan_objects`). @@ -1624,9 +1627,14 @@ pub async fn git_receive_pack( // no pooled pg connection while it waits. The lease rides the write-path // AdmissionGuard into the reaper (clone (a)) and spans the clean-path Tigris upload // in guard.release (clone (b)); it frees only when the LAST clone drops. steal_after - // is set well above any legitimate hold (a full receive-pack under - // git_service_timeout + the ~4s reaper cap + the Tigris upload), so the bounded-wait - // reclaim only ever fires for a genuinely leaked lease, never a merely-slow push. + // is sized above ONE legitimate hold (a full receive-pack under git_service_timeout + + // the ~4s reaper cap + the Tigris upload). It is NOT a guarantee that only a leaked + // lease is reclaimed: a waiter's timeout starts at acquire(), not at the head of the + // FIFO queue, so a same-repo backlog whose CUMULATIVE wait exceeds steal_after can + // steal while an earlier waiter is still writing. Correctness does not rest on the + // bound — on the non-disconnect path the retained pg advisory lock still serializes + // the stealer at acquire_write (a spurious 503, not a race); the only corruption-capable + // overlap is the ~4s disconnect/reap window, which the reaper-carried clone (a) covers. let lease_steal_after = std::time::Duration::from_secs(state.config.git_service_timeout_secs * 2 + 60); let lease = state diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index d88594e7..bd27a186 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -460,7 +460,10 @@ impl Drop for RepoWriteGuard { /// handler future was dropped before `release`), unlock on the pinned /// connection. `Drop` cannot await, so spawn a detached unlock — it runs on the /// same session (connection-affine). An off-runtime drop falls back to a log; - /// the ~60s stale-lock retry loop in `acquire_write` reclaims it. + /// the ~60s stale-lock retry loop in `acquire_write` reclaims it. On runtime + /// SHUTDOWN the spawned unlock task may be dropped before it polls, so the unlock + /// may not run — but shutdown tears down the pool, and closing the connection + /// releases the session-level advisory lock server-side, so this too is bounded. fn drop(&mut self) { if self.released || !self.locked { return;